1. ImageReader
1.1 概述
ImageReader允许应用程序直接获取渲染到surface的图形数据，并转换为图片，这里我用Presentation+VirtualDisplay+ImageReader做一个Demo来探讨ImageReader的实现原理。
Demo实现的功能是：Presentation指定显示在VirtualDisplay上，然后由ImageReader去获取VirtualDisplay上渲染的Presentation图像，获取之后将其转换为Bitmap，设置到一个ImageView中，并且可以通过修改Presentation的UI数据实现ImageView的刷新。
1.2 Demo代码
MainActivity onCreate：
@Override
protected void onCreate(Bundle savedInstanceState) {
super.onCreate(savedInstanceState);
setContentView(R.layout.activity_main);
mDisplayManager = (DisplayManager)this.getSystemService(Context.DISPLAY_SERVICE);
getWindowManager().getDefaultDisplay().getMetrics(outMetrics);
findViewById(.button).setOnClickListener(new View.OnClickListener() {
@Override
public void onClick(View v) {
//测试ImageReader
testVirtualDisplay();
}
});
imageView = findViewById(.image);
}testVirtualDisplay：
private void testVirtualDisplay(){
final int screenWidth = outMetrics.widthPixels;
final int screenHeight = outMetrics.heightPixels;
@SuppressLint("WrongConstant") ImageReader imageReader = ImageReader.newInstance(
screenWidth,
screenHeight,
PixelFormat.RGBA_8888, 1);
//创建虚拟屏，传入ImageReader的Surface
VirtualDisplay virtualDisplay = mDisplayManager.createVirtualDisplay("testVirtualDisplay",
WIDTH, HEIGHT, DENSITY, imageReader.getSurface(), DisplayManager.VIRTUAL_DISPLAY_FLAG_PRESENTATION);
//设置回调
imageReader.setOnImageAvailableListener(new ImageReader.OnImageAvailableListener() {
//回调中获取Surface的渲染数据
@Override
public void onImageAvailable(ImageReader reader) {
mImageReaderLock.lock();
try {
// Get the latest buffer.
Image image = reader.acquireLatestImage();
Log.d("dongjiao", "11image = :"+image);
if (image != null) {
Image.Plane[] planes = image.getPlanes();
ByteBuffer buffer = planes[0].getBuffer();
int pixelStride = planes[0].getPixelStride();
int rowStride = planes[0].getRowStride();
int rowPadding = rowStride - pixelStride * screenWidth;
Bitmap bitmap = Bitmap.createBitmap(screenWidth + rowPadding / pixelStride,
screenHeight, Bitmap.Config.ARGB_8888);
bitmap.copyPixelsFromBuffer(buffer);
//virtualDisplay.release();
imageView.setImageBitmap(bitmap);
image.close();
}
} finally {
mImageReaderLock.unlock();
}
}
}, handler);
DisplayManager displayManager = (DisplayManager) this.getSystemService(Context.DISPLAY_SERVICE);
Display[] presentationDisplays = displayManager.getDisplays(DisplayManager.DISPLAY_CATEGORY_PRESENTATION);
if (presentationDisplays.length > 0) {
Display display = presentationDisplays[0];
//TestPresentation指定显示在虚拟屏
TestPresentation presentation = new TestPresentation(this, display);
presentation.show();
findViewById(.button1).setOnClickListener(new View.OnClickListener() {
@Override
public void onClick(View v) {
//更新TestPresentation的UI
presentation.changeText();
}
});
}
}Demo代码很简单，首先MainActivity有两个button，BUTTON创建ImageReader，创建虚拟屏，设置ImageReader回调，创建TestPresentation，并将TestPresentation显示在虚拟屏上，此时ImageReader回调中就会收到TestPresentation渲染的UI数据，拿到数据之后将其转换为Bitmap，并设置到ImageView中。
BUTTON1会更新TestPresentation的UI数据，更新之后同样会回调ImageReader，进而再次更新ImageView。
1.3 图像数据流分析
来看TestPresentation的图像是如何传递到ImageReader的。
1.3.1 ImageReader初始化
protected ImageReader(int width, int height, int format, int maxImages, long usage) {
......
nativeInit(new WeakReference<>(this), width, height, format, maxImages, usage);
mSurface = nativeGetSurface();
......
}这里关注两个方法，nativeInit和nativeGetSurface，maxImages代码能同时获取buffer的数量，最小为1，
1.3.2 ImageReader_init
static void ImageReader_init(JNIEnv* env, jobject thiz, jobject weakThiz, jint width, jint height,
jint format, jint maxImages, jlong ndkUsage)
{
status_t res;
...
sp<JNIImageReaderContext> ctx(new JNIImageReaderContext(env, weakThiz, clazz, maxImages));
sp<IGraphicBufferProducer> gbProducer;
sp<IGraphicBufferConsumer> gbConsumer;
//创建BufferQueue
BufferQueue::createBufferQueue(&gbProducer, &gbConsumer);
//自定义Consumer
sp<BufferItemConsumer> bufferConsumer;
String8 consumerName = String8::format("ImageReader-%dx%df%xm%d-%d-%d",
width, height, format, maxImages, getpid(),
createProcessUniqueId());
......
ctx->setBufferConsumer(bufferConsumer);
bufferConsumer->setName(consumerName);
ctx->setProducer(gbProducer);
bufferConsumer->setFrameAvailableListener(ctx);
ImageReader_setNativeContext(env, thiz, ctx);
ctx->setBufferFormat(nativeFormat);
ctx->setBufferDataspace(nativeDataspace);
ctx->setBufferWidth(width);
ctx->setBufferHeight(height);
....
}
}这里核心是创建了ImageReader的native层对象JNIImageReaderContext，并且创建了一套生产者消费者模型，自定义了Consumer（BufferItemConsumer），
最重要的是setFrameAvailableListener的对象是JNIImageReaderContext，说明JNIImageReaderContext实现了onFrameAvailable接口，看看其具体实现：
void JNIImageReaderContext::onFrameAvailable(const BufferItem& /*item*/)
{
ALOGV("%s: frame available", __FUNCTION__);
bool needsDetach = false;
JNIEnv* env = getJNIEnv(&needsDetach);
if (env != NULL) {
env->CallStaticVoidMethod(mClazz, gImageReaderClassInfo.postEventFromNative, mWeakThiz);
} else {
ALOGW("onFrameAvailable event will not posted");
}
if (needsDetach) {
detachJNI();
}
}onFrameAvailable会回调到java层ImageReader的postEventFromNative方法，等下在回到java层看它的具体实现，先看Surface是怎么构建的：
1.3.3 ImageReader_getSurface
static jobject ImageReader_getSurface(JNIEnv* env, jobject thiz)
{
ALOGV("%s: ", __FUNCTION__);
//从JNIImageReaderContext中获取IGraphicBufferProducer
IGraphicBufferProducer* gbp = ImageReader_getProducer(env, thiz);
if (gbp == NULL) {
jniThrowRuntimeException(env, "Buffer consumer is uninitialized");
return NULL;
}
// 用Surface对IGraphicBufferProducer进行包装
return android_view_Surface_createFromIGraphicBufferProducer(env, gbp);
}
jobject android_view_Surface_createFromIGraphicBufferProducer(JNIEnv* env,
const sp<IGraphicBufferProducer>& bufferProducer) {
if (bufferProducer == NULL) {
return NULL;
}
sp<Surface> surface(new Surface(bufferProducer, true));
return android_view_Surface_createFromSurface(env, surface);
}这里就很简单了，new了一个Surface对IGraphicBufferProducer进行包装，这个Surface最终就是传给虚拟屏的。
接着回到java层来看postEventFromNative的实现：
1.3.4 ImageReader.postEventFromNative
private static void postEventFromNative(Object selfRef) {
@SuppressWarnings("unchecked")
WeakReference<ImageReader> weakSelf = (WeakReference<ImageReader>)selfRef;
final ImageReader ir = weakSelf.get();
if (ir == null) {
return;
}
final Handler handler;
synchronized (ir.mListenerLock) {
handler = ir.mListenerHandler;
}
if (handler != null) {
handler.sendEmptyMessage(0);
}
}这里通过ListenerHandler发送消息：
1.3.5 ListenerHandler.handleMessage
private final class ListenerHandler extends Handler {
public ListenerHandler(Looper looper) {
super(looper, null, true /*async*/);
}
@Override
public void handleMessage(Message msg) {
OnImageAvailableListener listener;
synchronized (mListenerLock) {
listener = mListener;
}
// It's dangerous to fire onImageAvailable() callback when the ImageReader is being
// closed, as application could acquire next image in the onImageAvailable() callback.
boolean isReaderValid = false;
synchronized (mCloseLock) {
isReaderValid = mIsReaderValid;
}
if (listener != null && isReaderValid) {
listener.onImageAvailable(ImageReader.this);
}
}
}看到了吗，回调了OnImageAvailableListener的onImageAvailable方法，这个就是前面Demo中我们给ImageReader设置的回调，回过头再看Demo，在onImageAvailable方法中做了什么：
ImageReader.OnImageAvailableListener() {
//回调中获取Surface的渲染数据
@Override
public void onImageAvailable(ImageReader reader) {
mImageReaderLock.lock();
try {
Image image = reader.acquireLatestImage();
Log.d("dongjiao", "11image = :"+image);
if (image != null) {
Image.Plane[] planes = image.getPlanes();
ByteBuffer buffer = planes[0].getBuffer();
int pixelStride = planes[0].getPixelStride();
int rowStride = planes[0].getRowStride();
int rowPadding = rowStride - pixelStride * screenWidth;
Bitmap bitmap = Bitmap.createBitmap(screenWidth + rowPadding / pixelStride,
screenHeight, Bitmap.Config.ARGB_8888);
bitmap.copyPixelsFromBuffer(buffer);
//virtualDisplay.release();
imageView.setImageBitmap(bitmap);
image.close();
}
} finally {
mImageReaderLock.unlock();
}
}
}, handler);调用了acquireLatestImage获取数据：
1.3.6 ImageReader.acquireLatestImage
public Image acquireLatestImage() {
Image image = acquireNextImage();
if (image == null) {
return null;
}
try {
for (;;) {
Image next = acquireNextImageNoThrowISE();
if (next == null) {
Image result = image;
image = null;
return result;
}
image.close();
image = next;
}
} finally {
if (image != null) {
image.close();
}
}
}acquireLatestImage会获取最新的一帧，所以里面有个循环，直到下一帧为空才返回，acquireNextImage和acquireNextImageNoThrowISE最终都是调用的acquireNextSurfaceImage，
1.3.7 ImageReader.acquireNextSurfaceImage
private int acquireNextSurfaceImage(SurfaceImage si) {
synchronized (mCloseLock) {
// A null image will eventually be returned if ImageReader is already closed.
int status = ACQUIRE_NO_BUFS;
if (mIsReaderValid) {
status = nativeImageSetup(si);
}
switch (status) {
case ACQUIRE_SUCCESS:
si.mIsImageValid = true;
case ACQUIRE_NO_BUFS:
case ACQUIRE_MAX_IMAGES:
break;
default:
throw new AssertionError("Unknown nativeImageSetup return code " + status);
}
// Only keep track the successfully acquired image, as the native buffer is only mapped
// for such case.
if (status == ACQUIRE_SUCCESS) {
mAcquiredImages.add(si);
}
return status;
}
}调用到native层去获取帧的数据，参数SurfaceImage用于保存帧的指针，
1.3.8 ImageReader_imageSetup
static jint ImageReader_imageSetup(JNIEnv* env, jobject thiz, jobject image) {
JNIImageReaderContext* ctx = ImageReader_getContext(env, thiz);
if (ctx == NULL) {
jniThrowException(env, "java/lang/IllegalStateException",
"ImageReader is not initialized or was already closed");
return -1;
}
//获取自定义BufferItemConsumer
BufferItemConsumer* bufferConsumer = ctx->getBufferConsumer();
//保证maxImages的作用，同一时间能够获取的maxImages数量的buffer
//当buffer为空时，需要Image close之后才能继续获取
BufferItem* buffer = ctx->getBufferItem();
if (buffer == NULL) {
ALOGW("Unable to acquire a buffer item, very likely client tried to acquire more than"
" maxImages buffers");
return ACQUIRE_MAX_IMAGES;
}
//获取buffer
status_t res = bufferConsumer->acquireBuffer(buffer, 0);
......
// 将获取到的buffer指针保存到java层传下来的SurfaceImage中
Image_setBufferItem(env, image, buffer);
//将buffer的一些参数保存到SurfaceImage中
env->SetLongField(image, gSurfaceImageClassInfo.mTimestamp,
static_cast<jlong>(buffer->mTimestamp));
auto transform = buffer->mTransform;
if (buffer->mTransformToDisplayInverse) {
transform |= NATIVE_WINDOW_TRANSFORM_INVERSE_DISPLAY;
}
env->SetIntField(image, gSurfaceImageClassInfo.mTransform,
static_cast<jint>(transform));
env->SetIntField(image, gSurfaceImageClassInfo.mScalingMode,
static_cast<jint>(buffer->mScalingMode));
return ACQUIRE_SUCCESS;
}acquireBuffer用于获取当前可用的buffer
1.3.9 BufferItemConsumer.acquireBuffer
status_t BufferItemConsumer::acquireBuffer(BufferItem *item,
nsecs_t presentWhen, bool waitForFence) {
status_t err;
if (!item) return BAD_VALUE;
Mutex::Autolock _l(mMutex);
//获取buffer
err = acquireBufferLocked(item, presentWhen);
if (err != OK) {
if (err != NO_BUFFER_AVAILABLE) {
BI_LOGE("Error acquiring buffer: %s (%d)", strerror(err), err);
}
return err;
}
//传入的waitForFence等于0，不需要等Fence
if (waitForFence) {
err = item->mFence->waitForever("BufferItemConsumer::acquireBuffer");
if (err != OK) {
BI_LOGE("Failed to wait for fence of acquired buffer: %s (%d)",
strerror(-err), err);
return err;
}
}
item->mGraphicBuffer = mSlots[item->mSlot].mGraphicBuffer;
return OK;
}真正获取buffer还是通过父类ConsumerBase，之后就到SurfaceFlinger流程了，后续buffer到java层之后还需要各种格式转换等处理，变成一个 ByteBuffer ，就可以写入Bitmap了，最后将此Bitmap设置到ImageView就完成了Demo的效果，Demo中的刷新UI不过是再走了一遍整个流程。
再来整理一遍ImageReader的数据流：
首先native层JNIImageReaderContext依靠onFrameAvailable接收到BufferQueue中有可用buffer的信号，接着通过JNI调用postEventFromNative告知java层ImageReader，postEventFromNative又继续通过回调onImageAvailable告知某个注册了OnImageAvailableListener的客户端，客户端在onImageAvailable中通过acquireLatestImage去获取BufferQueue中可用的buffer，最终是通过acquireBufferLocked获取，拿到的buffer之后就可以做自己的处理了。
这是ImageReader的数据流，那么数据是怎么流向ImageReader的呢？
ImageReader的数据来自VirtualDisplay，流向是这样的：
首先VirtualDisplay创建时接收了ImageReader的Surface，APP（Presentation）是一个可以显示在其他Display的Dialog，当它显示在VirtualDisplay时，SurfaceFlinger就会将它合成的数据渲染在VirtualDisplay的Surface上，因为此时VirtualDisplay的Surface来自ImageReader，自然渲染的数据就被放入了ImageReader的Surface的BufferQueue中，所以ImageReader就收到了
onFrameAvailable回调。
本文章为转载内容，我们尊重原作者对文章享有的著作权。如有内容错误或侵权问题，欢迎原作者联系我们进行内容更正或删除文章。
