1 /* 2 * Copyright 2013 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17 //#define LOG_NDEBUG 0 18 #define LOG_TAG "ImageReader_JNI" 19 #include "android_media_Utils.h" 20 #include <cutils/atomic.h> 21 #include <utils/Log.h> 22 #include <utils/misc.h> 23 #include <utils/List.h> 24 #include <utils/String8.h> 25 26 #include <cstdio> 27 28 #include <gui/BufferItemConsumer.h> 29 #include <gui/Surface.h> 30 31 #include <android_runtime/AndroidRuntime.h> 32 #include <android_runtime/android_view_Surface.h> 33 #include <android_runtime/android_hardware_HardwareBuffer.h> 34 #include <grallocusage/GrallocUsageConversion.h> 35 36 #include <private/android/AHardwareBufferHelpers.h> 37 38 #include <jni.h> 39 #include <nativehelper/JNIHelp.h> 40 41 #include <stdint.h> 42 #include <inttypes.h> 43 #include <android/hardware_buffer_jni.h> 44 45 #define ANDROID_MEDIA_IMAGEREADER_CTX_JNI_ID "mNativeContext" 46 #define ANDROID_MEDIA_SURFACEIMAGE_BUFFER_JNI_ID "mNativeBuffer" 47 #define ANDROID_MEDIA_SURFACEIMAGE_TS_JNI_ID "mTimestamp" 48 #define ANDROID_MEDIA_SURFACEIMAGE_TF_JNI_ID "mTransform" 49 #define ANDROID_MEDIA_SURFACEIMAGE_SM_JNI_ID "mScalingMode" 50 51 #define CONSUMER_BUFFER_USAGE_UNKNOWN 0; 52 // ---------------------------------------------------------------------------- 53 54 using namespace android; 55 56 57 enum { 58 ACQUIRE_SUCCESS = 0, 59 ACQUIRE_NO_BUFFERS = 1, 60 ACQUIRE_MAX_IMAGES = 2, 61 }; 62 63 static struct { 64 jfieldID mNativeContext; 65 jmethodID postEventFromNative; 66 } gImageReaderClassInfo; 67 68 static struct { 69 jfieldID mNativeBuffer; 70 jfieldID mTimestamp; 71 jfieldID mTransform; 72 jfieldID mScalingMode; 73 jfieldID mPlanes; 74 } gSurfaceImageClassInfo; 75 76 static struct { 77 jclass clazz; 78 jmethodID ctor; 79 } gSurfacePlaneClassInfo; 80 81 // Get an ID that's unique within this process. 82 static int32_t createProcessUniqueId() { 83 static volatile int32_t globalCounter = 0; 84 return android_atomic_inc(&globalCounter); 85 } 86 87 // ---------------------------------------------------------------------------- 88 89 class JNIImageReaderContext : public ConsumerBase::FrameAvailableListener 90 { 91 public: 92 JNIImageReaderContext(JNIEnv* env, jobject weakThiz, jclass clazz, int maxImages); 93 94 virtual ~JNIImageReaderContext(); 95 96 virtual void onFrameAvailable(const BufferItem& item); 97 98 BufferItem* getBufferItem(); 99 void returnBufferItem(BufferItem* buffer); 100 101 102 void setBufferConsumer(const sp<BufferItemConsumer>& consumer) { mConsumer = consumer; } 103 BufferItemConsumer* getBufferConsumer() { return mConsumer.get(); } 104 105 void setProducer(const sp<IGraphicBufferProducer>& producer) { mProducer = producer; } 106 IGraphicBufferProducer* getProducer() { return mProducer.get(); } 107 108 void setBufferFormat(int format) { mFormat = format; } 109 int getBufferFormat() { return mFormat; } 110 111 void setBufferDataspace(android_dataspace dataSpace) { mDataSpace = dataSpace; } 112 android_dataspace getBufferDataspace() { return mDataSpace; } 113 114 void setBufferWidth(int width) { mWidth = width; } 115 int getBufferWidth() { return mWidth; } 116 117 void setBufferHeight(int height) { mHeight = height; } 118 int getBufferHeight() { return mHeight; } 119 120 private: 121 static JNIEnv* getJNIEnv(bool* needsDetach); 122 static void detachJNI(); 123 124 List<BufferItem*> mBuffers; 125 sp<BufferItemConsumer> mConsumer; 126 sp<IGraphicBufferProducer> mProducer; 127 jobject mWeakThiz; 128 jclass mClazz; 129 int mFormat; 130 android_dataspace mDataSpace; 131 int mWidth; 132 int mHeight; 133 }; 134 135 JNIImageReaderContext::JNIImageReaderContext(JNIEnv* env, 136 jobject weakThiz, jclass clazz, int maxImages) : 137 mWeakThiz(env->NewGlobalRef(weakThiz)), 138 mClazz((jclass)env->NewGlobalRef(clazz)), 139 mFormat(0), 140 mDataSpace(HAL_DATASPACE_UNKNOWN), 141 mWidth(-1), 142 mHeight(-1) { 143 for (int i = 0; i < maxImages; i++) { 144 BufferItem* buffer = new BufferItem; 145 mBuffers.push_back(buffer); 146 } 147 } 148 149 JNIEnv* JNIImageReaderContext::getJNIEnv(bool* needsDetach) { 150 LOG_ALWAYS_FATAL_IF(needsDetach == NULL, "needsDetach is null!!!"); 151 *needsDetach = false; 152 JNIEnv* env = AndroidRuntime::getJNIEnv(); 153 if (env == NULL) { 154 JavaVMAttachArgs args = {JNI_VERSION_1_4, NULL, NULL}; 155 JavaVM* vm = AndroidRuntime::getJavaVM(); 156 int result = vm->AttachCurrentThread(&env, (void*) &args); 157 if (result != JNI_OK) { 158 ALOGE("thread attach failed: %#x", result); 159 return NULL; 160 } 161 *needsDetach = true; 162 } 163 return env; 164 } 165 166 void JNIImageReaderContext::detachJNI() { 167 JavaVM* vm = AndroidRuntime::getJavaVM(); 168 int result = vm->DetachCurrentThread(); 169 if (result != JNI_OK) { 170 ALOGE("thread detach failed: %#x", result); 171 } 172 } 173 174 BufferItem* JNIImageReaderContext::getBufferItem() { 175 if (mBuffers.empty()) { 176 return NULL; 177 } 178 // Return a BufferItem pointer and remove it from the list 179 List<BufferItem*>::iterator it = mBuffers.begin(); 180 BufferItem* buffer = *it; 181 mBuffers.erase(it); 182 return buffer; 183 } 184 185 void JNIImageReaderContext::returnBufferItem(BufferItem* buffer) { 186 buffer->mGraphicBuffer = nullptr; 187 mBuffers.push_back(buffer); 188 } 189 190 JNIImageReaderContext::~JNIImageReaderContext() { 191 bool needsDetach = false; 192 JNIEnv* env = getJNIEnv(&needsDetach); 193 if (env != NULL) { 194 env->DeleteGlobalRef(mWeakThiz); 195 env->DeleteGlobalRef(mClazz); 196 } else { 197 ALOGW("leaking JNI object references"); 198 } 199 if (needsDetach) { 200 detachJNI(); 201 } 202 203 // Delete buffer items. 204 for (List<BufferItem *>::iterator it = mBuffers.begin(); 205 it != mBuffers.end(); it++) { 206 delete *it; 207 } 208 209 if (mConsumer != 0) { 210 mConsumer.clear(); 211 } 212 } 213 214 void JNIImageReaderContext::onFrameAvailable(const BufferItem& /*item*/) 215 { 216 ALOGV("%s: frame available", __FUNCTION__); 217 bool needsDetach = false; 218 JNIEnv* env = getJNIEnv(&needsDetach); 219 if (env != NULL) { 220 env->CallStaticVoidMethod(mClazz, gImageReaderClassInfo.postEventFromNative, mWeakThiz); 221 } else { 222 ALOGW("onFrameAvailable event will not posted"); 223 } 224 if (needsDetach) { 225 detachJNI(); 226 } 227 } 228 229 // ---------------------------------------------------------------------------- 230 231 extern "C" { 232 233 static JNIImageReaderContext* ImageReader_getContext(JNIEnv* env, jobject thiz) 234 { 235 JNIImageReaderContext *ctx; 236 ctx = reinterpret_cast<JNIImageReaderContext *> 237 (env->GetLongField(thiz, gImageReaderClassInfo.mNativeContext)); 238 return ctx; 239 } 240 241 static IGraphicBufferProducer* ImageReader_getProducer(JNIEnv* env, jobject thiz) 242 { 243 ALOGV("%s:", __FUNCTION__); 244 JNIImageReaderContext* const ctx = ImageReader_getContext(env, thiz); 245 if (ctx == NULL) { 246 jniThrowRuntimeException(env, "ImageReaderContext is not initialized"); 247 return NULL; 248 } 249 250 return ctx->getProducer(); 251 } 252 253 static void ImageReader_setNativeContext(JNIEnv* env, 254 jobject thiz, sp<JNIImageReaderContext> ctx) 255 { 256 ALOGV("%s:", __FUNCTION__); 257 JNIImageReaderContext* const p = ImageReader_getContext(env, thiz); 258 if (ctx != 0) { 259 ctx->incStrong((void*)ImageReader_setNativeContext); 260 } 261 if (p) { 262 p->decStrong((void*)ImageReader_setNativeContext); 263 } 264 env->SetLongField(thiz, gImageReaderClassInfo.mNativeContext, 265 reinterpret_cast<jlong>(ctx.get())); 266 } 267 268 static BufferItemConsumer* ImageReader_getBufferConsumer(JNIEnv* env, jobject thiz) 269 { 270 ALOGV("%s:", __FUNCTION__); 271 JNIImageReaderContext* const ctx = ImageReader_getContext(env, thiz); 272 if (ctx == NULL) { 273 jniThrowRuntimeException(env, "ImageReaderContext is not initialized"); 274 return NULL; 275 } 276 277 return ctx->getBufferConsumer(); 278 } 279 280 static void Image_setBufferItem(JNIEnv* env, jobject thiz, 281 const BufferItem* buffer) 282 { 283 env->SetLongField(thiz, gSurfaceImageClassInfo.mNativeBuffer, reinterpret_cast<jlong>(buffer)); 284 } 285 286 static BufferItem* Image_getBufferItem(JNIEnv* env, jobject image) 287 { 288 return reinterpret_cast<BufferItem*>( 289 env->GetLongField(image, gSurfaceImageClassInfo.mNativeBuffer)); 290 } 291 292 293 // ---------------------------------------------------------------------------- 294 295 static void ImageReader_classInit(JNIEnv* env, jclass clazz) 296 { 297 ALOGV("%s:", __FUNCTION__); 298 299 jclass imageClazz = env->FindClass("android/media/ImageReader$SurfaceImage"); 300 LOG_ALWAYS_FATAL_IF(imageClazz == NULL, 301 "can't find android/graphics/ImageReader$SurfaceImage"); 302 gSurfaceImageClassInfo.mNativeBuffer = env->GetFieldID( 303 imageClazz, ANDROID_MEDIA_SURFACEIMAGE_BUFFER_JNI_ID, "J"); 304 LOG_ALWAYS_FATAL_IF(gSurfaceImageClassInfo.mNativeBuffer == NULL, 305 "can't find android/graphics/ImageReader.%s", 306 ANDROID_MEDIA_SURFACEIMAGE_BUFFER_JNI_ID); 307 308 gSurfaceImageClassInfo.mTimestamp = env->GetFieldID( 309 imageClazz, ANDROID_MEDIA_SURFACEIMAGE_TS_JNI_ID, "J"); 310 LOG_ALWAYS_FATAL_IF(gSurfaceImageClassInfo.mTimestamp == NULL, 311 "can't find android/graphics/ImageReader.%s", 312 ANDROID_MEDIA_SURFACEIMAGE_TS_JNI_ID); 313 314 gSurfaceImageClassInfo.mTransform = env->GetFieldID( 315 imageClazz, ANDROID_MEDIA_SURFACEIMAGE_TF_JNI_ID, "I"); 316 LOG_ALWAYS_FATAL_IF(gSurfaceImageClassInfo.mTransform == NULL, 317 "can't find android/graphics/ImageReader.%s", 318 ANDROID_MEDIA_SURFACEIMAGE_TF_JNI_ID); 319 320 gSurfaceImageClassInfo.mScalingMode = env->GetFieldID( 321 imageClazz, ANDROID_MEDIA_SURFACEIMAGE_SM_JNI_ID, "I"); 322 LOG_ALWAYS_FATAL_IF(gSurfaceImageClassInfo.mScalingMode == NULL, 323 "can't find android/graphics/ImageReader.%s", 324 ANDROID_MEDIA_SURFACEIMAGE_SM_JNI_ID); 325 326 gSurfaceImageClassInfo.mPlanes = env->GetFieldID( 327 imageClazz, "mPlanes", "[Landroid/media/ImageReader$SurfaceImage$SurfacePlane;"); 328 LOG_ALWAYS_FATAL_IF(gSurfaceImageClassInfo.mPlanes == NULL, 329 "can't find android/media/ImageReader$ReaderSurfaceImage.mPlanes"); 330 331 gImageReaderClassInfo.mNativeContext = env->GetFieldID( 332 clazz, ANDROID_MEDIA_IMAGEREADER_CTX_JNI_ID, "J"); 333 LOG_ALWAYS_FATAL_IF(gImageReaderClassInfo.mNativeContext == NULL, 334 "can't find android/graphics/ImageReader.%s", 335 ANDROID_MEDIA_IMAGEREADER_CTX_JNI_ID); 336 337 gImageReaderClassInfo.postEventFromNative = env->GetStaticMethodID( 338 clazz, "postEventFromNative", "(Ljava/lang/Object;)V"); 339 LOG_ALWAYS_FATAL_IF(gImageReaderClassInfo.postEventFromNative == NULL, 340 "can't find android/graphics/ImageReader.postEventFromNative"); 341 342 jclass planeClazz = env->FindClass("android/media/ImageReader$SurfaceImage$SurfacePlane"); 343 LOG_ALWAYS_FATAL_IF(planeClazz == NULL, "Can not find SurfacePlane class"); 344 // FindClass only gives a local reference of jclass object. 345 gSurfacePlaneClassInfo.clazz = (jclass) env->NewGlobalRef(planeClazz); 346 gSurfacePlaneClassInfo.ctor = env->GetMethodID(gSurfacePlaneClassInfo.clazz, "<init>", 347 "(Landroid/media/ImageReader$SurfaceImage;IILjava/nio/ByteBuffer;)V"); 348 LOG_ALWAYS_FATAL_IF(gSurfacePlaneClassInfo.ctor == NULL, 349 "Can not find SurfacePlane constructor"); 350 } 351 352 static void ImageReader_init(JNIEnv* env, jobject thiz, jobject weakThiz, jint width, jint height, 353 jint format, jint maxImages, jlong ndkUsage) 354 { 355 status_t res; 356 int nativeFormat; 357 android_dataspace nativeDataspace; 358 359 ALOGV("%s: width:%d, height: %d, format: 0x%x, maxImages:%d", 360 __FUNCTION__, width, height, format, maxImages); 361 362 PublicFormat publicFormat = static_cast<PublicFormat>(format); 363 nativeFormat = android_view_Surface_mapPublicFormatToHalFormat( 364 publicFormat); 365 nativeDataspace = android_view_Surface_mapPublicFormatToHalDataspace( 366 publicFormat); 367 368 jclass clazz = env->GetObjectClass(thiz); 369 if (clazz == NULL) { 370 jniThrowRuntimeException(env, "Can't find android/graphics/ImageReader"); 371 return; 372 } 373 sp<JNIImageReaderContext> ctx(new JNIImageReaderContext(env, weakThiz, clazz, maxImages)); 374 375 sp<IGraphicBufferProducer> gbProducer; 376 sp<IGraphicBufferConsumer> gbConsumer; 377 BufferQueue::createBufferQueue(&gbProducer, &gbConsumer); 378 sp<BufferItemConsumer> bufferConsumer; 379 String8 consumerName = String8::format("ImageReader-%dx%df%xm%d-%d-%d", 380 width, height, format, maxImages, getpid(), 381 createProcessUniqueId()); 382 uint32_t consumerUsage = GRALLOC_USAGE_SW_READ_OFTEN; 383 bool needUsageOverride = ndkUsage != CONSUMER_BUFFER_USAGE_UNKNOWN; 384 uint64_t outProducerUsage = 0; 385 uint64_t outConsumerUsage = 0; 386 android_hardware_HardwareBuffer_convertToGrallocUsageBits(&outProducerUsage, &outConsumerUsage, 387 ndkUsage, 0); 388 389 if (isFormatOpaque(nativeFormat)) { 390 // Use the SW_READ_NEVER usage to tell producer that this format is not for preview or video 391 // encoding. The only possibility will be ZSL output. 392 consumerUsage = GRALLOC_USAGE_SW_READ_NEVER; 393 if (needUsageOverride) { 394 consumerUsage = android_convertGralloc1To0Usage(0, outConsumerUsage); 395 } 396 } else if (needUsageOverride) { 397 ALOGW("Consumer usage override for non-opaque format is not implemented yet, " 398 "ignore the provided usage from the application"); 399 } 400 bufferConsumer = new BufferItemConsumer(gbConsumer, consumerUsage, maxImages, 401 /*controlledByApp*/true); 402 if (bufferConsumer == nullptr) { 403 jniThrowExceptionFmt(env, "java/lang/RuntimeException", 404 "Failed to allocate native buffer consumer for format 0x%x and usage 0x%x", 405 nativeFormat, consumerUsage); 406 return; 407 } 408 ctx->setBufferConsumer(bufferConsumer); 409 bufferConsumer->setName(consumerName); 410 411 ctx->setProducer(gbProducer); 412 bufferConsumer->setFrameAvailableListener(ctx); 413 ImageReader_setNativeContext(env, thiz, ctx); 414 ctx->setBufferFormat(nativeFormat); 415 ctx->setBufferDataspace(nativeDataspace); 416 ctx->setBufferWidth(width); 417 ctx->setBufferHeight(height); 418 419 // Set the width/height/format/dataspace to the bufferConsumer. 420 res = bufferConsumer->setDefaultBufferSize(width, height); 421 if (res != OK) { 422 jniThrowExceptionFmt(env, "java/lang/IllegalStateException", 423 "Failed to set buffer consumer default size (%dx%d) for format 0x%x", 424 width, height, nativeFormat); 425 return; 426 } 427 res = bufferConsumer->setDefaultBufferFormat(nativeFormat); 428 if (res != OK) { 429 jniThrowExceptionFmt(env, "java/lang/IllegalStateException", 430 "Failed to set buffer consumer default format 0x%x", nativeFormat); 431 } 432 res = bufferConsumer->setDefaultBufferDataSpace(nativeDataspace); 433 if (res != OK) { 434 jniThrowExceptionFmt(env, "java/lang/IllegalStateException", 435 "Failed to set buffer consumer default dataSpace 0x%x", nativeDataspace); 436 } 437 } 438 439 static void ImageReader_close(JNIEnv* env, jobject thiz) 440 { 441 ALOGV("%s:", __FUNCTION__); 442 443 JNIImageReaderContext* const ctx = ImageReader_getContext(env, thiz); 444 if (ctx == NULL) { 445 // ImageReader is already closed. 446 return; 447 } 448 449 BufferItemConsumer* consumer = NULL; 450 consumer = ImageReader_getBufferConsumer(env, thiz); 451 452 if (consumer != NULL) { 453 consumer->abandon(); 454 consumer->setFrameAvailableListener(NULL); 455 } 456 ImageReader_setNativeContext(env, thiz, NULL); 457 } 458 459 static sp<Fence> Image_unlockIfLocked(JNIEnv* env, jobject image) { 460 ALOGV("%s", __FUNCTION__); 461 BufferItem* buffer = Image_getBufferItem(env, image); 462 if (buffer == NULL) { 463 jniThrowException(env, "java/lang/IllegalStateException", 464 "Image is not initialized"); 465 return Fence::NO_FENCE; 466 } 467 468 // Is locked? 469 bool wasBufferLocked = false; 470 jobject planes = NULL; 471 if (!isFormatOpaque(buffer->mGraphicBuffer->getPixelFormat())) { 472 planes = env->GetObjectField(image, gSurfaceImageClassInfo.mPlanes); 473 } 474 wasBufferLocked = (planes != NULL); 475 if (wasBufferLocked) { 476 status_t res = OK; 477 int fenceFd = -1; 478 if (wasBufferLocked) { 479 res = buffer->mGraphicBuffer->unlockAsync(&fenceFd); 480 if (res != OK) { 481 jniThrowRuntimeException(env, "unlock buffer failed"); 482 return Fence::NO_FENCE; 483 } 484 } 485 sp<Fence> releaseFence = new Fence(fenceFd); 486 return releaseFence; 487 ALOGV("Successfully unlocked the image"); 488 } 489 return Fence::NO_FENCE; 490 } 491 492 static void ImageReader_imageRelease(JNIEnv* env, jobject thiz, jobject image) 493 { 494 ALOGV("%s:", __FUNCTION__); 495 JNIImageReaderContext* ctx = ImageReader_getContext(env, thiz); 496 if (ctx == NULL) { 497 ALOGW("ImageReader#close called before Image#close, consider calling Image#close first"); 498 return; 499 } 500 501 BufferItemConsumer* bufferConsumer = ctx->getBufferConsumer(); 502 BufferItem* buffer = Image_getBufferItem(env, image); 503 if (buffer == nullptr) { 504 // Release an already closed image is harmless. 505 return; 506 } 507 508 sp<Fence> releaseFence = Image_unlockIfLocked(env, image); 509 bufferConsumer->releaseBuffer(*buffer, releaseFence); 510 Image_setBufferItem(env, image, NULL); 511 ctx->returnBufferItem(buffer); 512 ALOGV("%s: Image (format: 0x%x) has been released", __FUNCTION__, ctx->getBufferFormat()); 513 } 514 515 static jint ImageReader_imageSetup(JNIEnv* env, jobject thiz, jobject image) { 516 ALOGV("%s:", __FUNCTION__); 517 JNIImageReaderContext* ctx = ImageReader_getContext(env, thiz); 518 if (ctx == NULL) { 519 jniThrowException(env, "java/lang/IllegalStateException", 520 "ImageReader is not initialized or was already closed"); 521 return -1; 522 } 523 524 BufferItemConsumer* bufferConsumer = ctx->getBufferConsumer(); 525 BufferItem* buffer = ctx->getBufferItem(); 526 if (buffer == NULL) { 527 ALOGW("Unable to acquire a buffer item, very likely client tried to acquire more than" 528 " maxImages buffers"); 529 return ACQUIRE_MAX_IMAGES; 530 } 531 532 status_t res = bufferConsumer->acquireBuffer(buffer, 0); 533 if (res != OK) { 534 ctx->returnBufferItem(buffer); 535 if (res != BufferQueue::NO_BUFFER_AVAILABLE) { 536 if (res == INVALID_OPERATION) { 537 // Max number of images were already acquired. 538 ALOGE("%s: Max number of buffers allowed are already acquired : %s (%d)", 539 __FUNCTION__, strerror(-res), res); 540 return ACQUIRE_MAX_IMAGES; 541 } else { 542 ALOGE("%s: Acquire image failed with some unknown error: %s (%d)", 543 __FUNCTION__, strerror(-res), res); 544 jniThrowExceptionFmt(env, "java/lang/IllegalStateException", 545 "Unknown error (%d) when we tried to acquire an image.", 546 res); 547 return ACQUIRE_NO_BUFFERS; 548 } 549 } 550 // This isn't really an error case, as the application may acquire buffer at any time. 551 return ACQUIRE_NO_BUFFERS; 552 } 553 554 // Add some extra checks for non-opaque formats. 555 if (!isFormatOpaque(ctx->getBufferFormat())) { 556 // Check if the left-top corner of the crop rect is origin, we currently assume this point is 557 // zero, will revisit this once this assumption turns out problematic. 558 Point lt = buffer->mCrop.leftTop(); 559 if (lt.x != 0 || lt.y != 0) { 560 jniThrowExceptionFmt(env, "java/lang/UnsupportedOperationException", 561 "crop left top corner [%d, %d] need to be at origin", lt.x, lt.y); 562 return -1; 563 } 564 565 // Check if the producer buffer configurations match what ImageReader configured. 566 int outputWidth = getBufferWidth(buffer); 567 int outputHeight = getBufferHeight(buffer); 568 569 int imgReaderFmt = ctx->getBufferFormat(); 570 int imageReaderWidth = ctx->getBufferWidth(); 571 int imageReaderHeight = ctx->getBufferHeight(); 572 int bufferFormat = buffer->mGraphicBuffer->getPixelFormat(); 573 if ((bufferFormat != HAL_PIXEL_FORMAT_BLOB) && (imgReaderFmt != HAL_PIXEL_FORMAT_BLOB) && 574 (imageReaderWidth != outputWidth || imageReaderHeight != outputHeight)) { 575 ALOGV("%s: Producer buffer size: %dx%d, doesn't match ImageReader configured size: %dx%d", 576 __FUNCTION__, outputWidth, outputHeight, imageReaderWidth, imageReaderHeight); 577 } 578 if (imgReaderFmt != bufferFormat) { 579 if (imgReaderFmt == HAL_PIXEL_FORMAT_YCbCr_420_888 && 580 isPossiblyYUV(bufferFormat)) { 581 // Treat formats that are compatible with flexible YUV 582 // (HAL_PIXEL_FORMAT_YCbCr_420_888) as HAL_PIXEL_FORMAT_YCbCr_420_888. 583 ALOGV("%s: Treat buffer format to 0x%x as HAL_PIXEL_FORMAT_YCbCr_420_888", 584 __FUNCTION__, bufferFormat); 585 } else if (imgReaderFmt == HAL_PIXEL_FORMAT_BLOB && 586 bufferFormat == HAL_PIXEL_FORMAT_RGBA_8888) { 587 // Using HAL_PIXEL_FORMAT_RGBA_8888 Gralloc buffers containing JPEGs to get around 588 // SW write limitations for (b/17379185). 589 ALOGV("%s: Receiving JPEG in HAL_PIXEL_FORMAT_RGBA_8888 buffer.", __FUNCTION__); 590 } else { 591 // Return the buffer to the queue. No need to provide fence, as this buffer wasn't 592 // used anywhere yet. 593 bufferConsumer->releaseBuffer(*buffer); 594 ctx->returnBufferItem(buffer); 595 596 // Throw exception 597 ALOGE("Producer output buffer format: 0x%x, ImageReader configured format: 0x%x", 598 bufferFormat, ctx->getBufferFormat()); 599 String8 msg; 600 msg.appendFormat("The producer output buffer format 0x%x doesn't " 601 "match the ImageReader's configured buffer format 0x%x.", 602 bufferFormat, ctx->getBufferFormat()); 603 jniThrowException(env, "java/lang/UnsupportedOperationException", 604 msg.string()); 605 return -1; 606 } 607 } 608 609 } 610 611 // Set SurfaceImage instance member variables 612 Image_setBufferItem(env, image, buffer); 613 env->SetLongField(image, gSurfaceImageClassInfo.mTimestamp, 614 static_cast<jlong>(buffer->mTimestamp)); 615 env->SetIntField(image, gSurfaceImageClassInfo.mTransform, 616 static_cast<jint>(buffer->mTransform)); 617 env->SetIntField(image, gSurfaceImageClassInfo.mScalingMode, 618 static_cast<jint>(buffer->mScalingMode)); 619 620 return ACQUIRE_SUCCESS; 621 } 622 623 static jint ImageReader_detachImage(JNIEnv* env, jobject thiz, jobject image) { 624 ALOGV("%s:", __FUNCTION__); 625 JNIImageReaderContext* ctx = ImageReader_getContext(env, thiz); 626 if (ctx == NULL) { 627 jniThrowException(env, "java/lang/IllegalStateException", "ImageReader was already closed"); 628 return -1; 629 } 630 631 BufferItemConsumer* bufferConsumer = ctx->getBufferConsumer(); 632 BufferItem* buffer = Image_getBufferItem(env, image); 633 if (!buffer) { 634 ALOGE( 635 "Image already released and can not be detached from ImageReader!!!"); 636 jniThrowException(env, "java/lang/IllegalStateException", 637 "Image detach from ImageReader failed: buffer was already released"); 638 return -1; 639 } 640 641 status_t res = OK; 642 Image_unlockIfLocked(env, image); 643 res = bufferConsumer->detachBuffer(buffer->mSlot); 644 if (res != OK) { 645 ALOGE("Image detach failed: %s (%d)!!!", strerror(-res), res); 646 jniThrowRuntimeException(env, 647 "nativeDetachImage failed for image!!!"); 648 return res; 649 } 650 return OK; 651 } 652 653 static void ImageReader_discardFreeBuffers(JNIEnv* env, jobject thiz) { 654 ALOGV("%s:", __FUNCTION__); 655 JNIImageReaderContext* ctx = ImageReader_getContext(env, thiz); 656 if (ctx == NULL) { 657 jniThrowException(env, "java/lang/IllegalStateException", "ImageReader was already closed"); 658 return; 659 } 660 661 BufferItemConsumer* bufferConsumer = ctx->getBufferConsumer(); 662 status_t res = bufferConsumer->discardFreeBuffers(); 663 if (res != OK) { 664 ALOGE("Buffer discard failed: %s (%d)", strerror(-res), res); 665 jniThrowRuntimeException(env, 666 "nativeDicardFreebuffers failed"); 667 } 668 } 669 670 static jobject ImageReader_getSurface(JNIEnv* env, jobject thiz) 671 { 672 ALOGV("%s: ", __FUNCTION__); 673 674 IGraphicBufferProducer* gbp = ImageReader_getProducer(env, thiz); 675 if (gbp == NULL) { 676 jniThrowRuntimeException(env, "Buffer consumer is uninitialized"); 677 return NULL; 678 } 679 680 // Wrap the IGBP in a Java-language Surface. 681 return android_view_Surface_createFromIGraphicBufferProducer(env, gbp); 682 } 683 684 static void Image_getLockedImage(JNIEnv* env, jobject thiz, LockedImage *image) { 685 ALOGV("%s", __FUNCTION__); 686 BufferItem* buffer = Image_getBufferItem(env, thiz); 687 if (buffer == NULL) { 688 jniThrowException(env, "java/lang/IllegalStateException", 689 "Image is not initialized"); 690 return; 691 } 692 693 status_t res = lockImageFromBuffer(buffer, 694 GRALLOC_USAGE_SW_READ_OFTEN, buffer->mFence->dup(), image); 695 if (res != OK) { 696 jniThrowExceptionFmt(env, "java/lang/RuntimeException", 697 "lock buffer failed for format 0x%x", 698 buffer->mGraphicBuffer->getPixelFormat()); 699 return; 700 } 701 702 // Carry over some fields from BufferItem. 703 image->crop = buffer->mCrop; 704 image->transform = buffer->mTransform; 705 image->scalingMode = buffer->mScalingMode; 706 image->timestamp = buffer->mTimestamp; 707 image->dataSpace = buffer->mDataSpace; 708 image->frameNumber = buffer->mFrameNumber; 709 710 ALOGV("%s: Successfully locked the image", __FUNCTION__); 711 // crop, transform, scalingMode, timestamp, and frameNumber should be set by producer, 712 // and we don't set them here. 713 } 714 715 static void Image_getLockedImageInfo(JNIEnv* env, LockedImage* buffer, int idx, 716 int32_t writerFormat, uint8_t **base, uint32_t *size, int *pixelStride, int *rowStride) { 717 ALOGV("%s", __FUNCTION__); 718 719 status_t res = getLockedImageInfo(buffer, idx, writerFormat, base, size, 720 pixelStride, rowStride); 721 if (res != OK) { 722 jniThrowExceptionFmt(env, "java/lang/UnsupportedOperationException", 723 "Pixel format: 0x%x is unsupported", buffer->flexFormat); 724 } 725 } 726 727 static jobjectArray Image_createSurfacePlanes(JNIEnv* env, jobject thiz, 728 int numPlanes, int readerFormat) 729 { 730 ALOGV("%s: create SurfacePlane array with size %d", __FUNCTION__, numPlanes); 731 int rowStride = 0; 732 int pixelStride = 0; 733 uint8_t *pData = NULL; 734 uint32_t dataSize = 0; 735 jobject byteBuffer = NULL; 736 737 PublicFormat publicReaderFormat = static_cast<PublicFormat>(readerFormat); 738 int halReaderFormat = android_view_Surface_mapPublicFormatToHalFormat( 739 publicReaderFormat); 740 741 if (isFormatOpaque(halReaderFormat) && numPlanes > 0) { 742 String8 msg; 743 msg.appendFormat("Format 0x%x is opaque, thus not writable, the number of planes (%d)" 744 " must be 0", halReaderFormat, numPlanes); 745 jniThrowException(env, "java/lang/IllegalArgumentException", msg.string()); 746 return NULL; 747 } 748 749 jobjectArray surfacePlanes = env->NewObjectArray(numPlanes, gSurfacePlaneClassInfo.clazz, 750 /*initial_element*/NULL); 751 if (surfacePlanes == NULL) { 752 jniThrowRuntimeException(env, "Failed to create SurfacePlane arrays," 753 " probably out of memory"); 754 return NULL; 755 } 756 if (isFormatOpaque(halReaderFormat)) { 757 // Return 0 element surface array. 758 return surfacePlanes; 759 } 760 761 LockedImage lockedImg = LockedImage(); 762 Image_getLockedImage(env, thiz, &lockedImg); 763 if (env->ExceptionCheck()) { 764 return NULL; 765 } 766 // Create all SurfacePlanes 767 for (int i = 0; i < numPlanes; i++) { 768 Image_getLockedImageInfo(env, &lockedImg, i, halReaderFormat, 769 &pData, &dataSize, &pixelStride, &rowStride); 770 byteBuffer = env->NewDirectByteBuffer(pData, dataSize); 771 if ((byteBuffer == NULL) && (env->ExceptionCheck() == false)) { 772 jniThrowException(env, "java/lang/IllegalStateException", 773 "Failed to allocate ByteBuffer"); 774 return NULL; 775 } 776 777 // Finally, create this SurfacePlane. 778 jobject surfacePlane = env->NewObject(gSurfacePlaneClassInfo.clazz, 779 gSurfacePlaneClassInfo.ctor, thiz, rowStride, pixelStride, byteBuffer); 780 env->SetObjectArrayElement(surfacePlanes, i, surfacePlane); 781 } 782 783 return surfacePlanes; 784 } 785 786 static jint Image_getWidth(JNIEnv* env, jobject thiz) 787 { 788 BufferItem* buffer = Image_getBufferItem(env, thiz); 789 return getBufferWidth(buffer); 790 } 791 792 static jint Image_getHeight(JNIEnv* env, jobject thiz) 793 { 794 BufferItem* buffer = Image_getBufferItem(env, thiz); 795 return getBufferHeight(buffer); 796 } 797 798 static jint Image_getFormat(JNIEnv* env, jobject thiz, jint readerFormat) 799 { 800 if (isFormatOpaque(readerFormat)) { 801 // Assuming opaque reader produce opaque images. 802 return static_cast<jint>(PublicFormat::PRIVATE); 803 } else { 804 BufferItem* buffer = Image_getBufferItem(env, thiz); 805 int readerHalFormat = android_view_Surface_mapPublicFormatToHalFormat( 806 static_cast<PublicFormat>(readerFormat)); 807 int32_t fmt = applyFormatOverrides( 808 buffer->mGraphicBuffer->getPixelFormat(), readerHalFormat); 809 // Override the image format to HAL_PIXEL_FORMAT_YCbCr_420_888 if the actual format is 810 // NV21 or YV12. This could only happen when the Gralloc HAL version is v0.1 thus doesn't 811 // support lockycbcr(), the CpuConsumer need to use the lock() method in the 812 // lockNextBuffer() call. For Gralloc HAL v0.2 or newer, this format should already be 813 // overridden to HAL_PIXEL_FORMAT_YCbCr_420_888 for the flexible YUV compatible formats. 814 if (isPossiblyYUV(fmt)) { 815 fmt = HAL_PIXEL_FORMAT_YCbCr_420_888; 816 } 817 PublicFormat publicFmt = android_view_Surface_mapHalFormatDataspaceToPublicFormat( 818 fmt, buffer->mDataSpace); 819 return static_cast<jint>(publicFmt); 820 } 821 } 822 823 static jobject Image_getHardwareBuffer(JNIEnv* env, jobject thiz) { 824 BufferItem* buffer = Image_getBufferItem(env, thiz); 825 AHardwareBuffer* b = AHardwareBuffer_from_GraphicBuffer(buffer->mGraphicBuffer.get()); 826 // don't user the public AHardwareBuffer_toHardwareBuffer() because this would force us 827 // to link against libandroid.so 828 return android_hardware_HardwareBuffer_createFromAHardwareBuffer(env, b); 829 } 830 831 } // extern "C" 832 833 // ---------------------------------------------------------------------------- 834 835 static const JNINativeMethod gImageReaderMethods[] = { 836 {"nativeClassInit", "()V", (void*)ImageReader_classInit }, 837 {"nativeInit", "(Ljava/lang/Object;IIIIJ)V", (void*)ImageReader_init }, 838 {"nativeClose", "()V", (void*)ImageReader_close }, 839 {"nativeReleaseImage", "(Landroid/media/Image;)V", (void*)ImageReader_imageRelease }, 840 {"nativeImageSetup", "(Landroid/media/Image;)I", (void*)ImageReader_imageSetup }, 841 {"nativeGetSurface", "()Landroid/view/Surface;", (void*)ImageReader_getSurface }, 842 {"nativeDetachImage", "(Landroid/media/Image;)I", (void*)ImageReader_detachImage }, 843 {"nativeDiscardFreeBuffers", "()V", (void*)ImageReader_discardFreeBuffers } 844 }; 845 846 static const JNINativeMethod gImageMethods[] = { 847 {"nativeCreatePlanes", "(II)[Landroid/media/ImageReader$SurfaceImage$SurfacePlane;", 848 (void*)Image_createSurfacePlanes }, 849 {"nativeGetWidth", "()I", (void*)Image_getWidth }, 850 {"nativeGetHeight", "()I", (void*)Image_getHeight }, 851 {"nativeGetFormat", "(I)I", (void*)Image_getFormat }, 852 {"nativeGetHardwareBuffer", "()Landroid/hardware/HardwareBuffer;", 853 (void*)Image_getHardwareBuffer }, 854 }; 855 856 int register_android_media_ImageReader(JNIEnv *env) { 857 858 int ret1 = AndroidRuntime::registerNativeMethods(env, 859 "android/media/ImageReader", gImageReaderMethods, NELEM(gImageReaderMethods)); 860 861 int ret2 = AndroidRuntime::registerNativeMethods(env, 862 "android/media/ImageReader$SurfaceImage", gImageMethods, NELEM(gImageMethods)); 863 864 return (ret1 || ret2); 865 } 866
