1 /* 2 * Copyright 2013 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17 //#define LOG_NDEBUG 0 18 #define LOG_TAG "ImageReader_JNI" 19 #include <utils/Log.h> 20 #include <utils/misc.h> 21 #include <utils/List.h> 22 #include <utils/String8.h> 23 24 #include <cstdio> 25 26 #include <gui/CpuConsumer.h> 27 #include <gui/Surface.h> 28 #include <camera3.h> 29 30 #include <android_runtime/AndroidRuntime.h> 31 #include <android_runtime/android_view_Surface.h> 32 33 #include <jni.h> 34 #include <JNIHelp.h> 35 36 #include <stdint.h> 37 #include <inttypes.h> 38 39 #define ALIGN(x, mask) ( ((x) + (mask) - 1) & ~((mask) - 1) ) 40 41 #define ANDROID_MEDIA_IMAGEREADER_CTX_JNI_ID "mNativeContext" 42 #define ANDROID_MEDIA_SURFACEIMAGE_BUFFER_JNI_ID "mLockedBuffer" 43 #define ANDROID_MEDIA_SURFACEIMAGE_TS_JNI_ID "mTimestamp" 44 45 // ---------------------------------------------------------------------------- 46 47 using namespace android; 48 49 enum { 50 IMAGE_READER_MAX_NUM_PLANES = 3, 51 }; 52 53 enum { 54 ACQUIRE_SUCCESS = 0, 55 ACQUIRE_NO_BUFFERS = 1, 56 ACQUIRE_MAX_IMAGES = 2, 57 }; 58 59 static struct { 60 jfieldID mNativeContext; 61 jmethodID postEventFromNative; 62 } gImageReaderClassInfo; 63 64 static struct { 65 jfieldID mLockedBuffer; 66 jfieldID mTimestamp; 67 } gSurfaceImageClassInfo; 68 69 static struct { 70 jclass clazz; 71 jmethodID ctor; 72 } gSurfacePlaneClassInfo; 73 74 // ---------------------------------------------------------------------------- 75 76 class JNIImageReaderContext : public CpuConsumer::FrameAvailableListener 77 { 78 public: 79 JNIImageReaderContext(JNIEnv* env, jobject weakThiz, jclass clazz, int maxImages); 80 81 virtual ~JNIImageReaderContext(); 82 83 virtual void onFrameAvailable(const BufferItem& item); 84 85 CpuConsumer::LockedBuffer* getLockedBuffer(); 86 87 void returnLockedBuffer(CpuConsumer::LockedBuffer* buffer); 88 89 void setCpuConsumer(const sp<CpuConsumer>& consumer) { mConsumer = consumer; } 90 CpuConsumer* getCpuConsumer() { return mConsumer.get(); } 91 92 void setProducer(const sp<IGraphicBufferProducer>& producer) { mProducer = producer; } 93 IGraphicBufferProducer* getProducer() { return mProducer.get(); } 94 95 void setBufferFormat(int format) { mFormat = format; } 96 int getBufferFormat() { return mFormat; } 97 98 void setBufferWidth(int width) { mWidth = width; } 99 int getBufferWidth() { return mWidth; } 100 101 void setBufferHeight(int height) { mHeight = height; } 102 int getBufferHeight() { return mHeight; } 103 104 private: 105 static JNIEnv* getJNIEnv(bool* needsDetach); 106 static void detachJNI(); 107 108 List<CpuConsumer::LockedBuffer*> mBuffers; 109 sp<CpuConsumer> mConsumer; 110 sp<IGraphicBufferProducer> mProducer; 111 jobject mWeakThiz; 112 jclass mClazz; 113 int mFormat; 114 int mWidth; 115 int mHeight; 116 }; 117 118 JNIImageReaderContext::JNIImageReaderContext(JNIEnv* env, 119 jobject weakThiz, jclass clazz, int maxImages) : 120 mWeakThiz(env->NewGlobalRef(weakThiz)), 121 mClazz((jclass)env->NewGlobalRef(clazz)) { 122 for (int i = 0; i < maxImages; i++) { 123 CpuConsumer::LockedBuffer *buffer = new CpuConsumer::LockedBuffer; 124 mBuffers.push_back(buffer); 125 } 126 } 127 128 JNIEnv* JNIImageReaderContext::getJNIEnv(bool* needsDetach) { 129 LOG_ALWAYS_FATAL_IF(needsDetach == NULL, "needsDetach is null!!!"); 130 *needsDetach = false; 131 JNIEnv* env = AndroidRuntime::getJNIEnv(); 132 if (env == NULL) { 133 JavaVMAttachArgs args = {JNI_VERSION_1_4, NULL, NULL}; 134 JavaVM* vm = AndroidRuntime::getJavaVM(); 135 int result = vm->AttachCurrentThread(&env, (void*) &args); 136 if (result != JNI_OK) { 137 ALOGE("thread attach failed: %#x", result); 138 return NULL; 139 } 140 *needsDetach = true; 141 } 142 return env; 143 } 144 145 void JNIImageReaderContext::detachJNI() { 146 JavaVM* vm = AndroidRuntime::getJavaVM(); 147 int result = vm->DetachCurrentThread(); 148 if (result != JNI_OK) { 149 ALOGE("thread detach failed: %#x", result); 150 } 151 } 152 153 CpuConsumer::LockedBuffer* JNIImageReaderContext::getLockedBuffer() { 154 if (mBuffers.empty()) { 155 return NULL; 156 } 157 // Return a LockedBuffer pointer and remove it from the list 158 List<CpuConsumer::LockedBuffer*>::iterator it = mBuffers.begin(); 159 CpuConsumer::LockedBuffer* buffer = *it; 160 mBuffers.erase(it); 161 return buffer; 162 } 163 164 void JNIImageReaderContext::returnLockedBuffer(CpuConsumer::LockedBuffer* buffer) { 165 mBuffers.push_back(buffer); 166 } 167 168 JNIImageReaderContext::~JNIImageReaderContext() { 169 bool needsDetach = false; 170 JNIEnv* env = getJNIEnv(&needsDetach); 171 if (env != NULL) { 172 env->DeleteGlobalRef(mWeakThiz); 173 env->DeleteGlobalRef(mClazz); 174 } else { 175 ALOGW("leaking JNI object references"); 176 } 177 if (needsDetach) { 178 detachJNI(); 179 } 180 181 // Delete LockedBuffers 182 for (List<CpuConsumer::LockedBuffer *>::iterator it = mBuffers.begin(); 183 it != mBuffers.end(); it++) { 184 delete *it; 185 } 186 mBuffers.clear(); 187 mConsumer.clear(); 188 } 189 190 void JNIImageReaderContext::onFrameAvailable(const BufferItem& /*item*/) 191 { 192 ALOGV("%s: frame available", __FUNCTION__); 193 bool needsDetach = false; 194 JNIEnv* env = getJNIEnv(&needsDetach); 195 if (env != NULL) { 196 env->CallStaticVoidMethod(mClazz, gImageReaderClassInfo.postEventFromNative, mWeakThiz); 197 } else { 198 ALOGW("onFrameAvailable event will not posted"); 199 } 200 if (needsDetach) { 201 detachJNI(); 202 } 203 } 204 205 // ---------------------------------------------------------------------------- 206 207 extern "C" { 208 209 static JNIImageReaderContext* ImageReader_getContext(JNIEnv* env, jobject thiz) 210 { 211 JNIImageReaderContext *ctx; 212 ctx = reinterpret_cast<JNIImageReaderContext *> 213 (env->GetLongField(thiz, gImageReaderClassInfo.mNativeContext)); 214 return ctx; 215 } 216 217 static CpuConsumer* ImageReader_getCpuConsumer(JNIEnv* env, jobject thiz) 218 { 219 ALOGV("%s:", __FUNCTION__); 220 JNIImageReaderContext* const ctx = ImageReader_getContext(env, thiz); 221 if (ctx == NULL) { 222 jniThrowRuntimeException(env, "ImageReaderContext is not initialized"); 223 return NULL; 224 } 225 return ctx->getCpuConsumer(); 226 } 227 228 static IGraphicBufferProducer* ImageReader_getProducer(JNIEnv* env, jobject thiz) 229 { 230 ALOGV("%s:", __FUNCTION__); 231 JNIImageReaderContext* const ctx = ImageReader_getContext(env, thiz); 232 if (ctx == NULL) { 233 jniThrowRuntimeException(env, "ImageReaderContext is not initialized"); 234 return NULL; 235 } 236 return ctx->getProducer(); 237 } 238 239 static void ImageReader_setNativeContext(JNIEnv* env, 240 jobject thiz, sp<JNIImageReaderContext> ctx) 241 { 242 ALOGV("%s:", __FUNCTION__); 243 JNIImageReaderContext* const p = ImageReader_getContext(env, thiz); 244 if (ctx != 0) { 245 ctx->incStrong((void*)ImageReader_setNativeContext); 246 } 247 if (p) { 248 p->decStrong((void*)ImageReader_setNativeContext); 249 } 250 env->SetLongField(thiz, gImageReaderClassInfo.mNativeContext, 251 reinterpret_cast<jlong>(ctx.get())); 252 } 253 254 static CpuConsumer::LockedBuffer* Image_getLockedBuffer(JNIEnv* env, jobject image) 255 { 256 return reinterpret_cast<CpuConsumer::LockedBuffer*>( 257 env->GetLongField(image, gSurfaceImageClassInfo.mLockedBuffer)); 258 } 259 260 static void Image_setBuffer(JNIEnv* env, jobject thiz, 261 const CpuConsumer::LockedBuffer* buffer) 262 { 263 env->SetLongField(thiz, gSurfaceImageClassInfo.mLockedBuffer, reinterpret_cast<jlong>(buffer)); 264 } 265 266 // Some formats like JPEG defined with different values between android.graphics.ImageFormat and 267 // graphics.h, need convert to the one defined in graphics.h here. 268 static int Image_getPixelFormat(JNIEnv* env, int format) 269 { 270 int jpegFormat; 271 jfieldID fid; 272 273 ALOGV("%s: format = 0x%x", __FUNCTION__, format); 274 275 jclass imageFormatClazz = env->FindClass("android/graphics/ImageFormat"); 276 ALOG_ASSERT(imageFormatClazz != NULL); 277 278 fid = env->GetStaticFieldID(imageFormatClazz, "JPEG", "I"); 279 jpegFormat = env->GetStaticIntField(imageFormatClazz, fid); 280 281 // Translate the JPEG to BLOB for camera purpose. 282 if (format == jpegFormat) { 283 format = HAL_PIXEL_FORMAT_BLOB; 284 } 285 286 return format; 287 } 288 289 static uint32_t Image_getJpegSize(CpuConsumer::LockedBuffer* buffer, bool usingRGBAOverride) 290 { 291 ALOG_ASSERT(buffer != NULL, "Input buffer is NULL!!!"); 292 uint32_t size = 0; 293 uint32_t width = buffer->width; 294 uint8_t* jpegBuffer = buffer->data; 295 296 if (usingRGBAOverride) { 297 width = (buffer->width + buffer->stride * (buffer->height - 1)) * 4; 298 } 299 300 // First check for JPEG transport header at the end of the buffer 301 uint8_t* header = jpegBuffer + (width - sizeof(struct camera3_jpeg_blob)); 302 struct camera3_jpeg_blob *blob = (struct camera3_jpeg_blob*)(header); 303 if (blob->jpeg_blob_id == CAMERA3_JPEG_BLOB_ID) { 304 size = blob->jpeg_size; 305 ALOGV("%s: Jpeg size = %d", __FUNCTION__, size); 306 } 307 308 // failed to find size, default to whole buffer 309 if (size == 0) { 310 /* 311 * This is a problem because not including the JPEG header 312 * means that in certain rare situations a regular JPEG blob 313 * will be misidentified as having a header, in which case 314 * we will get a garbage size value. 315 */ 316 ALOGW("%s: No JPEG header detected, defaulting to size=width=%d", 317 __FUNCTION__, width); 318 size = width; 319 } 320 321 return size; 322 } 323 324 static bool usingRGBAToJpegOverride(int32_t bufferFormat, int32_t readerCtxFormat) { 325 return readerCtxFormat == HAL_PIXEL_FORMAT_BLOB && bufferFormat == HAL_PIXEL_FORMAT_RGBA_8888; 326 } 327 328 static int32_t applyFormatOverrides(int32_t bufferFormat, int32_t readerCtxFormat) 329 { 330 // Using HAL_PIXEL_FORMAT_RGBA_8888 gralloc buffers containing JPEGs to get around SW 331 // write limitations for some platforms (b/17379185). 332 if (usingRGBAToJpegOverride(bufferFormat, readerCtxFormat)) { 333 return HAL_PIXEL_FORMAT_BLOB; 334 } 335 return bufferFormat; 336 } 337 338 static void Image_getLockedBufferInfo(JNIEnv* env, CpuConsumer::LockedBuffer* buffer, int idx, 339 uint8_t **base, uint32_t *size, int32_t readerFormat) 340 { 341 ALOG_ASSERT(buffer != NULL, "Input buffer is NULL!!!"); 342 ALOG_ASSERT(base != NULL, "base is NULL!!!"); 343 ALOG_ASSERT(size != NULL, "size is NULL!!!"); 344 ALOG_ASSERT((idx < IMAGE_READER_MAX_NUM_PLANES) && (idx >= 0)); 345 346 ALOGV("%s: buffer: %p", __FUNCTION__, buffer); 347 348 uint32_t dataSize, ySize, cSize, cStride; 349 uint8_t *cb, *cr; 350 uint8_t *pData = NULL; 351 int bytesPerPixel = 0; 352 353 dataSize = ySize = cSize = cStride = 0; 354 int32_t fmt = buffer->flexFormat; 355 356 bool usingRGBAOverride = usingRGBAToJpegOverride(fmt, readerFormat); 357 fmt = applyFormatOverrides(fmt, readerFormat); 358 switch (fmt) { 359 case HAL_PIXEL_FORMAT_YCbCr_420_888: 360 pData = 361 (idx == 0) ? 362 buffer->data : 363 (idx == 1) ? 364 buffer->dataCb : 365 buffer->dataCr; 366 // only map until last pixel 367 if (idx == 0) { 368 dataSize = buffer->stride * (buffer->height - 1) + buffer->width; 369 } else { 370 dataSize = buffer->chromaStride * (buffer->height / 2 - 1) + 371 buffer->chromaStep * (buffer->width / 2 - 1) + 1; 372 } 373 break; 374 // NV21 375 case HAL_PIXEL_FORMAT_YCrCb_420_SP: 376 cr = buffer->data + (buffer->stride * buffer->height); 377 cb = cr + 1; 378 // only map until last pixel 379 ySize = buffer->width * (buffer->height - 1) + buffer->width; 380 cSize = buffer->width * (buffer->height / 2 - 1) + buffer->width - 1; 381 382 pData = 383 (idx == 0) ? 384 buffer->data : 385 (idx == 1) ? 386 cb: 387 cr; 388 389 dataSize = (idx == 0) ? ySize : cSize; 390 break; 391 case HAL_PIXEL_FORMAT_YV12: 392 // Y and C stride need to be 16 pixel aligned. 393 LOG_ALWAYS_FATAL_IF(buffer->stride % 16, 394 "Stride is not 16 pixel aligned %d", buffer->stride); 395 396 ySize = buffer->stride * buffer->height; 397 cStride = ALIGN(buffer->stride / 2, 16); 398 cr = buffer->data + ySize; 399 cSize = cStride * buffer->height / 2; 400 cb = cr + cSize; 401 402 pData = 403 (idx == 0) ? 404 buffer->data : 405 (idx == 1) ? 406 cb : 407 cr; 408 dataSize = (idx == 0) ? ySize : cSize; 409 break; 410 case HAL_PIXEL_FORMAT_Y8: 411 // Single plane, 8bpp. 412 ALOG_ASSERT(idx == 0, "Wrong index: %d", idx); 413 414 pData = buffer->data; 415 dataSize = buffer->stride * buffer->height; 416 break; 417 case HAL_PIXEL_FORMAT_Y16: 418 bytesPerPixel = 2; 419 // Single plane, 16bpp, strides are specified in pixels, not in bytes 420 ALOG_ASSERT(idx == 0, "Wrong index: %d", idx); 421 422 pData = buffer->data; 423 dataSize = buffer->stride * buffer->height * bytesPerPixel; 424 break; 425 case HAL_PIXEL_FORMAT_BLOB: 426 // Used for JPEG data, height must be 1, width == size, single plane. 427 ALOG_ASSERT(idx == 0, "Wrong index: %d", idx); 428 ALOG_ASSERT(buffer->height == 1, "JPEG should has height value %d", buffer->height); 429 430 pData = buffer->data; 431 dataSize = Image_getJpegSize(buffer, usingRGBAOverride); 432 break; 433 case HAL_PIXEL_FORMAT_RAW_SENSOR: 434 // Single plane 16bpp bayer data. 435 bytesPerPixel = 2; 436 ALOG_ASSERT(idx == 0, "Wrong index: %d", idx); 437 pData = buffer->data; 438 dataSize = buffer->stride * buffer->height * bytesPerPixel; 439 break; 440 case HAL_PIXEL_FORMAT_RAW10: 441 // Single plane 10bpp bayer data. 442 ALOG_ASSERT(idx == 0, "Wrong index: %d", idx); 443 LOG_ALWAYS_FATAL_IF(buffer->width % 4, 444 "Width is not multiple of 4 %d", buffer->width); 445 LOG_ALWAYS_FATAL_IF(buffer->height % 2, 446 "Height is not even %d", buffer->height); 447 LOG_ALWAYS_FATAL_IF(buffer->stride < (buffer->width * 10 / 8), 448 "stride (%d) should be at least %d", 449 buffer->stride, buffer->width * 10 / 8); 450 pData = buffer->data; 451 dataSize = buffer->stride * buffer->height; 452 break; 453 case HAL_PIXEL_FORMAT_RGBA_8888: 454 case HAL_PIXEL_FORMAT_RGBX_8888: 455 // Single plane, 32bpp. 456 bytesPerPixel = 4; 457 ALOG_ASSERT(idx == 0, "Wrong index: %d", idx); 458 pData = buffer->data; 459 dataSize = buffer->stride * buffer->height * bytesPerPixel; 460 break; 461 case HAL_PIXEL_FORMAT_RGB_565: 462 // Single plane, 16bpp. 463 bytesPerPixel = 2; 464 ALOG_ASSERT(idx == 0, "Wrong index: %d", idx); 465 pData = buffer->data; 466 dataSize = buffer->stride * buffer->height * bytesPerPixel; 467 break; 468 case HAL_PIXEL_FORMAT_RGB_888: 469 // Single plane, 24bpp. 470 bytesPerPixel = 3; 471 ALOG_ASSERT(idx == 0, "Wrong index: %d", idx); 472 pData = buffer->data; 473 dataSize = buffer->stride * buffer->height * bytesPerPixel; 474 break; 475 default: 476 jniThrowExceptionFmt(env, "java/lang/UnsupportedOperationException", 477 "Pixel format: 0x%x is unsupported", fmt); 478 break; 479 } 480 481 *base = pData; 482 *size = dataSize; 483 } 484 485 static jint Image_imageGetPixelStride(JNIEnv* env, CpuConsumer::LockedBuffer* buffer, int idx, 486 int32_t readerFormat) 487 { 488 ALOGV("%s: buffer index: %d", __FUNCTION__, idx); 489 ALOG_ASSERT((idx < IMAGE_READER_MAX_NUM_PLANES) && (idx >= 0), "Index is out of range:%d", idx); 490 491 int pixelStride = 0; 492 ALOG_ASSERT(buffer != NULL, "buffer is NULL"); 493 494 int32_t fmt = buffer->flexFormat; 495 496 fmt = applyFormatOverrides(fmt, readerFormat); 497 498 switch (fmt) { 499 case HAL_PIXEL_FORMAT_YCbCr_420_888: 500 pixelStride = (idx == 0) ? 1 : buffer->chromaStep; 501 break; 502 case HAL_PIXEL_FORMAT_YCrCb_420_SP: 503 pixelStride = (idx == 0) ? 1 : 2; 504 break; 505 case HAL_PIXEL_FORMAT_Y8: 506 // Single plane 8bpp data. 507 ALOG_ASSERT(idx == 0, "Wrong index: %d", idx); 508 pixelStride; 509 break; 510 case HAL_PIXEL_FORMAT_YV12: 511 pixelStride = 1; 512 break; 513 case HAL_PIXEL_FORMAT_BLOB: 514 case HAL_PIXEL_FORMAT_RAW10: 515 // Blob is used for JPEG data, RAW10 is used for 10-bit raw data, they are 516 // single plane, row and pixel strides are 0. 517 ALOG_ASSERT(idx == 0, "Wrong index: %d", idx); 518 pixelStride = 0; 519 break; 520 case HAL_PIXEL_FORMAT_Y16: 521 case HAL_PIXEL_FORMAT_RAW_SENSOR: 522 case HAL_PIXEL_FORMAT_RGB_565: 523 // Single plane 16bpp data. 524 ALOG_ASSERT(idx == 0, "Wrong index: %d", idx); 525 pixelStride = 2; 526 break; 527 case HAL_PIXEL_FORMAT_RGBA_8888: 528 case HAL_PIXEL_FORMAT_RGBX_8888: 529 ALOG_ASSERT(idx == 0, "Wrong index: %d", idx); 530 pixelStride = 4; 531 break; 532 case HAL_PIXEL_FORMAT_RGB_888: 533 // Single plane, 24bpp. 534 ALOG_ASSERT(idx == 0, "Wrong index: %d", idx); 535 pixelStride = 3; 536 break; 537 default: 538 jniThrowExceptionFmt(env, "java/lang/UnsupportedOperationException", 539 "Pixel format: 0x%x is unsupported", fmt); 540 break; 541 } 542 543 return pixelStride; 544 } 545 546 static jint Image_imageGetRowStride(JNIEnv* env, CpuConsumer::LockedBuffer* buffer, int idx, 547 int32_t readerFormat) 548 { 549 ALOGV("%s: buffer index: %d", __FUNCTION__, idx); 550 ALOG_ASSERT((idx < IMAGE_READER_MAX_NUM_PLANES) && (idx >= 0)); 551 552 int rowStride = 0; 553 ALOG_ASSERT(buffer != NULL, "buffer is NULL"); 554 555 int32_t fmt = buffer->flexFormat; 556 557 fmt = applyFormatOverrides(fmt, readerFormat); 558 559 switch (fmt) { 560 case HAL_PIXEL_FORMAT_YCbCr_420_888: 561 rowStride = (idx == 0) ? buffer->stride : buffer->chromaStride; 562 break; 563 case HAL_PIXEL_FORMAT_YCrCb_420_SP: 564 rowStride = buffer->width; 565 break; 566 case HAL_PIXEL_FORMAT_YV12: 567 LOG_ALWAYS_FATAL_IF(buffer->stride % 16, 568 "Stride is not 16 pixel aligned %d", buffer->stride); 569 rowStride = (idx == 0) ? buffer->stride : ALIGN(buffer->stride / 2, 16); 570 break; 571 case HAL_PIXEL_FORMAT_BLOB: 572 // Blob is used for JPEG data, RAW10 is used for 10-bit raw data, they are 573 // single plane, row and pixel strides are 0. 574 ALOG_ASSERT(idx == 0, "Wrong index: %d", idx); 575 rowStride = 0; 576 break; 577 case HAL_PIXEL_FORMAT_RAW10: 578 ALOG_ASSERT(idx == 0, "Wrong index: %d", idx); 579 rowStride = buffer->stride; 580 break; 581 case HAL_PIXEL_FORMAT_Y8: 582 ALOG_ASSERT(idx == 0, "Wrong index: %d", idx); 583 LOG_ALWAYS_FATAL_IF(buffer->stride % 16, 584 "Stride is not 16 pixel aligned %d", buffer->stride); 585 rowStride = buffer->stride; 586 break; 587 case HAL_PIXEL_FORMAT_Y16: 588 case HAL_PIXEL_FORMAT_RAW_SENSOR: 589 // In native side, strides are specified in pixels, not in bytes. 590 // Single plane 16bpp bayer data. even width/height, 591 // row stride multiple of 16 pixels (32 bytes) 592 ALOG_ASSERT(idx == 0, "Wrong index: %d", idx); 593 LOG_ALWAYS_FATAL_IF(buffer->stride % 16, 594 "Stride is not 16 pixel aligned %d", buffer->stride); 595 rowStride = buffer->stride * 2; 596 break; 597 case HAL_PIXEL_FORMAT_RGB_565: 598 ALOG_ASSERT(idx == 0, "Wrong index: %d", idx); 599 rowStride = buffer->stride * 2; 600 break; 601 case HAL_PIXEL_FORMAT_RGBA_8888: 602 case HAL_PIXEL_FORMAT_RGBX_8888: 603 ALOG_ASSERT(idx == 0, "Wrong index: %d", idx); 604 rowStride = buffer->stride * 4; 605 break; 606 case HAL_PIXEL_FORMAT_RGB_888: 607 // Single plane, 24bpp. 608 ALOG_ASSERT(idx == 0, "Wrong index: %d", idx); 609 rowStride = buffer->stride * 3; 610 break; 611 default: 612 ALOGE("%s Pixel format: 0x%x is unsupported", __FUNCTION__, fmt); 613 jniThrowException(env, "java/lang/UnsupportedOperationException", 614 "unsupported buffer format"); 615 break; 616 } 617 618 return rowStride; 619 } 620 621 static int Image_getBufferWidth(CpuConsumer::LockedBuffer* buffer) { 622 if (buffer == NULL) return -1; 623 624 if (!buffer->crop.isEmpty()) { 625 return buffer->crop.getWidth(); 626 } 627 return buffer->width; 628 } 629 630 static int Image_getBufferHeight(CpuConsumer::LockedBuffer* buffer) { 631 if (buffer == NULL) return -1; 632 633 if (!buffer->crop.isEmpty()) { 634 return buffer->crop.getHeight(); 635 } 636 return buffer->height; 637 } 638 639 // ---------------------------------------------------------------------------- 640 641 static void ImageReader_classInit(JNIEnv* env, jclass clazz) 642 { 643 ALOGV("%s:", __FUNCTION__); 644 645 jclass imageClazz = env->FindClass("android/media/ImageReader$SurfaceImage"); 646 LOG_ALWAYS_FATAL_IF(imageClazz == NULL, 647 "can't find android/graphics/ImageReader$SurfaceImage"); 648 gSurfaceImageClassInfo.mLockedBuffer = env->GetFieldID( 649 imageClazz, ANDROID_MEDIA_SURFACEIMAGE_BUFFER_JNI_ID, "J"); 650 LOG_ALWAYS_FATAL_IF(gSurfaceImageClassInfo.mLockedBuffer == NULL, 651 "can't find android/graphics/ImageReader.%s", 652 ANDROID_MEDIA_SURFACEIMAGE_BUFFER_JNI_ID); 653 654 gSurfaceImageClassInfo.mTimestamp = env->GetFieldID( 655 imageClazz, ANDROID_MEDIA_SURFACEIMAGE_TS_JNI_ID, "J"); 656 LOG_ALWAYS_FATAL_IF(gSurfaceImageClassInfo.mTimestamp == NULL, 657 "can't find android/graphics/ImageReader.%s", 658 ANDROID_MEDIA_SURFACEIMAGE_TS_JNI_ID); 659 660 gImageReaderClassInfo.mNativeContext = env->GetFieldID( 661 clazz, ANDROID_MEDIA_IMAGEREADER_CTX_JNI_ID, "J"); 662 LOG_ALWAYS_FATAL_IF(gImageReaderClassInfo.mNativeContext == NULL, 663 "can't find android/graphics/ImageReader.%s", 664 ANDROID_MEDIA_IMAGEREADER_CTX_JNI_ID); 665 666 gImageReaderClassInfo.postEventFromNative = env->GetStaticMethodID( 667 clazz, "postEventFromNative", "(Ljava/lang/Object;)V"); 668 LOG_ALWAYS_FATAL_IF(gImageReaderClassInfo.postEventFromNative == NULL, 669 "can't find android/graphics/ImageReader.postEventFromNative"); 670 671 jclass planeClazz = env->FindClass("android/media/ImageReader$SurfaceImage$SurfacePlane"); 672 LOG_ALWAYS_FATAL_IF(planeClazz == NULL, "Can not find SurfacePlane class"); 673 // FindClass only gives a local reference of jclass object. 674 gSurfacePlaneClassInfo.clazz = (jclass) env->NewGlobalRef(planeClazz); 675 gSurfacePlaneClassInfo.ctor = env->GetMethodID(gSurfacePlaneClassInfo.clazz, "<init>", 676 "(Landroid/media/ImageReader$SurfaceImage;III)V"); 677 LOG_ALWAYS_FATAL_IF(gSurfacePlaneClassInfo.ctor == NULL, 678 "Can not find SurfacePlane constructor"); 679 } 680 681 static void ImageReader_init(JNIEnv* env, jobject thiz, jobject weakThiz, 682 jint width, jint height, jint format, jint maxImages) 683 { 684 status_t res; 685 int nativeFormat; 686 687 ALOGV("%s: width:%d, height: %d, format: 0x%x, maxImages:%d", 688 __FUNCTION__, width, height, format, maxImages); 689 690 nativeFormat = Image_getPixelFormat(env, format); 691 692 sp<IGraphicBufferProducer> gbProducer; 693 sp<IGraphicBufferConsumer> gbConsumer; 694 BufferQueue::createBufferQueue(&gbProducer, &gbConsumer); 695 sp<CpuConsumer> consumer = new CpuConsumer(gbConsumer, maxImages, 696 /*controlledByApp*/true); 697 // TODO: throw dvm exOutOfMemoryError? 698 if (consumer == NULL) { 699 jniThrowRuntimeException(env, "Failed to allocate native CpuConsumer"); 700 return; 701 } 702 703 jclass clazz = env->GetObjectClass(thiz); 704 if (clazz == NULL) { 705 jniThrowRuntimeException(env, "Can't find android/graphics/ImageReader"); 706 return; 707 } 708 sp<JNIImageReaderContext> ctx(new JNIImageReaderContext(env, weakThiz, clazz, maxImages)); 709 ctx->setCpuConsumer(consumer); 710 ctx->setProducer(gbProducer); 711 consumer->setFrameAvailableListener(ctx); 712 ImageReader_setNativeContext(env, thiz, ctx); 713 ctx->setBufferFormat(nativeFormat); 714 ctx->setBufferWidth(width); 715 ctx->setBufferHeight(height); 716 717 // Set the width/height/format to the CpuConsumer 718 res = consumer->setDefaultBufferSize(width, height); 719 if (res != OK) { 720 jniThrowException(env, "java/lang/IllegalStateException", 721 "Failed to set CpuConsumer buffer size"); 722 return; 723 } 724 res = consumer->setDefaultBufferFormat(nativeFormat); 725 if (res != OK) { 726 jniThrowException(env, "java/lang/IllegalStateException", 727 "Failed to set CpuConsumer buffer format"); 728 } 729 } 730 731 static void ImageReader_close(JNIEnv* env, jobject thiz) 732 { 733 ALOGV("%s:", __FUNCTION__); 734 735 JNIImageReaderContext* const ctx = ImageReader_getContext(env, thiz); 736 if (ctx == NULL) { 737 // ImageReader is already closed. 738 return; 739 } 740 741 CpuConsumer* consumer = ImageReader_getCpuConsumer(env, thiz); 742 if (consumer != NULL) { 743 consumer->abandon(); 744 consumer->setFrameAvailableListener(NULL); 745 } 746 ImageReader_setNativeContext(env, thiz, NULL); 747 } 748 749 static void ImageReader_imageRelease(JNIEnv* env, jobject thiz, jobject image) 750 { 751 ALOGV("%s:", __FUNCTION__); 752 JNIImageReaderContext* ctx = ImageReader_getContext(env, thiz); 753 if (ctx == NULL) { 754 ALOGW("ImageReader#close called before Image#close, consider calling Image#close first"); 755 return; 756 } 757 758 CpuConsumer* consumer = ctx->getCpuConsumer(); 759 CpuConsumer::LockedBuffer* buffer = Image_getLockedBuffer(env, image); 760 if (!buffer) { 761 ALOGW("Image already released!!!"); 762 return; 763 } 764 consumer->unlockBuffer(*buffer); 765 Image_setBuffer(env, image, NULL); 766 ctx->returnLockedBuffer(buffer); 767 } 768 769 static jint ImageReader_imageSetup(JNIEnv* env, jobject thiz, 770 jobject image) 771 { 772 ALOGV("%s:", __FUNCTION__); 773 JNIImageReaderContext* ctx = ImageReader_getContext(env, thiz); 774 if (ctx == NULL) { 775 jniThrowRuntimeException(env, "ImageReaderContext is not initialized"); 776 return -1; 777 } 778 779 CpuConsumer* consumer = ctx->getCpuConsumer(); 780 CpuConsumer::LockedBuffer* buffer = ctx->getLockedBuffer(); 781 if (buffer == NULL) { 782 ALOGW("Unable to acquire a lockedBuffer, very likely client tries to lock more than" 783 " maxImages buffers"); 784 return ACQUIRE_MAX_IMAGES; 785 } 786 status_t res = consumer->lockNextBuffer(buffer); 787 if (res != NO_ERROR) { 788 ctx->returnLockedBuffer(buffer); 789 if (res != BAD_VALUE /*no buffers*/) { 790 if (res == NOT_ENOUGH_DATA) { 791 return ACQUIRE_MAX_IMAGES; 792 } else { 793 ALOGE("%s Fail to lockNextBuffer with error: %d ", 794 __FUNCTION__, res); 795 jniThrowExceptionFmt(env, "java/lang/AssertionError", 796 "Unknown error (%d) when we tried to lock buffer.", 797 res); 798 } 799 } 800 return ACQUIRE_NO_BUFFERS; 801 } 802 803 if (buffer->flexFormat == HAL_PIXEL_FORMAT_YCrCb_420_SP) { 804 jniThrowException(env, "java/lang/UnsupportedOperationException", 805 "NV21 format is not supported by ImageReader"); 806 return -1; 807 } 808 809 // Check if the left-top corner of the crop rect is origin, we currently assume this point is 810 // zero, will revist this once this assumption turns out problematic. 811 Point lt = buffer->crop.leftTop(); 812 if (lt.x != 0 || lt.y != 0) { 813 jniThrowExceptionFmt(env, "java/lang/UnsupportedOperationException", 814 "crop left top corner [%d, %d] need to be at origin", lt.x, lt.y); 815 return -1; 816 } 817 818 // Check if the producer buffer configurations match what ImageReader configured. 819 int outputWidth = Image_getBufferWidth(buffer); 820 int outputHeight = Image_getBufferHeight(buffer); 821 822 int imgReaderFmt = ctx->getBufferFormat(); 823 int imageReaderWidth = ctx->getBufferWidth(); 824 int imageReaderHeight = ctx->getBufferHeight(); 825 if ((buffer->format != HAL_PIXEL_FORMAT_BLOB) && (imgReaderFmt != HAL_PIXEL_FORMAT_BLOB) && 826 (imageReaderWidth != outputWidth || imageReaderHeight != outputHeight)) { 827 ALOGV("%s: Producer buffer size: %dx%d, doesn't match ImageReader configured size: %dx%d", 828 __FUNCTION__, outputWidth, outputHeight, imageReaderWidth, imageReaderHeight); 829 } 830 831 int bufFmt = buffer->format; 832 if (imgReaderFmt == HAL_PIXEL_FORMAT_YCbCr_420_888) { 833 bufFmt = buffer->flexFormat; 834 } 835 if (imgReaderFmt != bufFmt) { 836 if (imgReaderFmt == HAL_PIXEL_FORMAT_YCbCr_420_888 && (bufFmt == 837 HAL_PIXEL_FORMAT_YCrCb_420_SP || bufFmt == HAL_PIXEL_FORMAT_YV12)) { 838 // Special casing for when producer switches to a format compatible with flexible YUV 839 // (HAL_PIXEL_FORMAT_YCbCr_420_888). 840 ctx->setBufferFormat(bufFmt); 841 ALOGD("%s: Overriding buffer format YUV_420_888 to %x.", __FUNCTION__, bufFmt); 842 } else if (imgReaderFmt == HAL_PIXEL_FORMAT_BLOB && bufFmt == HAL_PIXEL_FORMAT_RGBA_8888) { 843 // Using HAL_PIXEL_FORMAT_RGBA_8888 gralloc buffers containing JPEGs to get around SW 844 // write limitations for (b/17379185). 845 ALOGD("%s: Receiving JPEG in HAL_PIXEL_FORMAT_RGBA_8888 buffer.", __FUNCTION__); 846 } else { 847 // Return the buffer to the queue. 848 consumer->unlockBuffer(*buffer); 849 ctx->returnLockedBuffer(buffer); 850 851 // Throw exception 852 ALOGE("Producer output buffer format: 0x%x, ImageReader configured format: 0x%x", 853 buffer->format, ctx->getBufferFormat()); 854 String8 msg; 855 msg.appendFormat("The producer output buffer format 0x%x doesn't " 856 "match the ImageReader's configured buffer format 0x%x.", 857 bufFmt, ctx->getBufferFormat()); 858 jniThrowException(env, "java/lang/UnsupportedOperationException", 859 msg.string()); 860 return -1; 861 } 862 } 863 // Set SurfaceImage instance member variables 864 Image_setBuffer(env, image, buffer); 865 env->SetLongField(image, gSurfaceImageClassInfo.mTimestamp, 866 static_cast<jlong>(buffer->timestamp)); 867 868 return ACQUIRE_SUCCESS; 869 } 870 871 static jobject ImageReader_getSurface(JNIEnv* env, jobject thiz) 872 { 873 ALOGV("%s: ", __FUNCTION__); 874 875 IGraphicBufferProducer* gbp = ImageReader_getProducer(env, thiz); 876 if (gbp == NULL) { 877 jniThrowRuntimeException(env, "CpuConsumer is uninitialized"); 878 return NULL; 879 } 880 881 // Wrap the IGBP in a Java-language Surface. 882 return android_view_Surface_createFromIGraphicBufferProducer(env, gbp); 883 } 884 885 static jobject Image_createSurfacePlane(JNIEnv* env, jobject thiz, int idx, int readerFormat) 886 { 887 int rowStride, pixelStride; 888 ALOGV("%s: buffer index: %d", __FUNCTION__, idx); 889 890 CpuConsumer::LockedBuffer* buffer = Image_getLockedBuffer(env, thiz); 891 892 ALOG_ASSERT(buffer != NULL); 893 if (buffer == NULL) { 894 jniThrowException(env, "java/lang/IllegalStateException", "Image was released"); 895 } 896 897 readerFormat = Image_getPixelFormat(env, readerFormat); 898 899 rowStride = Image_imageGetRowStride(env, buffer, idx, readerFormat); 900 pixelStride = Image_imageGetPixelStride(env, buffer, idx, readerFormat); 901 902 jobject surfPlaneObj = env->NewObject(gSurfacePlaneClassInfo.clazz, 903 gSurfacePlaneClassInfo.ctor, thiz, idx, rowStride, pixelStride); 904 905 return surfPlaneObj; 906 } 907 908 static jobject Image_getByteBuffer(JNIEnv* env, jobject thiz, int idx, int readerFormat) 909 { 910 uint8_t *base = NULL; 911 uint32_t size = 0; 912 jobject byteBuffer; 913 914 ALOGV("%s: buffer index: %d", __FUNCTION__, idx); 915 916 CpuConsumer::LockedBuffer* buffer = Image_getLockedBuffer(env, thiz); 917 918 if (buffer == NULL) { 919 jniThrowException(env, "java/lang/IllegalStateException", "Image was released"); 920 } 921 922 readerFormat = Image_getPixelFormat(env, readerFormat); 923 924 // Create byteBuffer from native buffer 925 Image_getLockedBufferInfo(env, buffer, idx, &base, &size, readerFormat); 926 927 if (size > static_cast<uint32_t>(INT32_MAX)) { 928 // Byte buffer have 'int capacity', so check the range 929 jniThrowExceptionFmt(env, "java/lang/IllegalStateException", 930 "Size too large for bytebuffer capacity %" PRIu32, size); 931 return NULL; 932 } 933 934 byteBuffer = env->NewDirectByteBuffer(base, size); 935 // TODO: throw dvm exOutOfMemoryError? 936 if ((byteBuffer == NULL) && (env->ExceptionCheck() == false)) { 937 jniThrowException(env, "java/lang/IllegalStateException", "Failed to allocate ByteBuffer"); 938 } 939 940 return byteBuffer; 941 } 942 943 static jint Image_getWidth(JNIEnv* env, jobject thiz) 944 { 945 CpuConsumer::LockedBuffer* buffer = Image_getLockedBuffer(env, thiz); 946 return Image_getBufferWidth(buffer); 947 } 948 949 static jint Image_getHeight(JNIEnv* env, jobject thiz) 950 { 951 CpuConsumer::LockedBuffer* buffer = Image_getLockedBuffer(env, thiz); 952 return Image_getBufferHeight(buffer); 953 } 954 955 956 } // extern "C" 957 958 // ---------------------------------------------------------------------------- 959 960 static JNINativeMethod gImageReaderMethods[] = { 961 {"nativeClassInit", "()V", (void*)ImageReader_classInit }, 962 {"nativeInit", "(Ljava/lang/Object;IIII)V", (void*)ImageReader_init }, 963 {"nativeClose", "()V", (void*)ImageReader_close }, 964 {"nativeReleaseImage", "(Landroid/media/Image;)V", (void*)ImageReader_imageRelease }, 965 {"nativeImageSetup", "(Landroid/media/Image;)I", (void*)ImageReader_imageSetup }, 966 {"nativeGetSurface", "()Landroid/view/Surface;", (void*)ImageReader_getSurface }, 967 }; 968 969 static JNINativeMethod gImageMethods[] = { 970 {"nativeImageGetBuffer", "(II)Ljava/nio/ByteBuffer;", (void*)Image_getByteBuffer }, 971 {"nativeCreatePlane", "(II)Landroid/media/ImageReader$SurfaceImage$SurfacePlane;", 972 (void*)Image_createSurfacePlane }, 973 {"nativeGetWidth", "()I", (void*)Image_getWidth }, 974 {"nativeGetHeight", "()I", (void*)Image_getHeight }, 975 }; 976 977 int register_android_media_ImageReader(JNIEnv *env) { 978 979 int ret1 = AndroidRuntime::registerNativeMethods(env, 980 "android/media/ImageReader", gImageReaderMethods, NELEM(gImageReaderMethods)); 981 982 int ret2 = AndroidRuntime::registerNativeMethods(env, 983 "android/media/ImageReader$SurfaceImage", gImageMethods, NELEM(gImageMethods)); 984 985 return (ret1 || ret2); 986 } 987
