1 /* 2 * Copyright 2015 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17 //#define LOG_NDEBUG 0 18 #define LOG_TAG "ImageWriter_JNI" 19 #include <utils/Log.h> 20 #include <utils/String8.h> 21 22 #include <gui/IProducerListener.h> 23 #include <gui/Surface.h> 24 #include <gui/CpuConsumer.h> 25 #include <android_runtime/AndroidRuntime.h> 26 #include <android_runtime/android_view_Surface.h> 27 #include <camera3.h> 28 29 #include <jni.h> 30 #include <JNIHelp.h> 31 32 #include <stdint.h> 33 #include <inttypes.h> 34 35 #define ALIGN(x, mask) ( ((x) + (mask) - 1) & ~((mask) - 1) ) 36 37 #define IMAGE_BUFFER_JNI_ID "mNativeBuffer" 38 39 // ---------------------------------------------------------------------------- 40 41 using namespace android; 42 43 enum { 44 IMAGE_WRITER_MAX_NUM_PLANES = 3, 45 }; 46 47 static struct { 48 jmethodID postEventFromNative; 49 jfieldID mWriterFormat; 50 } gImageWriterClassInfo; 51 52 static struct { 53 jfieldID mNativeBuffer; 54 jfieldID mNativeFenceFd; 55 jfieldID mPlanes; 56 } gSurfaceImageClassInfo; 57 58 static struct { 59 jclass clazz; 60 jmethodID ctor; 61 } gSurfacePlaneClassInfo; 62 63 typedef CpuConsumer::LockedBuffer LockedImage; 64 65 // ---------------------------------------------------------------------------- 66 67 class JNIImageWriterContext : public BnProducerListener { 68 public: 69 JNIImageWriterContext(JNIEnv* env, jobject weakThiz, jclass clazz); 70 71 virtual ~JNIImageWriterContext(); 72 73 // Implementation of IProducerListener, used to notify the ImageWriter that the consumer 74 // has returned a buffer and it is ready for ImageWriter to dequeue. 75 virtual void onBufferReleased(); 76 77 void setProducer(const sp<Surface>& producer) { mProducer = producer; } 78 Surface* getProducer() { return mProducer.get(); } 79 80 void setBufferFormat(int format) { mFormat = format; } 81 int getBufferFormat() { return mFormat; } 82 83 void setBufferWidth(int width) { mWidth = width; } 84 int getBufferWidth() { return mWidth; } 85 86 void setBufferHeight(int height) { mHeight = height; } 87 int getBufferHeight() { return mHeight; } 88 89 private: 90 static JNIEnv* getJNIEnv(bool* needsDetach); 91 static void detachJNI(); 92 93 sp<Surface> mProducer; 94 jobject mWeakThiz; 95 jclass mClazz; 96 int mFormat; 97 int mWidth; 98 int mHeight; 99 }; 100 101 JNIImageWriterContext::JNIImageWriterContext(JNIEnv* env, jobject weakThiz, jclass clazz) : 102 mWeakThiz(env->NewGlobalRef(weakThiz)), 103 mClazz((jclass)env->NewGlobalRef(clazz)), 104 mFormat(0), 105 mWidth(-1), 106 mHeight(-1) { 107 } 108 109 JNIImageWriterContext::~JNIImageWriterContext() { 110 ALOGV("%s", __FUNCTION__); 111 bool needsDetach = false; 112 JNIEnv* env = getJNIEnv(&needsDetach); 113 if (env != NULL) { 114 env->DeleteGlobalRef(mWeakThiz); 115 env->DeleteGlobalRef(mClazz); 116 } else { 117 ALOGW("leaking JNI object references"); 118 } 119 if (needsDetach) { 120 detachJNI(); 121 } 122 123 mProducer.clear(); 124 } 125 126 JNIEnv* JNIImageWriterContext::getJNIEnv(bool* needsDetach) { 127 ALOGV("%s", __FUNCTION__); 128 LOG_ALWAYS_FATAL_IF(needsDetach == NULL, "needsDetach is null!!!"); 129 *needsDetach = false; 130 JNIEnv* env = AndroidRuntime::getJNIEnv(); 131 if (env == NULL) { 132 JavaVMAttachArgs args = {JNI_VERSION_1_4, NULL, NULL}; 133 JavaVM* vm = AndroidRuntime::getJavaVM(); 134 int result = vm->AttachCurrentThread(&env, (void*) &args); 135 if (result != JNI_OK) { 136 ALOGE("thread attach failed: %#x", result); 137 return NULL; 138 } 139 *needsDetach = true; 140 } 141 return env; 142 } 143 144 void JNIImageWriterContext::detachJNI() { 145 ALOGV("%s", __FUNCTION__); 146 JavaVM* vm = AndroidRuntime::getJavaVM(); 147 int result = vm->DetachCurrentThread(); 148 if (result != JNI_OK) { 149 ALOGE("thread detach failed: %#x", result); 150 } 151 } 152 153 void JNIImageWriterContext::onBufferReleased() { 154 ALOGV("%s: buffer released", __FUNCTION__); 155 bool needsDetach = false; 156 JNIEnv* env = getJNIEnv(&needsDetach); 157 if (env != NULL) { 158 // Detach the buffer every time when a buffer consumption is done, 159 // need let this callback give a BufferItem, then only detach if it was attached to this 160 // Writer. Do the detach unconditionally for opaque format now. see b/19977520 161 if (mFormat == HAL_PIXEL_FORMAT_IMPLEMENTATION_DEFINED) { 162 sp<Fence> fence; 163 sp<GraphicBuffer> buffer; 164 ALOGV("%s: One buffer is detached", __FUNCTION__); 165 mProducer->detachNextBuffer(&buffer, &fence); 166 } 167 168 env->CallStaticVoidMethod(mClazz, gImageWriterClassInfo.postEventFromNative, mWeakThiz); 169 } else { 170 ALOGW("onBufferReleased event will not posted"); 171 } 172 173 if (needsDetach) { 174 detachJNI(); 175 } 176 } 177 178 // ---------------------------------------------------------------------------- 179 180 extern "C" { 181 182 // -------------------------------Private method declarations-------------- 183 184 static bool isPossiblyYUV(PixelFormat format); 185 static void Image_setNativeContext(JNIEnv* env, jobject thiz, 186 sp<GraphicBuffer> buffer, int fenceFd); 187 static void Image_getNativeContext(JNIEnv* env, jobject thiz, 188 GraphicBuffer** buffer, int* fenceFd); 189 static void Image_unlockIfLocked(JNIEnv* env, jobject thiz); 190 static bool isFormatOpaque(int format); 191 192 // --------------------------ImageWriter methods--------------------------------------- 193 194 static void ImageWriter_classInit(JNIEnv* env, jclass clazz) { 195 ALOGV("%s:", __FUNCTION__); 196 jclass imageClazz = env->FindClass("android/media/ImageWriter$WriterSurfaceImage"); 197 LOG_ALWAYS_FATAL_IF(imageClazz == NULL, 198 "can't find android/media/ImageWriter$WriterSurfaceImage"); 199 gSurfaceImageClassInfo.mNativeBuffer = env->GetFieldID( 200 imageClazz, IMAGE_BUFFER_JNI_ID, "J"); 201 LOG_ALWAYS_FATAL_IF(gSurfaceImageClassInfo.mNativeBuffer == NULL, 202 "can't find android/media/ImageWriter$WriterSurfaceImage.%s", IMAGE_BUFFER_JNI_ID); 203 204 gSurfaceImageClassInfo.mNativeFenceFd = env->GetFieldID( 205 imageClazz, "mNativeFenceFd", "I"); 206 LOG_ALWAYS_FATAL_IF(gSurfaceImageClassInfo.mNativeFenceFd == NULL, 207 "can't find android/media/ImageWriter$WriterSurfaceImage.mNativeFenceFd"); 208 209 gSurfaceImageClassInfo.mPlanes = env->GetFieldID( 210 imageClazz, "mPlanes", "[Landroid/media/ImageWriter$WriterSurfaceImage$SurfacePlane;"); 211 LOG_ALWAYS_FATAL_IF(gSurfaceImageClassInfo.mPlanes == NULL, 212 "can't find android/media/ImageWriter$WriterSurfaceImage.mPlanes"); 213 214 gImageWriterClassInfo.postEventFromNative = env->GetStaticMethodID( 215 clazz, "postEventFromNative", "(Ljava/lang/Object;)V"); 216 LOG_ALWAYS_FATAL_IF(gImageWriterClassInfo.postEventFromNative == NULL, 217 "can't find android/media/ImageWriter.postEventFromNative"); 218 219 gImageWriterClassInfo.mWriterFormat = env->GetFieldID( 220 clazz, "mWriterFormat", "I"); 221 LOG_ALWAYS_FATAL_IF(gImageWriterClassInfo.mWriterFormat == NULL, 222 "can't find android/media/ImageWriter.mWriterFormat"); 223 224 jclass planeClazz = env->FindClass("android/media/ImageWriter$WriterSurfaceImage$SurfacePlane"); 225 LOG_ALWAYS_FATAL_IF(planeClazz == NULL, "Can not find SurfacePlane class"); 226 // FindClass only gives a local reference of jclass object. 227 gSurfacePlaneClassInfo.clazz = (jclass) env->NewGlobalRef(planeClazz); 228 gSurfacePlaneClassInfo.ctor = env->GetMethodID(gSurfacePlaneClassInfo.clazz, "<init>", 229 "(Landroid/media/ImageWriter$WriterSurfaceImage;IILjava/nio/ByteBuffer;)V"); 230 LOG_ALWAYS_FATAL_IF(gSurfacePlaneClassInfo.ctor == NULL, 231 "Can not find SurfacePlane constructor"); 232 } 233 234 static jlong ImageWriter_init(JNIEnv* env, jobject thiz, jobject weakThiz, jobject jsurface, 235 jint maxImages) { 236 status_t res; 237 238 ALOGV("%s: maxImages:%d", __FUNCTION__, maxImages); 239 240 sp<Surface> surface(android_view_Surface_getSurface(env, jsurface)); 241 if (surface == NULL) { 242 jniThrowException(env, 243 "java/lang/IllegalArgumentException", 244 "The surface has been released"); 245 return 0; 246 } 247 sp<IGraphicBufferProducer> bufferProducer = surface->getIGraphicBufferProducer(); 248 249 jclass clazz = env->GetObjectClass(thiz); 250 if (clazz == NULL) { 251 jniThrowRuntimeException(env, "Can't find android/graphics/ImageWriter"); 252 return 0; 253 } 254 sp<JNIImageWriterContext> ctx(new JNIImageWriterContext(env, weakThiz, clazz)); 255 256 sp<Surface> producer = new Surface(bufferProducer, /*controlledByApp*/false); 257 ctx->setProducer(producer); 258 /** 259 * NATIVE_WINDOW_API_CPU isn't a good choice here, as it makes the bufferQueue not connectable 260 * after disconnect. MEDIA or CAMERA are treated the same internally. The producer listener 261 * will be cleared after disconnect call. 262 */ 263 producer->connect(/*api*/NATIVE_WINDOW_API_CAMERA, /*listener*/ctx); 264 jlong nativeCtx = reinterpret_cast<jlong>(ctx.get()); 265 266 // Get the dimension and format of the producer. 267 sp<ANativeWindow> anw = producer; 268 int32_t width, height, format; 269 if ((res = anw->query(anw.get(), NATIVE_WINDOW_WIDTH, &width)) != OK) { 270 ALOGE("%s: Query Surface width failed: %s (%d)", __FUNCTION__, strerror(-res), res); 271 jniThrowRuntimeException(env, "Failed to query Surface width"); 272 return 0; 273 } 274 ctx->setBufferWidth(width); 275 276 if ((res = anw->query(anw.get(), NATIVE_WINDOW_HEIGHT, &height)) != OK) { 277 ALOGE("%s: Query Surface height failed: %s (%d)", __FUNCTION__, strerror(-res), res); 278 jniThrowRuntimeException(env, "Failed to query Surface height"); 279 return 0; 280 } 281 ctx->setBufferHeight(height); 282 283 if ((res = anw->query(anw.get(), NATIVE_WINDOW_FORMAT, &format)) != OK) { 284 ALOGE("%s: Query Surface format failed: %s (%d)", __FUNCTION__, strerror(-res), res); 285 jniThrowRuntimeException(env, "Failed to query Surface format"); 286 return 0; 287 } 288 ctx->setBufferFormat(format); 289 env->SetIntField(thiz, gImageWriterClassInfo.mWriterFormat, reinterpret_cast<jint>(format)); 290 291 292 if (!isFormatOpaque(format)) { 293 res = native_window_set_usage(anw.get(), GRALLOC_USAGE_SW_WRITE_OFTEN); 294 if (res != OK) { 295 ALOGE("%s: Configure usage %08x for format %08x failed: %s (%d)", 296 __FUNCTION__, static_cast<unsigned int>(GRALLOC_USAGE_SW_WRITE_OFTEN), 297 format, strerror(-res), res); 298 jniThrowRuntimeException(env, "Failed to SW_WRITE_OFTEN configure usage"); 299 return 0; 300 } 301 } 302 303 int minUndequeuedBufferCount = 0; 304 res = anw->query(anw.get(), 305 NATIVE_WINDOW_MIN_UNDEQUEUED_BUFFERS, &minUndequeuedBufferCount); 306 if (res != OK) { 307 ALOGE("%s: Query producer undequeued buffer count failed: %s (%d)", 308 __FUNCTION__, strerror(-res), res); 309 jniThrowRuntimeException(env, "Query producer undequeued buffer count failed"); 310 return 0; 311 } 312 313 size_t totalBufferCount = maxImages + minUndequeuedBufferCount; 314 res = native_window_set_buffer_count(anw.get(), totalBufferCount); 315 if (res != OK) { 316 ALOGE("%s: Set buffer count failed: %s (%d)", __FUNCTION__, strerror(-res), res); 317 jniThrowRuntimeException(env, "Set buffer count failed"); 318 return 0; 319 } 320 321 if (ctx != 0) { 322 ctx->incStrong((void*)ImageWriter_init); 323 } 324 return nativeCtx; 325 } 326 327 static void ImageWriter_dequeueImage(JNIEnv* env, jobject thiz, jlong nativeCtx, jobject image) { 328 ALOGV("%s", __FUNCTION__); 329 JNIImageWriterContext* const ctx = reinterpret_cast<JNIImageWriterContext *>(nativeCtx); 330 if (ctx == NULL || thiz == NULL) { 331 jniThrowException(env, "java/lang/IllegalStateException", 332 "ImageWriterContext is not initialized"); 333 return; 334 } 335 336 sp<ANativeWindow> anw = ctx->getProducer(); 337 android_native_buffer_t *anb = NULL; 338 int fenceFd = -1; 339 status_t res = anw->dequeueBuffer(anw.get(), &anb, &fenceFd); 340 if (res != OK) { 341 ALOGE("%s: Dequeue buffer failed: %s (%d)", __FUNCTION__, strerror(-res), res); 342 switch (res) { 343 case NO_INIT: 344 jniThrowException(env, "java/lang/IllegalStateException", 345 "Surface has been abandoned"); 346 break; 347 default: 348 // TODO: handle other error cases here. 349 jniThrowRuntimeException(env, "dequeue buffer failed"); 350 } 351 return; 352 } 353 // New GraphicBuffer object doesn't own the handle, thus the native buffer 354 // won't be freed when this object is destroyed. 355 sp<GraphicBuffer> buffer(new GraphicBuffer(anb, /*keepOwnership*/false)); 356 357 // Note that: 358 // 1. No need to lock buffer now, will only lock it when the first getPlanes() is called. 359 // 2. Fence will be saved to mNativeFenceFd, and will consumed by lock/queue/cancel buffer 360 // later. 361 // 3. need use lockAsync here, as it will handle the dequeued fence for us automatically. 362 363 // Finally, set the native info into image object. 364 Image_setNativeContext(env, image, buffer, fenceFd); 365 } 366 367 static void ImageWriter_close(JNIEnv* env, jobject thiz, jlong nativeCtx) { 368 ALOGV("%s:", __FUNCTION__); 369 JNIImageWriterContext* const ctx = reinterpret_cast<JNIImageWriterContext *>(nativeCtx); 370 if (ctx == NULL || thiz == NULL) { 371 // ImageWriter is already closed. 372 return; 373 } 374 375 ANativeWindow* producer = ctx->getProducer(); 376 if (producer != NULL) { 377 /** 378 * NATIVE_WINDOW_API_CPU isn't a good choice here, as it makes the bufferQueue not 379 * connectable after disconnect. MEDIA or CAMERA are treated the same internally. 380 * The producer listener will be cleared after disconnect call. 381 */ 382 status_t res = native_window_api_disconnect(producer, /*api*/NATIVE_WINDOW_API_CAMERA); 383 /** 384 * This is not an error. if client calling process dies, the window will 385 * also die and all calls to it will return DEAD_OBJECT, thus it's already 386 * "disconnected" 387 */ 388 if (res == DEAD_OBJECT) { 389 ALOGW("%s: While disconnecting ImageWriter from native window, the" 390 " native window died already", __FUNCTION__); 391 } else if (res != OK) { 392 ALOGE("%s: native window disconnect failed: %s (%d)", 393 __FUNCTION__, strerror(-res), res); 394 jniThrowRuntimeException(env, "Native window disconnect failed"); 395 return; 396 } 397 } 398 399 ctx->decStrong((void*)ImageWriter_init); 400 } 401 402 static void ImageWriter_cancelImage(JNIEnv* env, jobject thiz, jlong nativeCtx, jobject image) { 403 ALOGV("%s", __FUNCTION__); 404 JNIImageWriterContext* const ctx = reinterpret_cast<JNIImageWriterContext *>(nativeCtx); 405 if (ctx == NULL || thiz == NULL) { 406 jniThrowException(env, "java/lang/IllegalStateException", 407 "ImageWriterContext is not initialized"); 408 return; 409 } 410 411 sp<ANativeWindow> anw = ctx->getProducer(); 412 413 GraphicBuffer *buffer = NULL; 414 int fenceFd = -1; 415 Image_getNativeContext(env, image, &buffer, &fenceFd); 416 if (buffer == NULL) { 417 jniThrowException(env, "java/lang/IllegalStateException", 418 "Image is not initialized"); 419 return; 420 } 421 422 // Unlock the image if it was locked 423 Image_unlockIfLocked(env, image); 424 425 anw->cancelBuffer(anw.get(), buffer, fenceFd); 426 427 Image_setNativeContext(env, image, NULL, -1); 428 } 429 430 static void ImageWriter_queueImage(JNIEnv* env, jobject thiz, jlong nativeCtx, jobject image, 431 jlong timestampNs, jint left, jint top, jint right, jint bottom) { 432 ALOGV("%s", __FUNCTION__); 433 JNIImageWriterContext* const ctx = reinterpret_cast<JNIImageWriterContext *>(nativeCtx); 434 if (ctx == NULL || thiz == NULL) { 435 jniThrowException(env, "java/lang/IllegalStateException", 436 "ImageWriterContext is not initialized"); 437 return; 438 } 439 440 status_t res = OK; 441 sp<ANativeWindow> anw = ctx->getProducer(); 442 443 GraphicBuffer *buffer = NULL; 444 int fenceFd = -1; 445 Image_getNativeContext(env, image, &buffer, &fenceFd); 446 if (buffer == NULL) { 447 jniThrowException(env, "java/lang/IllegalStateException", 448 "Image is not initialized"); 449 return; 450 } 451 452 // Unlock image if it was locked. 453 Image_unlockIfLocked(env, image); 454 455 // Set timestamp 456 ALOGV("timestamp to be queued: %" PRId64, timestampNs); 457 res = native_window_set_buffers_timestamp(anw.get(), timestampNs); 458 if (res != OK) { 459 jniThrowRuntimeException(env, "Set timestamp failed"); 460 return; 461 } 462 463 // Set crop 464 android_native_rect_t cropRect; 465 cropRect.left = left; 466 cropRect.top = top; 467 cropRect.right = right; 468 cropRect.bottom = bottom; 469 res = native_window_set_crop(anw.get(), &cropRect); 470 if (res != OK) { 471 jniThrowRuntimeException(env, "Set crop rect failed"); 472 return; 473 } 474 475 // Finally, queue input buffer 476 res = anw->queueBuffer(anw.get(), buffer, fenceFd); 477 if (res != OK) { 478 ALOGE("%s: Queue buffer failed: %s (%d)", __FUNCTION__, strerror(-res), res); 479 switch (res) { 480 case NO_INIT: 481 jniThrowException(env, "java/lang/IllegalStateException", 482 "Surface has been abandoned"); 483 break; 484 default: 485 // TODO: handle other error cases here. 486 jniThrowRuntimeException(env, "Queue input buffer failed"); 487 } 488 return; 489 } 490 491 // Clear the image native context: end of this image's lifecycle in public API. 492 Image_setNativeContext(env, image, NULL, -1); 493 } 494 495 static jint ImageWriter_attachAndQueueImage(JNIEnv* env, jobject thiz, jlong nativeCtx, 496 jlong nativeBuffer, jint imageFormat, jlong timestampNs, jint left, jint top, 497 jint right, jint bottom) { 498 ALOGV("%s", __FUNCTION__); 499 JNIImageWriterContext* const ctx = reinterpret_cast<JNIImageWriterContext *>(nativeCtx); 500 if (ctx == NULL || thiz == NULL) { 501 jniThrowException(env, "java/lang/IllegalStateException", 502 "ImageWriterContext is not initialized"); 503 return -1; 504 } 505 506 sp<Surface> surface = ctx->getProducer(); 507 status_t res = OK; 508 if (!isFormatOpaque(imageFormat)) { 509 // TODO: need implement, see b/19962027 510 jniThrowRuntimeException(env, 511 "nativeAttachImage for non-opaque image is not implement yet!!!"); 512 return -1; 513 } 514 515 if (!isFormatOpaque(ctx->getBufferFormat())) { 516 jniThrowException(env, "java/lang/IllegalStateException", 517 "Trying to attach an opaque image into a non-opaque ImageWriter"); 518 return -1; 519 } 520 521 // Image is guaranteed to be from ImageReader at this point, so it is safe to 522 // cast to BufferItem pointer. 523 BufferItem* opaqueBuffer = reinterpret_cast<BufferItem*>(nativeBuffer); 524 if (opaqueBuffer == NULL) { 525 jniThrowException(env, "java/lang/IllegalStateException", 526 "Image is not initialized or already closed"); 527 return -1; 528 } 529 530 // Step 1. Attach Image 531 res = surface->attachBuffer(opaqueBuffer->mGraphicBuffer.get()); 532 if (res != OK) { 533 ALOGE("Attach image failed: %s (%d)", strerror(-res), res); 534 switch (res) { 535 case NO_INIT: 536 jniThrowException(env, "java/lang/IllegalStateException", 537 "Surface has been abandoned"); 538 break; 539 default: 540 // TODO: handle other error cases here. 541 jniThrowRuntimeException(env, "nativeAttachImage failed!!!"); 542 } 543 return res; 544 } 545 sp < ANativeWindow > anw = surface; 546 547 // Step 2. Set timestamp and crop. Note that we do not need unlock the image because 548 // it was not locked. 549 ALOGV("timestamp to be queued: %" PRId64, timestampNs); 550 res = native_window_set_buffers_timestamp(anw.get(), timestampNs); 551 if (res != OK) { 552 jniThrowRuntimeException(env, "Set timestamp failed"); 553 return res; 554 } 555 556 android_native_rect_t cropRect; 557 cropRect.left = left; 558 cropRect.top = top; 559 cropRect.right = right; 560 cropRect.bottom = bottom; 561 res = native_window_set_crop(anw.get(), &cropRect); 562 if (res != OK) { 563 jniThrowRuntimeException(env, "Set crop rect failed"); 564 return res; 565 } 566 567 // Step 3. Queue Image. 568 res = anw->queueBuffer(anw.get(), opaqueBuffer->mGraphicBuffer.get(), /*fenceFd*/ 569 -1); 570 if (res != OK) { 571 ALOGE("%s: Queue buffer failed: %s (%d)", __FUNCTION__, strerror(-res), res); 572 switch (res) { 573 case NO_INIT: 574 jniThrowException(env, "java/lang/IllegalStateException", 575 "Surface has been abandoned"); 576 break; 577 default: 578 // TODO: handle other error cases here. 579 jniThrowRuntimeException(env, "Queue input buffer failed"); 580 } 581 return res; 582 } 583 584 // Do not set the image native context. Since it would overwrite the existing native context 585 // of the image that is from ImageReader, the subsequent image close will run into issues. 586 587 return res; 588 } 589 590 // --------------------------Image methods--------------------------------------- 591 592 static void Image_getNativeContext(JNIEnv* env, jobject thiz, 593 GraphicBuffer** buffer, int* fenceFd) { 594 ALOGV("%s", __FUNCTION__); 595 if (buffer != NULL) { 596 GraphicBuffer *gb = reinterpret_cast<GraphicBuffer *> 597 (env->GetLongField(thiz, gSurfaceImageClassInfo.mNativeBuffer)); 598 *buffer = gb; 599 } 600 601 if (fenceFd != NULL) { 602 *fenceFd = reinterpret_cast<jint>(env->GetIntField( 603 thiz, gSurfaceImageClassInfo.mNativeFenceFd)); 604 } 605 } 606 607 static void Image_setNativeContext(JNIEnv* env, jobject thiz, 608 sp<GraphicBuffer> buffer, int fenceFd) { 609 ALOGV("%s:", __FUNCTION__); 610 GraphicBuffer* p = NULL; 611 Image_getNativeContext(env, thiz, &p, /*fenceFd*/NULL); 612 if (buffer != 0) { 613 buffer->incStrong((void*)Image_setNativeContext); 614 } 615 if (p) { 616 p->decStrong((void*)Image_setNativeContext); 617 } 618 env->SetLongField(thiz, gSurfaceImageClassInfo.mNativeBuffer, 619 reinterpret_cast<jlong>(buffer.get())); 620 621 env->SetIntField(thiz, gSurfaceImageClassInfo.mNativeFenceFd, reinterpret_cast<jint>(fenceFd)); 622 } 623 624 static void Image_unlockIfLocked(JNIEnv* env, jobject thiz) { 625 ALOGV("%s", __FUNCTION__); 626 GraphicBuffer* buffer; 627 Image_getNativeContext(env, thiz, &buffer, NULL); 628 if (buffer == NULL) { 629 jniThrowException(env, "java/lang/IllegalStateException", 630 "Image is not initialized"); 631 return; 632 } 633 634 // Is locked? 635 bool isLocked = false; 636 jobject planes = NULL; 637 if (!isFormatOpaque(buffer->getPixelFormat())) { 638 planes = env->GetObjectField(thiz, gSurfaceImageClassInfo.mPlanes); 639 } 640 isLocked = (planes != NULL); 641 if (isLocked) { 642 // no need to use fence here, as we it will be consumed by either cancel or queue buffer. 643 status_t res = buffer->unlock(); 644 if (res != OK) { 645 jniThrowRuntimeException(env, "unlock buffer failed"); 646 } 647 ALOGV("Successfully unlocked the image"); 648 } 649 } 650 651 static jint Image_getWidth(JNIEnv* env, jobject thiz) { 652 ALOGV("%s", __FUNCTION__); 653 GraphicBuffer* buffer; 654 Image_getNativeContext(env, thiz, &buffer, NULL); 655 if (buffer == NULL) { 656 jniThrowException(env, "java/lang/IllegalStateException", 657 "Image is not initialized"); 658 return -1; 659 } 660 661 return buffer->getWidth(); 662 } 663 664 static jint Image_getHeight(JNIEnv* env, jobject thiz) { 665 ALOGV("%s", __FUNCTION__); 666 GraphicBuffer* buffer; 667 Image_getNativeContext(env, thiz, &buffer, NULL); 668 if (buffer == NULL) { 669 jniThrowException(env, "java/lang/IllegalStateException", 670 "Image is not initialized"); 671 return -1; 672 } 673 674 return buffer->getHeight(); 675 } 676 677 // Some formats like JPEG defined with different values between android.graphics.ImageFormat and 678 // graphics.h, need convert to the one defined in graphics.h here. 679 static int Image_getPixelFormat(JNIEnv* env, int format) { 680 int jpegFormat; 681 jfieldID fid; 682 683 ALOGV("%s: format = 0x%x", __FUNCTION__, format); 684 685 jclass imageFormatClazz = env->FindClass("android/graphics/ImageFormat"); 686 ALOG_ASSERT(imageFormatClazz != NULL); 687 688 fid = env->GetStaticFieldID(imageFormatClazz, "JPEG", "I"); 689 jpegFormat = env->GetStaticIntField(imageFormatClazz, fid); 690 691 // Translate the JPEG to BLOB for camera purpose. 692 if (format == jpegFormat) { 693 format = HAL_PIXEL_FORMAT_BLOB; 694 } 695 696 return format; 697 } 698 699 static jint Image_getFormat(JNIEnv* env, jobject thiz) { 700 ALOGV("%s", __FUNCTION__); 701 GraphicBuffer* buffer; 702 Image_getNativeContext(env, thiz, &buffer, NULL); 703 if (buffer == NULL) { 704 jniThrowException(env, "java/lang/IllegalStateException", 705 "Image is not initialized"); 706 return 0; 707 } 708 709 return Image_getPixelFormat(env, buffer->getPixelFormat()); 710 } 711 712 static void Image_setFenceFd(JNIEnv* env, jobject thiz, int fenceFd) { 713 ALOGV("%s:", __FUNCTION__); 714 env->SetIntField(thiz, gSurfaceImageClassInfo.mNativeFenceFd, reinterpret_cast<jint>(fenceFd)); 715 } 716 717 static void Image_getLockedImage(JNIEnv* env, jobject thiz, LockedImage *image) { 718 ALOGV("%s", __FUNCTION__); 719 GraphicBuffer* buffer; 720 int fenceFd = -1; 721 Image_getNativeContext(env, thiz, &buffer, &fenceFd); 722 if (buffer == NULL) { 723 jniThrowException(env, "java/lang/IllegalStateException", 724 "Image is not initialized"); 725 return; 726 } 727 728 void* pData = NULL; 729 android_ycbcr ycbcr = android_ycbcr(); 730 status_t res; 731 int format = Image_getFormat(env, thiz); 732 int flexFormat = format; 733 if (isPossiblyYUV(format)) { 734 // ImageWriter doesn't use crop by itself, app sets it, use the no crop version. 735 res = buffer->lockAsyncYCbCr(GRALLOC_USAGE_SW_WRITE_OFTEN, &ycbcr, fenceFd); 736 // Clear the fenceFd as it is already consumed by lock call. 737 Image_setFenceFd(env, thiz, /*fenceFd*/-1); 738 if (res != OK) { 739 jniThrowRuntimeException(env, "lockAsyncYCbCr failed for YUV buffer"); 740 return; 741 } 742 pData = ycbcr.y; 743 flexFormat = HAL_PIXEL_FORMAT_YCbCr_420_888; 744 } 745 746 // lockAsyncYCbCr for YUV is unsuccessful. 747 if (pData == NULL) { 748 res = buffer->lockAsync(GRALLOC_USAGE_SW_WRITE_OFTEN, &pData, fenceFd); 749 if (res != OK) { 750 jniThrowRuntimeException(env, "lockAsync failed"); 751 return; 752 } 753 } 754 755 image->data = reinterpret_cast<uint8_t*>(pData); 756 image->width = buffer->getWidth(); 757 image->height = buffer->getHeight(); 758 image->format = format; 759 image->flexFormat = flexFormat; 760 image->stride = (ycbcr.y != NULL) ? static_cast<uint32_t>(ycbcr.ystride) : buffer->getStride(); 761 762 image->dataCb = reinterpret_cast<uint8_t*>(ycbcr.cb); 763 image->dataCr = reinterpret_cast<uint8_t*>(ycbcr.cr); 764 image->chromaStride = static_cast<uint32_t>(ycbcr.cstride); 765 image->chromaStep = static_cast<uint32_t>(ycbcr.chroma_step); 766 ALOGV("Successfully locked the image"); 767 // crop, transform, scalingMode, timestamp, and frameNumber should be set by producer, 768 // and we don't set them here. 769 } 770 771 static bool usingRGBAToJpegOverride(int32_t bufferFormat, int32_t writerCtxFormat) { 772 return writerCtxFormat == HAL_PIXEL_FORMAT_BLOB && bufferFormat == HAL_PIXEL_FORMAT_RGBA_8888; 773 } 774 775 static int32_t applyFormatOverrides(int32_t bufferFormat, int32_t writerCtxFormat) 776 { 777 // Using HAL_PIXEL_FORMAT_RGBA_8888 gralloc buffers containing JPEGs to get around SW 778 // write limitations for some platforms (b/17379185). 779 if (usingRGBAToJpegOverride(bufferFormat, writerCtxFormat)) { 780 return HAL_PIXEL_FORMAT_BLOB; 781 } 782 return bufferFormat; 783 } 784 785 static uint32_t Image_getJpegSize(LockedImage* buffer, bool usingRGBAOverride) { 786 ALOGV("%s", __FUNCTION__); 787 ALOG_ASSERT(buffer != NULL, "Input buffer is NULL!!!"); 788 uint32_t size = 0; 789 uint32_t width = buffer->width; 790 uint8_t* jpegBuffer = buffer->data; 791 792 if (usingRGBAOverride) { 793 width = (buffer->width + buffer->stride * (buffer->height - 1)) * 4; 794 } 795 796 // First check for JPEG transport header at the end of the buffer 797 uint8_t* header = jpegBuffer + (width - sizeof(struct camera3_jpeg_blob)); 798 struct camera3_jpeg_blob *blob = (struct camera3_jpeg_blob*)(header); 799 if (blob->jpeg_blob_id == CAMERA3_JPEG_BLOB_ID) { 800 size = blob->jpeg_size; 801 ALOGV("%s: Jpeg size = %d", __FUNCTION__, size); 802 } 803 804 // failed to find size, default to whole buffer 805 if (size == 0) { 806 /* 807 * This is a problem because not including the JPEG header 808 * means that in certain rare situations a regular JPEG blob 809 * will be misidentified as having a header, in which case 810 * we will get a garbage size value. 811 */ 812 ALOGW("%s: No JPEG header detected, defaulting to size=width=%d", 813 __FUNCTION__, width); 814 size = width; 815 } 816 817 return size; 818 } 819 820 static void Image_getLockedImageInfo(JNIEnv* env, LockedImage* buffer, int idx, 821 int32_t writerFormat, uint8_t **base, uint32_t *size, int *pixelStride, int *rowStride) { 822 ALOGV("%s", __FUNCTION__); 823 ALOG_ASSERT(buffer != NULL, "Input buffer is NULL!!!"); 824 ALOG_ASSERT(base != NULL, "base is NULL!!!"); 825 ALOG_ASSERT(size != NULL, "size is NULL!!!"); 826 ALOG_ASSERT(pixelStride != NULL, "pixelStride is NULL!!!"); 827 ALOG_ASSERT(rowStride != NULL, "rowStride is NULL!!!"); 828 ALOG_ASSERT((idx < IMAGE_WRITER_MAX_NUM_PLANES) && (idx >= 0)); 829 830 ALOGV("%s: buffer: %p", __FUNCTION__, buffer); 831 832 uint32_t dataSize, ySize, cSize, cStride; 833 uint32_t pStride = 0, rStride = 0; 834 uint8_t *cb, *cr; 835 uint8_t *pData = NULL; 836 int bytesPerPixel = 0; 837 838 dataSize = ySize = cSize = cStride = 0; 839 int32_t fmt = buffer->flexFormat; 840 841 bool usingRGBAOverride = usingRGBAToJpegOverride(fmt, writerFormat); 842 fmt = applyFormatOverrides(fmt, writerFormat); 843 switch (fmt) { 844 case HAL_PIXEL_FORMAT_YCbCr_420_888: 845 pData = 846 (idx == 0) ? 847 buffer->data : 848 (idx == 1) ? 849 buffer->dataCb : 850 buffer->dataCr; 851 // only map until last pixel 852 if (idx == 0) { 853 pStride = 1; 854 rStride = buffer->stride; 855 dataSize = buffer->stride * (buffer->height - 1) + buffer->width; 856 } else { 857 pStride = buffer->chromaStep; 858 rStride = buffer->chromaStride; 859 dataSize = buffer->chromaStride * (buffer->height / 2 - 1) + 860 buffer->chromaStep * (buffer->width / 2 - 1) + 1; 861 } 862 break; 863 // NV21 864 case HAL_PIXEL_FORMAT_YCrCb_420_SP: 865 cr = buffer->data + (buffer->stride * buffer->height); 866 cb = cr + 1; 867 // only map until last pixel 868 ySize = buffer->width * (buffer->height - 1) + buffer->width; 869 cSize = buffer->width * (buffer->height / 2 - 1) + buffer->width - 1; 870 871 pData = 872 (idx == 0) ? 873 buffer->data : 874 (idx == 1) ? 875 cb: 876 cr; 877 878 dataSize = (idx == 0) ? ySize : cSize; 879 pStride = (idx == 0) ? 1 : 2; 880 rStride = buffer->width; 881 break; 882 case HAL_PIXEL_FORMAT_YV12: 883 // Y and C stride need to be 16 pixel aligned. 884 LOG_ALWAYS_FATAL_IF(buffer->stride % 16, 885 "Stride is not 16 pixel aligned %d", buffer->stride); 886 887 ySize = buffer->stride * buffer->height; 888 cStride = ALIGN(buffer->stride / 2, 16); 889 cr = buffer->data + ySize; 890 cSize = cStride * buffer->height / 2; 891 cb = cr + cSize; 892 893 pData = 894 (idx == 0) ? 895 buffer->data : 896 (idx == 1) ? 897 cb : 898 cr; 899 dataSize = (idx == 0) ? ySize : cSize; 900 pStride = 1; 901 rStride = (idx == 0) ? buffer->stride : ALIGN(buffer->stride / 2, 16); 902 break; 903 case HAL_PIXEL_FORMAT_Y8: 904 // Single plane, 8bpp. 905 ALOG_ASSERT(idx == 0, "Wrong index: %d", idx); 906 907 pData = buffer->data; 908 dataSize = buffer->stride * buffer->height; 909 pStride = 1; 910 rStride = buffer->stride; 911 break; 912 case HAL_PIXEL_FORMAT_Y16: 913 bytesPerPixel = 2; 914 // Single plane, 16bpp, strides are specified in pixels, not in bytes 915 ALOG_ASSERT(idx == 0, "Wrong index: %d", idx); 916 917 pData = buffer->data; 918 dataSize = buffer->stride * buffer->height * bytesPerPixel; 919 pStride = bytesPerPixel; 920 rStride = buffer->stride * 2; 921 break; 922 case HAL_PIXEL_FORMAT_BLOB: 923 // Used for JPEG data, height must be 1, width == size, single plane. 924 ALOG_ASSERT(idx == 0, "Wrong index: %d", idx); 925 ALOG_ASSERT(buffer->height == 1, "JPEG should has height value %d", buffer->height); 926 927 pData = buffer->data; 928 dataSize = Image_getJpegSize(buffer, usingRGBAOverride); 929 pStride = bytesPerPixel; 930 rowStride = 0; 931 break; 932 case HAL_PIXEL_FORMAT_RAW16: 933 // Single plane 16bpp bayer data. 934 bytesPerPixel = 2; 935 ALOG_ASSERT(idx == 0, "Wrong index: %d", idx); 936 pData = buffer->data; 937 dataSize = buffer->stride * buffer->height * bytesPerPixel; 938 pStride = bytesPerPixel; 939 rStride = buffer->stride * 2; 940 break; 941 case HAL_PIXEL_FORMAT_RAW10: 942 // Single plane 10bpp bayer data. 943 ALOG_ASSERT(idx == 0, "Wrong index: %d", idx); 944 LOG_ALWAYS_FATAL_IF(buffer->width % 4, 945 "Width is not multiple of 4 %d", buffer->width); 946 LOG_ALWAYS_FATAL_IF(buffer->height % 2, 947 "Height is not even %d", buffer->height); 948 LOG_ALWAYS_FATAL_IF(buffer->stride < (buffer->width * 10 / 8), 949 "stride (%d) should be at least %d", 950 buffer->stride, buffer->width * 10 / 8); 951 pData = buffer->data; 952 dataSize = buffer->stride * buffer->height; 953 pStride = 0; 954 rStride = buffer->stride; 955 break; 956 case HAL_PIXEL_FORMAT_RGBA_8888: 957 case HAL_PIXEL_FORMAT_RGBX_8888: 958 // Single plane, 32bpp. 959 bytesPerPixel = 4; 960 ALOG_ASSERT(idx == 0, "Wrong index: %d", idx); 961 pData = buffer->data; 962 dataSize = buffer->stride * buffer->height * bytesPerPixel; 963 pStride = bytesPerPixel; 964 rStride = buffer->stride * 4; 965 break; 966 case HAL_PIXEL_FORMAT_RGB_565: 967 // Single plane, 16bpp. 968 bytesPerPixel = 2; 969 ALOG_ASSERT(idx == 0, "Wrong index: %d", idx); 970 pData = buffer->data; 971 dataSize = buffer->stride * buffer->height * bytesPerPixel; 972 pStride = bytesPerPixel; 973 rStride = buffer->stride * 2; 974 break; 975 case HAL_PIXEL_FORMAT_RGB_888: 976 // Single plane, 24bpp. 977 bytesPerPixel = 3; 978 ALOG_ASSERT(idx == 0, "Wrong index: %d", idx); 979 pData = buffer->data; 980 dataSize = buffer->stride * buffer->height * bytesPerPixel; 981 pStride = bytesPerPixel; 982 rStride = buffer->stride * 3; 983 break; 984 default: 985 jniThrowExceptionFmt(env, "java/lang/UnsupportedOperationException", 986 "Pixel format: 0x%x is unsupported", fmt); 987 break; 988 } 989 990 *base = pData; 991 *size = dataSize; 992 *pixelStride = pStride; 993 *rowStride = rStride; 994 } 995 996 static jobjectArray Image_createSurfacePlanes(JNIEnv* env, jobject thiz, 997 int numPlanes, int writerFormat) { 998 ALOGV("%s: create SurfacePlane array with size %d", __FUNCTION__, numPlanes); 999 int rowStride, pixelStride; 1000 uint8_t *pData; 1001 uint32_t dataSize; 1002 jobject byteBuffer; 1003 1004 int format = Image_getFormat(env, thiz); 1005 if (isFormatOpaque(format) && numPlanes > 0) { 1006 String8 msg; 1007 msg.appendFormat("Format 0x%x is opaque, thus not writable, the number of planes (%d)" 1008 " must be 0", format, numPlanes); 1009 jniThrowException(env, "java/lang/IllegalArgumentException", msg.string()); 1010 return NULL; 1011 } 1012 1013 jobjectArray surfacePlanes = env->NewObjectArray(numPlanes, gSurfacePlaneClassInfo.clazz, 1014 /*initial_element*/NULL); 1015 if (surfacePlanes == NULL) { 1016 jniThrowRuntimeException(env, "Failed to create SurfacePlane arrays," 1017 " probably out of memory"); 1018 return NULL; 1019 } 1020 if (isFormatOpaque(format)) { 1021 return surfacePlanes; 1022 } 1023 1024 // Buildup buffer info: rowStride, pixelStride and byteBuffers. 1025 LockedImage lockedImg = LockedImage(); 1026 Image_getLockedImage(env, thiz, &lockedImg); 1027 1028 // Create all SurfacePlanes 1029 writerFormat = Image_getPixelFormat(env, writerFormat); 1030 for (int i = 0; i < numPlanes; i++) { 1031 Image_getLockedImageInfo(env, &lockedImg, i, writerFormat, 1032 &pData, &dataSize, &pixelStride, &rowStride); 1033 byteBuffer = env->NewDirectByteBuffer(pData, dataSize); 1034 if ((byteBuffer == NULL) && (env->ExceptionCheck() == false)) { 1035 jniThrowException(env, "java/lang/IllegalStateException", 1036 "Failed to allocate ByteBuffer"); 1037 return NULL; 1038 } 1039 1040 // Finally, create this SurfacePlane. 1041 jobject surfacePlane = env->NewObject(gSurfacePlaneClassInfo.clazz, 1042 gSurfacePlaneClassInfo.ctor, thiz, rowStride, pixelStride, byteBuffer); 1043 env->SetObjectArrayElement(surfacePlanes, i, surfacePlane); 1044 } 1045 1046 return surfacePlanes; 1047 } 1048 1049 // -------------------------------Private convenience methods-------------------- 1050 1051 static bool isFormatOpaque(int format) { 1052 // Only treat IMPLEMENTATION_DEFINED as an opaque format for now. 1053 return format == HAL_PIXEL_FORMAT_IMPLEMENTATION_DEFINED; 1054 } 1055 1056 static bool isPossiblyYUV(PixelFormat format) { 1057 switch (static_cast<int>(format)) { 1058 case HAL_PIXEL_FORMAT_RGBA_8888: 1059 case HAL_PIXEL_FORMAT_RGBX_8888: 1060 case HAL_PIXEL_FORMAT_RGB_888: 1061 case HAL_PIXEL_FORMAT_RGB_565: 1062 case HAL_PIXEL_FORMAT_BGRA_8888: 1063 case HAL_PIXEL_FORMAT_Y8: 1064 case HAL_PIXEL_FORMAT_Y16: 1065 case HAL_PIXEL_FORMAT_RAW16: 1066 case HAL_PIXEL_FORMAT_RAW10: 1067 case HAL_PIXEL_FORMAT_RAW_OPAQUE: 1068 case HAL_PIXEL_FORMAT_BLOB: 1069 case HAL_PIXEL_FORMAT_IMPLEMENTATION_DEFINED: 1070 return false; 1071 1072 case HAL_PIXEL_FORMAT_YV12: 1073 case HAL_PIXEL_FORMAT_YCbCr_420_888: 1074 case HAL_PIXEL_FORMAT_YCbCr_422_SP: 1075 case HAL_PIXEL_FORMAT_YCrCb_420_SP: 1076 case HAL_PIXEL_FORMAT_YCbCr_422_I: 1077 default: 1078 return true; 1079 } 1080 } 1081 1082 } // extern "C" 1083 1084 // ---------------------------------------------------------------------------- 1085 1086 static JNINativeMethod gImageWriterMethods[] = { 1087 {"nativeClassInit", "()V", (void*)ImageWriter_classInit }, 1088 {"nativeInit", "(Ljava/lang/Object;Landroid/view/Surface;I)J", 1089 (void*)ImageWriter_init }, 1090 {"nativeClose", "(J)V", (void*)ImageWriter_close }, 1091 {"nativeAttachAndQueueImage", "(JJIJIIII)I", (void*)ImageWriter_attachAndQueueImage }, 1092 {"nativeDequeueInputImage", "(JLandroid/media/Image;)V", (void*)ImageWriter_dequeueImage }, 1093 {"nativeQueueInputImage", "(JLandroid/media/Image;JIIII)V", (void*)ImageWriter_queueImage }, 1094 {"cancelImage", "(JLandroid/media/Image;)V", (void*)ImageWriter_cancelImage }, 1095 }; 1096 1097 static JNINativeMethod gImageMethods[] = { 1098 {"nativeCreatePlanes", "(II)[Landroid/media/ImageWriter$WriterSurfaceImage$SurfacePlane;", 1099 (void*)Image_createSurfacePlanes }, 1100 {"nativeGetWidth", "()I", (void*)Image_getWidth }, 1101 {"nativeGetHeight", "()I", (void*)Image_getHeight }, 1102 {"nativeGetFormat", "()I", (void*)Image_getFormat }, 1103 }; 1104 1105 int register_android_media_ImageWriter(JNIEnv *env) { 1106 1107 int ret1 = AndroidRuntime::registerNativeMethods(env, 1108 "android/media/ImageWriter", gImageWriterMethods, NELEM(gImageWriterMethods)); 1109 1110 int ret2 = AndroidRuntime::registerNativeMethods(env, 1111 "android/media/ImageWriter$WriterSurfaceImage", gImageMethods, NELEM(gImageMethods)); 1112 1113 return (ret1 || ret2); 1114 } 1115 1116
