1 /* 2 * Copyright 2015 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17 package android.media; 18 19 import android.graphics.ImageFormat; 20 import android.graphics.PixelFormat; 21 import android.graphics.Rect; 22 import android.hardware.camera2.utils.SurfaceUtils; 23 import android.hardware.HardwareBuffer; 24 import android.os.Handler; 25 import android.os.Looper; 26 import android.os.Message; 27 import android.util.Size; 28 import android.view.Surface; 29 30 import dalvik.system.VMRuntime; 31 32 import java.lang.ref.WeakReference; 33 import java.nio.ByteBuffer; 34 import java.nio.ByteOrder; 35 import java.nio.NioUtils; 36 import java.util.List; 37 import java.util.concurrent.CopyOnWriteArrayList; 38 39 /** 40 * <p> 41 * The ImageWriter class allows an application to produce Image data into a 42 * {@link android.view.Surface}, and have it be consumed by another component 43 * like {@link android.hardware.camera2.CameraDevice CameraDevice}. 44 * </p> 45 * <p> 46 * Several Android API classes can provide input {@link android.view.Surface 47 * Surface} objects for ImageWriter to produce data into, including 48 * {@link MediaCodec MediaCodec} (encoder), 49 * {@link android.hardware.camera2.CameraCaptureSession CameraCaptureSession} 50 * (reprocessing input), {@link ImageReader}, etc. 51 * </p> 52 * <p> 53 * The input Image data is encapsulated in {@link Image} objects. To produce 54 * Image data into a destination {@link android.view.Surface Surface}, the 55 * application can get an input Image via {@link #dequeueInputImage} then write 56 * Image data into it. Multiple such {@link Image} objects can be dequeued at 57 * the same time and queued back in any order, up to the number specified by the 58 * {@code maxImages} constructor parameter. 59 * </p> 60 * <p> 61 * If the application already has an Image from {@link ImageReader}, the 62 * application can directly queue this Image into the ImageWriter (via 63 * {@link #queueInputImage}), potentially with zero buffer copies. This 64 * even works if the image format of the ImageWriter is 65 * {@link ImageFormat#PRIVATE PRIVATE}, and prior to Android P is the only 66 * way to enqueue images into such an ImageWriter. Starting in Android P 67 * private images may also be accessed through their hardware buffers 68 * (when available) through the {@link Image#getHardwareBuffer()} method. 69 * Attempting to access the planes of a private image, will return an 70 * empty array. 71 * </p> 72 * <p> 73 * Once new input Images are queued into an ImageWriter, it's up to the 74 * downstream components (e.g. {@link ImageReader} or 75 * {@link android.hardware.camera2.CameraDevice}) to consume the Images. If the 76 * downstream components cannot consume the Images at least as fast as the 77 * ImageWriter production rate, the {@link #dequeueInputImage} call will 78 * eventually block and the application will have to drop input frames. 79 * </p> 80 * <p> 81 * If the consumer component that provided the input {@link android.view.Surface Surface} 82 * abandons the {@link android.view.Surface Surface}, {@link #queueInputImage queueing} 83 * or {@link #dequeueInputImage dequeueing} an {@link Image} will throw an 84 * {@link IllegalStateException}. 85 * </p> 86 */ 87 public class ImageWriter implements AutoCloseable { 88 private final Object mListenerLock = new Object(); 89 private OnImageReleasedListener mListener; 90 private ListenerHandler mListenerHandler; 91 private long mNativeContext; 92 93 // Field below is used by native code, do not access or modify. 94 private int mWriterFormat; 95 96 private final int mMaxImages; 97 // Keep track of the currently dequeued Image. This need to be thread safe as the images 98 // could be closed by different threads (e.g., application thread and GC thread). 99 private List<Image> mDequeuedImages = new CopyOnWriteArrayList<>(); 100 private int mEstimatedNativeAllocBytes; 101 102 /** 103 * <p> 104 * Create a new ImageWriter. 105 * </p> 106 * <p> 107 * The {@code maxImages} parameter determines the maximum number of 108 * {@link Image} objects that can be be dequeued from the 109 * {@code ImageWriter} simultaneously. Requesting more buffers will use up 110 * more memory, so it is important to use only the minimum number necessary. 111 * </p> 112 * <p> 113 * The input Image size and format depend on the Surface that is provided by 114 * the downstream consumer end-point. 115 * </p> 116 * 117 * @param surface The destination Surface this writer produces Image data 118 * into. 119 * @param maxImages The maximum number of Images the user will want to 120 * access simultaneously for producing Image data. This should be 121 * as small as possible to limit memory use. Once maxImages 122 * Images are dequeued by the user, one of them has to be queued 123 * back before a new Image can be dequeued for access via 124 * {@link #dequeueInputImage()}. 125 * @return a new ImageWriter instance. 126 */ 127 public static ImageWriter newInstance(Surface surface, int maxImages) { 128 return new ImageWriter(surface, maxImages, ImageFormat.UNKNOWN); 129 } 130 131 /** 132 * <p> 133 * Create a new ImageWriter with given number of max Images and format. 134 * </p> 135 * <p> 136 * The {@code maxImages} parameter determines the maximum number of 137 * {@link Image} objects that can be be dequeued from the 138 * {@code ImageWriter} simultaneously. Requesting more buffers will use up 139 * more memory, so it is important to use only the minimum number necessary. 140 * </p> 141 * <p> 142 * The format specifies the image format of this ImageWriter. The format 143 * from the {@code surface} will be overridden with this format. For example, 144 * if the surface is obtained from a {@link android.graphics.SurfaceTexture}, the default 145 * format may be {@link PixelFormat#RGBA_8888}. If the application creates an ImageWriter 146 * with this surface and {@link ImageFormat#PRIVATE}, this ImageWriter will be able to operate 147 * with {@link ImageFormat#PRIVATE} Images. 148 * </p> 149 * <p> 150 * Note that the consumer end-point may or may not be able to support Images with different 151 * format, for such case, the application should only use this method if the consumer is able 152 * to consume such images. 153 * </p> 154 * <p> 155 * The input Image size depends on the Surface that is provided by 156 * the downstream consumer end-point. 157 * </p> 158 * 159 * @param surface The destination Surface this writer produces Image data 160 * into. 161 * @param maxImages The maximum number of Images the user will want to 162 * access simultaneously for producing Image data. This should be 163 * as small as possible to limit memory use. Once maxImages 164 * Images are dequeued by the user, one of them has to be queued 165 * back before a new Image can be dequeued for access via 166 * {@link #dequeueInputImage()}. 167 * @param format The format of this ImageWriter. It can be any valid format specified by 168 * {@link ImageFormat} or {@link PixelFormat}. 169 * 170 * @return a new ImageWriter instance. 171 * @hide 172 */ 173 public static ImageWriter newInstance(Surface surface, int maxImages, int format) { 174 if (!ImageFormat.isPublicFormat(format) && !PixelFormat.isPublicFormat(format)) { 175 throw new IllegalArgumentException("Invalid format is specified: " + format); 176 } 177 return new ImageWriter(surface, maxImages, format); 178 } 179 180 /** 181 * @hide 182 */ 183 protected ImageWriter(Surface surface, int maxImages, int format) { 184 if (surface == null || maxImages < 1) { 185 throw new IllegalArgumentException("Illegal input argument: surface " + surface 186 + ", maxImages: " + maxImages); 187 } 188 189 mMaxImages = maxImages; 190 191 if (format == ImageFormat.UNKNOWN) { 192 format = SurfaceUtils.getSurfaceFormat(surface); 193 } 194 // Note that the underlying BufferQueue is working in synchronous mode 195 // to avoid dropping any buffers. 196 mNativeContext = nativeInit(new WeakReference<>(this), surface, maxImages, format); 197 198 // Estimate the native buffer allocation size and register it so it gets accounted for 199 // during GC. Note that this doesn't include the buffers required by the buffer queue 200 // itself and the buffers requested by the producer. 201 // Only include memory for 1 buffer, since actually accounting for the memory used is 202 // complex, and 1 buffer is enough for the VM to treat the ImageWriter as being of some 203 // size. 204 Size surfSize = SurfaceUtils.getSurfaceSize(surface); 205 mEstimatedNativeAllocBytes = 206 ImageUtils.getEstimatedNativeAllocBytes(surfSize.getWidth(),surfSize.getHeight(), 207 format, /*buffer count*/ 1); 208 VMRuntime.getRuntime().registerNativeAllocation(mEstimatedNativeAllocBytes); 209 } 210 211 /** 212 * <p> 213 * Maximum number of Images that can be dequeued from the ImageWriter 214 * simultaneously (for example, with {@link #dequeueInputImage()}). 215 * </p> 216 * <p> 217 * An Image is considered dequeued after it's returned by 218 * {@link #dequeueInputImage()} from ImageWriter, and until the Image is 219 * sent back to ImageWriter via {@link #queueInputImage}, or 220 * {@link Image#close()}. 221 * </p> 222 * <p> 223 * Attempting to dequeue more than {@code maxImages} concurrently will 224 * result in the {@link #dequeueInputImage()} function throwing an 225 * {@link IllegalStateException}. 226 * </p> 227 * 228 * @return Maximum number of Images that can be dequeued from this 229 * ImageWriter. 230 * @see #dequeueInputImage 231 * @see #queueInputImage 232 * @see Image#close 233 */ 234 public int getMaxImages() { 235 return mMaxImages; 236 } 237 238 /** 239 * <p> 240 * Dequeue the next available input Image for the application to produce 241 * data into. 242 * </p> 243 * <p> 244 * This method requests a new input Image from ImageWriter. The application 245 * owns this Image after this call. Once the application fills the Image 246 * data, it is expected to return this Image back to ImageWriter for 247 * downstream consumer components (e.g. 248 * {@link android.hardware.camera2.CameraDevice}) to consume. The Image can 249 * be returned to ImageWriter via {@link #queueInputImage} or 250 * {@link Image#close()}. 251 * </p> 252 * <p> 253 * This call will block if all available input images have been queued by 254 * the application and the downstream consumer has not yet consumed any. 255 * When an Image is consumed by the downstream consumer and released, an 256 * {@link OnImageReleasedListener#onImageReleased} callback will be fired, 257 * which indicates that there is one input Image available. For non- 258 * {@link ImageFormat#PRIVATE PRIVATE} formats ( 259 * {@link ImageWriter#getFormat()} != {@link ImageFormat#PRIVATE}), it is 260 * recommended to dequeue the next Image only after this callback is fired, 261 * in the steady state. 262 * </p> 263 * <p> 264 * If the format of ImageWriter is {@link ImageFormat#PRIVATE PRIVATE} ( 265 * {@link ImageWriter#getFormat()} == {@link ImageFormat#PRIVATE}), the 266 * image buffer is accessible to the application only through the hardware 267 * buffer obtained through {@link Image#getHardwareBuffer()}. (On Android 268 * versions prior to P, dequeueing private buffers will cause an 269 * {@link IllegalStateException} to be thrown). Alternatively, 270 * the application can acquire images from some other component (e.g. an 271 * {@link ImageReader}), and queue them directly to this ImageWriter via the 272 * {@link ImageWriter#queueInputImage queueInputImage()} method. 273 * </p> 274 * 275 * @return The next available input Image from this ImageWriter. 276 * @throws IllegalStateException if {@code maxImages} Images are currently 277 * dequeued, or the input {@link android.view.Surface Surface} 278 * has been abandoned by the consumer component that provided 279 * the {@link android.view.Surface Surface}. Prior to Android 280 * P, throws if the ImageWriter format is 281 * {@link ImageFormat#PRIVATE PRIVATE}. 282 * @see #queueInputImage 283 * @see Image#close 284 */ 285 public Image dequeueInputImage() { 286 if (mDequeuedImages.size() >= mMaxImages) { 287 throw new IllegalStateException("Already dequeued max number of Images " + mMaxImages); 288 } 289 WriterSurfaceImage newImage = new WriterSurfaceImage(this); 290 nativeDequeueInputImage(mNativeContext, newImage); 291 mDequeuedImages.add(newImage); 292 newImage.mIsImageValid = true; 293 return newImage; 294 } 295 296 /** 297 * <p> 298 * Queue an input {@link Image} back to ImageWriter for the downstream 299 * consumer to access. 300 * </p> 301 * <p> 302 * The input {@link Image} could be from ImageReader (acquired via 303 * {@link ImageReader#acquireNextImage} or 304 * {@link ImageReader#acquireLatestImage}), or from this ImageWriter 305 * (acquired via {@link #dequeueInputImage}). In the former case, the Image 306 * data will be moved to this ImageWriter. Note that the Image properties 307 * (size, format, strides, etc.) must be the same as the properties of the 308 * images dequeued from this ImageWriter, or this method will throw an 309 * {@link IllegalArgumentException}. In the latter case, the application has 310 * filled the input image with data. This method then passes the filled 311 * buffer to the downstream consumer. In both cases, it's up to the caller 312 * to ensure that the Image timestamp (in nanoseconds) is correctly set, as 313 * the downstream component may want to use it to indicate the Image data 314 * capture time. 315 * </p> 316 * <p> 317 * After this method is called and the downstream consumer consumes and 318 * releases the Image, an {@link OnImageReleasedListener#onImageReleased} 319 * callback will fire. The application can use this callback to avoid 320 * sending Images faster than the downstream consumer processing rate in 321 * steady state. 322 * </p> 323 * <p> 324 * Passing in an Image from some other component (e.g. an 325 * {@link ImageReader}) requires a free input Image from this ImageWriter as 326 * the destination. In this case, this call will block, as 327 * {@link #dequeueInputImage} does, if there are no free Images available. 328 * To avoid blocking, the application should ensure that there is at least 329 * one free Image available in this ImageWriter before calling this method. 330 * </p> 331 * <p> 332 * After this call, the input Image is no longer valid for further access, 333 * as if the Image is {@link Image#close closed}. Attempting to access the 334 * {@link ByteBuffer ByteBuffers} returned by an earlier 335 * {@link Image.Plane#getBuffer Plane#getBuffer} call will result in an 336 * {@link IllegalStateException}. 337 * </p> 338 * 339 * @param image The Image to be queued back to ImageWriter for future 340 * consumption. 341 * @throws IllegalStateException if the image was already queued previously, 342 * or the image was aborted previously, or the input 343 * {@link android.view.Surface Surface} has been abandoned by the 344 * consumer component that provided the 345 * {@link android.view.Surface Surface}. 346 * @see #dequeueInputImage() 347 */ 348 public void queueInputImage(Image image) { 349 if (image == null) { 350 throw new IllegalArgumentException("image shouldn't be null"); 351 } 352 boolean ownedByMe = isImageOwnedByMe(image); 353 if (ownedByMe && !(((WriterSurfaceImage) image).mIsImageValid)) { 354 throw new IllegalStateException("Image from ImageWriter is invalid"); 355 } 356 357 // For images from other components, need to detach first, then attach. 358 if (!ownedByMe) { 359 if (!(image.getOwner() instanceof ImageReader)) { 360 throw new IllegalArgumentException("Only images from ImageReader can be queued to" 361 + " ImageWriter, other image source is not supported yet!"); 362 } 363 364 ImageReader prevOwner = (ImageReader) image.getOwner(); 365 366 prevOwner.detachImage(image); 367 attachAndQueueInputImage(image); 368 // This clears the native reference held by the original owner. 369 // When this Image is detached later by this ImageWriter, the 370 // native memory won't be leaked. 371 image.close(); 372 return; 373 } 374 375 Rect crop = image.getCropRect(); 376 nativeQueueInputImage(mNativeContext, image, image.getTimestamp(), crop.left, crop.top, 377 crop.right, crop.bottom, image.getTransform(), image.getScalingMode()); 378 379 /** 380 * Only remove and cleanup the Images that are owned by this 381 * ImageWriter. Images detached from other owners are only temporarily 382 * owned by this ImageWriter and will be detached immediately after they 383 * are released by downstream consumers, so there is no need to keep 384 * track of them in mDequeuedImages. 385 */ 386 if (ownedByMe) { 387 mDequeuedImages.remove(image); 388 // Do not call close here, as close is essentially cancel image. 389 WriterSurfaceImage wi = (WriterSurfaceImage) image; 390 wi.clearSurfacePlanes(); 391 wi.mIsImageValid = false; 392 } 393 } 394 395 /** 396 * Get the ImageWriter format. 397 * <p> 398 * This format may be different than the Image format returned by 399 * {@link Image#getFormat()}. However, if the ImageWriter format is 400 * {@link ImageFormat#PRIVATE PRIVATE}, calling {@link #dequeueInputImage()} 401 * will result in an {@link IllegalStateException}. 402 * </p> 403 * 404 * @return The ImageWriter format. 405 */ 406 public int getFormat() { 407 return mWriterFormat; 408 } 409 410 /** 411 * ImageWriter callback interface, used to to asynchronously notify the 412 * application of various ImageWriter events. 413 */ 414 public interface OnImageReleasedListener { 415 /** 416 * <p> 417 * Callback that is called when an input Image is released back to 418 * ImageWriter after the data consumption. 419 * </p> 420 * <p> 421 * The client can use this callback to be notified that an input Image 422 * has been consumed and released by the downstream consumer. More 423 * specifically, this callback will be fired for below cases: 424 * <li>The application dequeues an input Image via the 425 * {@link ImageWriter#dequeueInputImage dequeueInputImage()} method, 426 * uses it, and then queues it back to this ImageWriter via the 427 * {@link ImageWriter#queueInputImage queueInputImage()} method. After 428 * the downstream consumer uses and releases this image to this 429 * ImageWriter, this callback will be fired. This image will be 430 * available to be dequeued after this callback.</li> 431 * <li>The application obtains an Image from some other component (e.g. 432 * an {@link ImageReader}), uses it, and then queues it to this 433 * ImageWriter via {@link ImageWriter#queueInputImage queueInputImage()}. 434 * After the downstream consumer uses and releases this image to this 435 * ImageWriter, this callback will be fired.</li> 436 * </p> 437 * 438 * @param writer the ImageWriter the callback is associated with. 439 * @see ImageWriter 440 * @see Image 441 */ 442 void onImageReleased(ImageWriter writer); 443 } 444 445 /** 446 * Register a listener to be invoked when an input Image is returned to the 447 * ImageWriter. 448 * 449 * @param listener The listener that will be run. 450 * @param handler The handler on which the listener should be invoked, or 451 * null if the listener should be invoked on the calling thread's 452 * looper. 453 * @throws IllegalArgumentException If no handler specified and the calling 454 * thread has no looper. 455 */ 456 public void setOnImageReleasedListener(OnImageReleasedListener listener, Handler handler) { 457 synchronized (mListenerLock) { 458 if (listener != null) { 459 Looper looper = handler != null ? handler.getLooper() : Looper.myLooper(); 460 if (looper == null) { 461 throw new IllegalArgumentException( 462 "handler is null but the current thread is not a looper"); 463 } 464 if (mListenerHandler == null || mListenerHandler.getLooper() != looper) { 465 mListenerHandler = new ListenerHandler(looper); 466 } 467 mListener = listener; 468 } else { 469 mListener = null; 470 mListenerHandler = null; 471 } 472 } 473 } 474 475 /** 476 * Free up all the resources associated with this ImageWriter. 477 * <p> 478 * After calling this method, this ImageWriter cannot be used. Calling any 479 * methods on this ImageWriter and Images previously provided by 480 * {@link #dequeueInputImage()} will result in an 481 * {@link IllegalStateException}, and attempting to write into 482 * {@link ByteBuffer ByteBuffers} returned by an earlier 483 * {@link Image.Plane#getBuffer Plane#getBuffer} call will have undefined 484 * behavior. 485 * </p> 486 */ 487 @Override 488 public void close() { 489 setOnImageReleasedListener(null, null); 490 for (Image image : mDequeuedImages) { 491 image.close(); 492 } 493 mDequeuedImages.clear(); 494 nativeClose(mNativeContext); 495 mNativeContext = 0; 496 497 if (mEstimatedNativeAllocBytes > 0) { 498 VMRuntime.getRuntime().registerNativeFree(mEstimatedNativeAllocBytes); 499 mEstimatedNativeAllocBytes = 0; 500 } 501 } 502 503 @Override 504 protected void finalize() throws Throwable { 505 try { 506 close(); 507 } finally { 508 super.finalize(); 509 } 510 } 511 512 /** 513 * <p> 514 * Attach and queue input Image to this ImageWriter. 515 * </p> 516 * <p> 517 * When the format of an Image is {@link ImageFormat#PRIVATE PRIVATE}, or 518 * the source Image is so large that copying its data is too expensive, this 519 * method can be used to migrate the source Image into ImageWriter without a 520 * data copy, and then queue it to this ImageWriter. The source Image must 521 * be detached from its previous owner already, or this call will throw an 522 * {@link IllegalStateException}. 523 * </p> 524 * <p> 525 * After this call, the ImageWriter takes ownership of this Image. This 526 * ownership will automatically be removed from this writer after the 527 * consumer releases this Image, that is, after 528 * {@link OnImageReleasedListener#onImageReleased}. The caller is responsible for 529 * closing this Image through {@link Image#close()} to free up the resources 530 * held by this Image. 531 * </p> 532 * 533 * @param image The source Image to be attached and queued into this 534 * ImageWriter for downstream consumer to use. 535 * @throws IllegalStateException if the Image is not detached from its 536 * previous owner, or the Image is already attached to this 537 * ImageWriter, or the source Image is invalid. 538 */ 539 private void attachAndQueueInputImage(Image image) { 540 if (image == null) { 541 throw new IllegalArgumentException("image shouldn't be null"); 542 } 543 if (isImageOwnedByMe(image)) { 544 throw new IllegalArgumentException( 545 "Can not attach an image that is owned ImageWriter already"); 546 } 547 /** 548 * Throw ISE if the image is not attachable, which means that it is 549 * either owned by other entity now, or completely non-attachable (some 550 * stand-alone images are not backed by native gralloc buffer, thus not 551 * attachable). 552 */ 553 if (!image.isAttachable()) { 554 throw new IllegalStateException("Image was not detached from last owner, or image " 555 + " is not detachable"); 556 } 557 558 // TODO: what if attach failed, throw RTE or detach a slot then attach? 559 // need do some cleanup to make sure no orphaned 560 // buffer caused leak. 561 Rect crop = image.getCropRect(); 562 nativeAttachAndQueueImage(mNativeContext, image.getNativeContext(), image.getFormat(), 563 image.getTimestamp(), crop.left, crop.top, crop.right, crop.bottom, 564 image.getTransform(), image.getScalingMode()); 565 } 566 567 /** 568 * This custom handler runs asynchronously so callbacks don't get queued 569 * behind UI messages. 570 */ 571 private final class ListenerHandler extends Handler { 572 public ListenerHandler(Looper looper) { 573 super(looper, null, true /* async */); 574 } 575 576 @Override 577 public void handleMessage(Message msg) { 578 OnImageReleasedListener listener; 579 synchronized (mListenerLock) { 580 listener = mListener; 581 } 582 if (listener != null) { 583 listener.onImageReleased(ImageWriter.this); 584 } 585 } 586 } 587 588 /** 589 * Called from Native code when an Event happens. This may be called from an 590 * arbitrary Binder thread, so access to the ImageWriter must be 591 * synchronized appropriately. 592 */ 593 private static void postEventFromNative(Object selfRef) { 594 @SuppressWarnings("unchecked") 595 WeakReference<ImageWriter> weakSelf = (WeakReference<ImageWriter>) selfRef; 596 final ImageWriter iw = weakSelf.get(); 597 if (iw == null) { 598 return; 599 } 600 601 final Handler handler; 602 synchronized (iw.mListenerLock) { 603 handler = iw.mListenerHandler; 604 } 605 if (handler != null) { 606 handler.sendEmptyMessage(0); 607 } 608 } 609 610 /** 611 * <p> 612 * Abort the Images that were dequeued from this ImageWriter, and return 613 * them to this writer for reuse. 614 * </p> 615 * <p> 616 * This method is used for the cases where the application dequeued the 617 * Image, may have filled the data, but does not want the downstream 618 * component to consume it. The Image will be returned to this ImageWriter 619 * for reuse after this call, and the ImageWriter will immediately have an 620 * Image available to be dequeued. This aborted Image will be invisible to 621 * the downstream consumer, as if nothing happened. 622 * </p> 623 * 624 * @param image The Image to be aborted. 625 * @see #dequeueInputImage() 626 * @see Image#close() 627 */ 628 private void abortImage(Image image) { 629 if (image == null) { 630 throw new IllegalArgumentException("image shouldn't be null"); 631 } 632 633 if (!mDequeuedImages.contains(image)) { 634 throw new IllegalStateException("It is illegal to abort some image that is not" 635 + " dequeued yet"); 636 } 637 638 WriterSurfaceImage wi = (WriterSurfaceImage) image; 639 if (!wi.mIsImageValid) { 640 return; 641 } 642 643 /** 644 * We only need abort Images that are owned and dequeued by ImageWriter. 645 * For attached Images, no need to abort, as there are only two cases: 646 * attached + queued successfully, and attach failed. Neither of the 647 * cases need abort. 648 */ 649 cancelImage(mNativeContext, image); 650 mDequeuedImages.remove(image); 651 wi.clearSurfacePlanes(); 652 wi.mIsImageValid = false; 653 } 654 655 private boolean isImageOwnedByMe(Image image) { 656 if (!(image instanceof WriterSurfaceImage)) { 657 return false; 658 } 659 WriterSurfaceImage wi = (WriterSurfaceImage) image; 660 if (wi.getOwner() != this) { 661 return false; 662 } 663 664 return true; 665 } 666 667 private static class WriterSurfaceImage extends android.media.Image { 668 private ImageWriter mOwner; 669 // This field is used by native code, do not access or modify. 670 private long mNativeBuffer; 671 private int mNativeFenceFd = -1; 672 private SurfacePlane[] mPlanes; 673 private int mHeight = -1; 674 private int mWidth = -1; 675 private int mFormat = -1; 676 // When this default timestamp is used, timestamp for the input Image 677 // will be generated automatically when queueInputBuffer is called. 678 private final long DEFAULT_TIMESTAMP = Long.MIN_VALUE; 679 private long mTimestamp = DEFAULT_TIMESTAMP; 680 681 private int mTransform = 0; //Default no transform 682 private int mScalingMode = 0; //Default frozen scaling mode 683 684 public WriterSurfaceImage(ImageWriter writer) { 685 mOwner = writer; 686 } 687 688 @Override 689 public int getFormat() { 690 throwISEIfImageIsInvalid(); 691 692 if (mFormat == -1) { 693 mFormat = nativeGetFormat(); 694 } 695 return mFormat; 696 } 697 698 @Override 699 public int getWidth() { 700 throwISEIfImageIsInvalid(); 701 702 if (mWidth == -1) { 703 mWidth = nativeGetWidth(); 704 } 705 706 return mWidth; 707 } 708 709 @Override 710 public int getHeight() { 711 throwISEIfImageIsInvalid(); 712 713 if (mHeight == -1) { 714 mHeight = nativeGetHeight(); 715 } 716 717 return mHeight; 718 } 719 720 @Override 721 public int getTransform() { 722 throwISEIfImageIsInvalid(); 723 724 return mTransform; 725 } 726 727 @Override 728 public int getScalingMode() { 729 throwISEIfImageIsInvalid(); 730 731 return mScalingMode; 732 } 733 734 @Override 735 public long getTimestamp() { 736 throwISEIfImageIsInvalid(); 737 738 return mTimestamp; 739 } 740 741 @Override 742 public void setTimestamp(long timestamp) { 743 throwISEIfImageIsInvalid(); 744 745 mTimestamp = timestamp; 746 } 747 748 @Override 749 public HardwareBuffer getHardwareBuffer() { 750 throwISEIfImageIsInvalid(); 751 752 return nativeGetHardwareBuffer(); 753 } 754 755 @Override 756 public Plane[] getPlanes() { 757 throwISEIfImageIsInvalid(); 758 759 if (mPlanes == null) { 760 int numPlanes = ImageUtils.getNumPlanesForFormat(getFormat()); 761 mPlanes = nativeCreatePlanes(numPlanes, getOwner().getFormat()); 762 } 763 764 return mPlanes.clone(); 765 } 766 767 @Override 768 boolean isAttachable() { 769 throwISEIfImageIsInvalid(); 770 // Don't allow Image to be detached from ImageWriter for now, as no 771 // detach API is exposed. 772 return false; 773 } 774 775 @Override 776 ImageWriter getOwner() { 777 throwISEIfImageIsInvalid(); 778 779 return mOwner; 780 } 781 782 @Override 783 long getNativeContext() { 784 throwISEIfImageIsInvalid(); 785 786 return mNativeBuffer; 787 } 788 789 @Override 790 public void close() { 791 if (mIsImageValid) { 792 getOwner().abortImage(this); 793 } 794 } 795 796 @Override 797 protected final void finalize() throws Throwable { 798 try { 799 close(); 800 } finally { 801 super.finalize(); 802 } 803 } 804 805 private void clearSurfacePlanes() { 806 if (mIsImageValid && mPlanes != null) { 807 for (int i = 0; i < mPlanes.length; i++) { 808 if (mPlanes[i] != null) { 809 mPlanes[i].clearBuffer(); 810 mPlanes[i] = null; 811 } 812 } 813 } 814 } 815 816 private class SurfacePlane extends android.media.Image.Plane { 817 private ByteBuffer mBuffer; 818 final private int mPixelStride; 819 final private int mRowStride; 820 821 // SurfacePlane instance is created by native code when SurfaceImage#getPlanes() is 822 // called 823 private SurfacePlane(int rowStride, int pixelStride, ByteBuffer buffer) { 824 mRowStride = rowStride; 825 mPixelStride = pixelStride; 826 mBuffer = buffer; 827 /** 828 * Set the byteBuffer order according to host endianness (native 829 * order), otherwise, the byteBuffer order defaults to 830 * ByteOrder.BIG_ENDIAN. 831 */ 832 mBuffer.order(ByteOrder.nativeOrder()); 833 } 834 835 @Override 836 public int getRowStride() { 837 throwISEIfImageIsInvalid(); 838 return mRowStride; 839 } 840 841 @Override 842 public int getPixelStride() { 843 throwISEIfImageIsInvalid(); 844 return mPixelStride; 845 } 846 847 @Override 848 public ByteBuffer getBuffer() { 849 throwISEIfImageIsInvalid(); 850 return mBuffer; 851 } 852 853 private void clearBuffer() { 854 // Need null check first, as the getBuffer() may not be called 855 // before an Image is closed. 856 if (mBuffer == null) { 857 return; 858 } 859 860 if (mBuffer.isDirect()) { 861 NioUtils.freeDirectBuffer(mBuffer); 862 } 863 mBuffer = null; 864 } 865 866 } 867 868 // Create the SurfacePlane object and fill the information 869 private synchronized native SurfacePlane[] nativeCreatePlanes(int numPlanes, int writerFmt); 870 871 private synchronized native int nativeGetWidth(); 872 873 private synchronized native int nativeGetHeight(); 874 875 private synchronized native int nativeGetFormat(); 876 877 private synchronized native HardwareBuffer nativeGetHardwareBuffer(); 878 } 879 880 // Native implemented ImageWriter methods. 881 private synchronized native long nativeInit(Object weakSelf, Surface surface, int maxImgs, 882 int format); 883 884 private synchronized native void nativeClose(long nativeCtx); 885 886 private synchronized native void nativeDequeueInputImage(long nativeCtx, Image wi); 887 888 private synchronized native void nativeQueueInputImage(long nativeCtx, Image image, 889 long timestampNs, int left, int top, int right, int bottom, int transform, 890 int scalingMode); 891 892 private synchronized native int nativeAttachAndQueueImage(long nativeCtx, 893 long imageNativeBuffer, int imageFormat, long timestampNs, int left, 894 int top, int right, int bottom, int transform, int scalingMode); 895 896 private synchronized native void cancelImage(long nativeCtx, Image image); 897 898 /** 899 * We use a class initializer to allow the native code to cache some field 900 * offsets. 901 */ 902 private static native void nativeClassInit(); 903 904 static { 905 System.loadLibrary("media_jni"); 906 nativeClassInit(); 907 } 908 } 909