1 /* 2 * Copyright (C) 2014 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17 package android.hardware.camera2.legacy; 18 19 import android.graphics.SurfaceTexture; 20 import android.hardware.Camera; 21 import android.hardware.camera2.CameraCharacteristics; 22 import android.hardware.camera2.CaptureRequest; 23 import android.hardware.camera2.impl.CameraDeviceImpl; 24 import android.hardware.camera2.utils.SubmitInfo; 25 import android.hardware.camera2.utils.SizeAreaComparator; 26 import android.hardware.camera2.impl.CameraMetadataNative; 27 import android.os.ConditionVariable; 28 import android.os.Handler; 29 import android.os.Message; 30 import android.os.SystemClock; 31 import android.util.Log; 32 import android.util.MutableLong; 33 import android.util.Pair; 34 import android.util.Size; 35 import android.view.Surface; 36 37 import java.io.IOException; 38 import java.util.ArrayList; 39 import java.util.Collection; 40 import java.util.Collections; 41 import java.util.Iterator; 42 import java.util.List; 43 import java.util.concurrent.TimeUnit; 44 import java.util.concurrent.atomic.AtomicBoolean; 45 46 import static com.android.internal.util.Preconditions.*; 47 48 /** 49 * This class executes requests to the {@link Camera}. 50 * 51 * <p> 52 * The main components of this class are: 53 * - A message queue of requests to the {@link Camera}. 54 * - A thread that consumes requests to the {@link Camera} and executes them. 55 * - A {@link GLThreadManager} that draws to the configured output {@link Surface}s. 56 * - An {@link CameraDeviceState} state machine that manages the callbacks for various operations. 57 * </p> 58 */ 59 @SuppressWarnings("deprecation") 60 public class RequestThreadManager { 61 private final String TAG; 62 private final int mCameraId; 63 private final RequestHandlerThread mRequestThread; 64 65 private static final boolean DEBUG = false; 66 // For slightly more spammy messages that will get repeated every frame 67 private static final boolean VERBOSE = false; 68 private Camera mCamera; 69 private final CameraCharacteristics mCharacteristics; 70 71 private final CameraDeviceState mDeviceState; 72 private final CaptureCollector mCaptureCollector; 73 private final LegacyFocusStateMapper mFocusStateMapper; 74 private final LegacyFaceDetectMapper mFaceDetectMapper; 75 76 private static final int MSG_CONFIGURE_OUTPUTS = 1; 77 private static final int MSG_SUBMIT_CAPTURE_REQUEST = 2; 78 private static final int MSG_CLEANUP = 3; 79 80 private static final int MAX_IN_FLIGHT_REQUESTS = 2; 81 82 private static final int PREVIEW_FRAME_TIMEOUT = 1000; // ms 83 private static final int JPEG_FRAME_TIMEOUT = 4000; // ms (same as CTS for API2) 84 private static final int REQUEST_COMPLETE_TIMEOUT = JPEG_FRAME_TIMEOUT; 85 86 private static final float ASPECT_RATIO_TOLERANCE = 0.01f; 87 private boolean mPreviewRunning = false; 88 89 private final List<Surface> mPreviewOutputs = new ArrayList<>(); 90 private final List<Surface> mCallbackOutputs = new ArrayList<>(); 91 private GLThreadManager mGLThreadManager; 92 private SurfaceTexture mPreviewTexture; 93 private Camera.Parameters mParams; 94 95 private final List<Long> mJpegSurfaceIds = new ArrayList<>(); 96 97 private Size mIntermediateBufferSize; 98 99 private final RequestQueue mRequestQueue = new RequestQueue(mJpegSurfaceIds); 100 private LegacyRequest mLastRequest = null; 101 private SurfaceTexture mDummyTexture; 102 private Surface mDummySurface; 103 104 private final Object mIdleLock = new Object(); 105 private final FpsCounter mPrevCounter = new FpsCounter("Incoming Preview"); 106 private final FpsCounter mRequestCounter = new FpsCounter("Incoming Requests"); 107 108 private final AtomicBoolean mQuit = new AtomicBoolean(false); 109 110 // Stuff JPEGs into HAL_PIXEL_FORMAT_RGBA_8888 gralloc buffers to get around SW write 111 // limitations for (b/17379185). 112 private static final boolean USE_BLOB_FORMAT_OVERRIDE = true; 113 114 /** 115 * Container object for Configure messages. 116 */ 117 private static class ConfigureHolder { 118 public final ConditionVariable condition; 119 public final Collection<Pair<Surface, Size>> surfaces; 120 121 public ConfigureHolder(ConditionVariable condition, Collection<Pair<Surface, 122 Size>> surfaces) { 123 this.condition = condition; 124 this.surfaces = surfaces; 125 } 126 } 127 128 /** 129 * Counter class used to calculate and log the current FPS of frame production. 130 */ 131 public static class FpsCounter { 132 //TODO: Hook this up to SystTrace? 133 private static final String TAG = "FpsCounter"; 134 private int mFrameCount = 0; 135 private long mLastTime = 0; 136 private long mLastPrintTime = 0; 137 private double mLastFps = 0; 138 private final String mStreamType; 139 private static final long NANO_PER_SECOND = 1000000000; //ns 140 141 public FpsCounter(String streamType) { 142 mStreamType = streamType; 143 } 144 145 public synchronized void countFrame() { 146 mFrameCount++; 147 long nextTime = SystemClock.elapsedRealtimeNanos(); 148 if (mLastTime == 0) { 149 mLastTime = nextTime; 150 } 151 if (nextTime > mLastTime + NANO_PER_SECOND) { 152 long elapsed = nextTime - mLastTime; 153 mLastFps = mFrameCount * (NANO_PER_SECOND / (double) elapsed); 154 mFrameCount = 0; 155 mLastTime = nextTime; 156 } 157 } 158 159 public synchronized double checkFps() { 160 return mLastFps; 161 } 162 163 public synchronized void staggeredLog() { 164 if (mLastTime > mLastPrintTime + 5 * NANO_PER_SECOND) { 165 mLastPrintTime = mLastTime; 166 Log.d(TAG, "FPS for " + mStreamType + " stream: " + mLastFps ); 167 } 168 } 169 170 public synchronized void countAndLog() { 171 countFrame(); 172 staggeredLog(); 173 } 174 } 175 /** 176 * Fake preview for jpeg captures when there is no active preview 177 */ 178 private void createDummySurface() { 179 if (mDummyTexture == null || mDummySurface == null) { 180 mDummyTexture = new SurfaceTexture(/*ignored*/0); 181 // TODO: use smallest default sizes 182 mDummyTexture.setDefaultBufferSize(640, 480); 183 mDummySurface = new Surface(mDummyTexture); 184 } 185 } 186 187 private final Camera.ErrorCallback mErrorCallback = new Camera.ErrorCallback() { 188 @Override 189 public void onError(int i, Camera camera) { 190 switch(i) { 191 case Camera.CAMERA_ERROR_EVICTED: { 192 flush(); 193 mDeviceState.setError( 194 CameraDeviceImpl.CameraDeviceCallbacks.ERROR_CAMERA_DISCONNECTED); 195 } break; 196 default: { 197 Log.e(TAG, "Received error " + i + " from the Camera1 ErrorCallback"); 198 mDeviceState.setError( 199 CameraDeviceImpl.CameraDeviceCallbacks.ERROR_CAMERA_DEVICE); 200 } break; 201 } 202 } 203 }; 204 205 private final ConditionVariable mReceivedJpeg = new ConditionVariable(false); 206 207 private final Camera.PictureCallback mJpegCallback = new Camera.PictureCallback() { 208 @Override 209 public void onPictureTaken(byte[] data, Camera camera) { 210 Log.i(TAG, "Received jpeg."); 211 Pair<RequestHolder, Long> captureInfo = mCaptureCollector.jpegProduced(); 212 if (captureInfo == null || captureInfo.first == null) { 213 Log.e(TAG, "Dropping jpeg frame."); 214 return; 215 } 216 RequestHolder holder = captureInfo.first; 217 long timestamp = captureInfo.second; 218 for (Surface s : holder.getHolderTargets()) { 219 try { 220 if (LegacyCameraDevice.containsSurfaceId(s, mJpegSurfaceIds)) { 221 Log.i(TAG, "Producing jpeg buffer..."); 222 223 int totalSize = data.length + LegacyCameraDevice.nativeGetJpegFooterSize(); 224 totalSize = (totalSize + 3) & ~0x3; // round up to nearest octonibble 225 LegacyCameraDevice.setNextTimestamp(s, timestamp); 226 227 if (USE_BLOB_FORMAT_OVERRIDE) { 228 // Override to RGBA_8888 format. 229 LegacyCameraDevice.setSurfaceFormat(s, 230 LegacyMetadataMapper.HAL_PIXEL_FORMAT_RGBA_8888); 231 232 int dimen = (int) Math.ceil(Math.sqrt(totalSize)); 233 dimen = (dimen + 0xf) & ~0xf; // round up to nearest multiple of 16 234 LegacyCameraDevice.setSurfaceDimens(s, dimen, dimen); 235 LegacyCameraDevice.produceFrame(s, data, dimen, dimen, 236 CameraMetadataNative.NATIVE_JPEG_FORMAT); 237 } else { 238 LegacyCameraDevice.setSurfaceDimens(s, totalSize, /*height*/1); 239 LegacyCameraDevice.produceFrame(s, data, totalSize, /*height*/1, 240 CameraMetadataNative.NATIVE_JPEG_FORMAT); 241 } 242 } 243 } catch (LegacyExceptionUtils.BufferQueueAbandonedException e) { 244 Log.w(TAG, "Surface abandoned, dropping frame. ", e); 245 } 246 } 247 248 mReceivedJpeg.open(); 249 } 250 }; 251 252 private final Camera.ShutterCallback mJpegShutterCallback = new Camera.ShutterCallback() { 253 @Override 254 public void onShutter() { 255 mCaptureCollector.jpegCaptured(SystemClock.elapsedRealtimeNanos()); 256 } 257 }; 258 259 private final SurfaceTexture.OnFrameAvailableListener mPreviewCallback = 260 new SurfaceTexture.OnFrameAvailableListener() { 261 @Override 262 public void onFrameAvailable(SurfaceTexture surfaceTexture) { 263 if (DEBUG) { 264 mPrevCounter.countAndLog(); 265 } 266 mGLThreadManager.queueNewFrame(); 267 } 268 }; 269 270 private void stopPreview() { 271 if (VERBOSE) { 272 Log.v(TAG, "stopPreview - preview running? " + mPreviewRunning); 273 } 274 if (mPreviewRunning) { 275 mCamera.stopPreview(); 276 mPreviewRunning = false; 277 } 278 } 279 280 private void startPreview() { 281 if (VERBOSE) { 282 Log.v(TAG, "startPreview - preview running? " + mPreviewRunning); 283 } 284 if (!mPreviewRunning) { 285 // XX: CameraClient:;startPreview is not getting called after a stop 286 mCamera.startPreview(); 287 mPreviewRunning = true; 288 } 289 } 290 291 private void doJpegCapturePrepare(RequestHolder request) throws IOException { 292 if (DEBUG) Log.d(TAG, "doJpegCapturePrepare - preview running? " + mPreviewRunning); 293 294 if (!mPreviewRunning) { 295 if (DEBUG) Log.d(TAG, "doJpegCapture - create fake surface"); 296 297 createDummySurface(); 298 mCamera.setPreviewTexture(mDummyTexture); 299 startPreview(); 300 } 301 } 302 303 private void doJpegCapture(RequestHolder request) { 304 if (DEBUG) Log.d(TAG, "doJpegCapturePrepare"); 305 306 mCamera.takePicture(mJpegShutterCallback, /*raw*/null, mJpegCallback); 307 mPreviewRunning = false; 308 } 309 310 private void doPreviewCapture(RequestHolder request) throws IOException { 311 if (VERBOSE) { 312 Log.v(TAG, "doPreviewCapture - preview running? " + mPreviewRunning); 313 } 314 315 if (mPreviewRunning) { 316 return; // Already running 317 } 318 319 if (mPreviewTexture == null) { 320 throw new IllegalStateException( 321 "Preview capture called with no preview surfaces configured."); 322 } 323 324 mPreviewTexture.setDefaultBufferSize(mIntermediateBufferSize.getWidth(), 325 mIntermediateBufferSize.getHeight()); 326 mCamera.setPreviewTexture(mPreviewTexture); 327 328 startPreview(); 329 } 330 331 private void configureOutputs(Collection<Pair<Surface, Size>> outputs) { 332 if (DEBUG) { 333 String outputsStr = outputs == null ? "null" : (outputs.size() + " surfaces"); 334 Log.d(TAG, "configureOutputs with " + outputsStr); 335 } 336 337 try { 338 stopPreview(); 339 } catch (RuntimeException e) { 340 Log.e(TAG, "Received device exception in configure call: ", e); 341 mDeviceState.setError( 342 CameraDeviceImpl.CameraDeviceCallbacks.ERROR_CAMERA_DEVICE); 343 return; 344 } 345 346 /* 347 * Try to release the previous preview's surface texture earlier if we end up 348 * using a different one; this also reduces the likelihood of getting into a deadlock 349 * when disconnecting from the old previous texture at a later time. 350 */ 351 try { 352 mCamera.setPreviewTexture(/*surfaceTexture*/null); 353 } catch (IOException e) { 354 Log.w(TAG, "Failed to clear prior SurfaceTexture, may cause GL deadlock: ", e); 355 } catch (RuntimeException e) { 356 Log.e(TAG, "Received device exception in configure call: ", e); 357 mDeviceState.setError( 358 CameraDeviceImpl.CameraDeviceCallbacks.ERROR_CAMERA_DEVICE); 359 return; 360 } 361 362 if (mGLThreadManager != null) { 363 mGLThreadManager.waitUntilStarted(); 364 mGLThreadManager.ignoreNewFrames(); 365 mGLThreadManager.waitUntilIdle(); 366 } 367 resetJpegSurfaceFormats(mCallbackOutputs); 368 369 for (Surface s : mCallbackOutputs) { 370 try { 371 LegacyCameraDevice.disconnectSurface(s); 372 } catch (LegacyExceptionUtils.BufferQueueAbandonedException e) { 373 Log.w(TAG, "Surface abandoned, skipping...", e); 374 } 375 } 376 mPreviewOutputs.clear(); 377 mCallbackOutputs.clear(); 378 mJpegSurfaceIds.clear(); 379 mPreviewTexture = null; 380 381 List<Size> previewOutputSizes = new ArrayList<>(); 382 List<Size> callbackOutputSizes = new ArrayList<>(); 383 384 int facing = mCharacteristics.get(CameraCharacteristics.LENS_FACING); 385 int orientation = mCharacteristics.get(CameraCharacteristics.SENSOR_ORIENTATION); 386 if (outputs != null) { 387 for (Pair<Surface, Size> outPair : outputs) { 388 Surface s = outPair.first; 389 Size outSize = outPair.second; 390 try { 391 int format = LegacyCameraDevice.detectSurfaceType(s); 392 LegacyCameraDevice.setSurfaceOrientation(s, facing, orientation); 393 switch (format) { 394 case CameraMetadataNative.NATIVE_JPEG_FORMAT: 395 if (USE_BLOB_FORMAT_OVERRIDE) { 396 // Override to RGBA_8888 format. 397 LegacyCameraDevice.setSurfaceFormat(s, 398 LegacyMetadataMapper.HAL_PIXEL_FORMAT_RGBA_8888); 399 } 400 mJpegSurfaceIds.add(LegacyCameraDevice.getSurfaceId(s)); 401 mCallbackOutputs.add(s); 402 callbackOutputSizes.add(outSize); 403 404 // LegacyCameraDevice is the producer of JPEG output surfaces 405 // so LegacyCameraDevice needs to connect to the surfaces. 406 LegacyCameraDevice.connectSurface(s); 407 break; 408 default: 409 LegacyCameraDevice.setScalingMode(s, LegacyCameraDevice. 410 NATIVE_WINDOW_SCALING_MODE_SCALE_TO_WINDOW); 411 mPreviewOutputs.add(s); 412 previewOutputSizes.add(outSize); 413 break; 414 } 415 } catch (LegacyExceptionUtils.BufferQueueAbandonedException e) { 416 Log.w(TAG, "Surface abandoned, skipping...", e); 417 } 418 } 419 } 420 try { 421 mParams = mCamera.getParameters(); 422 } catch (RuntimeException e) { 423 Log.e(TAG, "Received device exception: ", e); 424 mDeviceState.setError( 425 CameraDeviceImpl.CameraDeviceCallbacks.ERROR_CAMERA_DEVICE); 426 return; 427 } 428 429 List<int[]> supportedFpsRanges = mParams.getSupportedPreviewFpsRange(); 430 int[] bestRange = getPhotoPreviewFpsRange(supportedFpsRanges); 431 if (DEBUG) { 432 Log.d(TAG, "doPreviewCapture - Selected range [" + 433 bestRange[Camera.Parameters.PREVIEW_FPS_MIN_INDEX] + "," + 434 bestRange[Camera.Parameters.PREVIEW_FPS_MAX_INDEX] + "]"); 435 } 436 mParams.setPreviewFpsRange(bestRange[Camera.Parameters.PREVIEW_FPS_MIN_INDEX], 437 bestRange[Camera.Parameters.PREVIEW_FPS_MAX_INDEX]); 438 439 Size smallestSupportedJpegSize = calculatePictureSize(mCallbackOutputs, 440 callbackOutputSizes, mParams); 441 442 if (previewOutputSizes.size() > 0) { 443 444 Size largestOutput = SizeAreaComparator.findLargestByArea(previewOutputSizes); 445 446 // Find largest jpeg dimension - assume to have the same aspect ratio as sensor. 447 Size largestJpegDimen = ParameterUtils.getLargestSupportedJpegSizeByArea(mParams); 448 449 Size chosenJpegDimen = (smallestSupportedJpegSize != null) ? smallestSupportedJpegSize 450 : largestJpegDimen; 451 452 List<Size> supportedPreviewSizes = ParameterUtils.convertSizeList( 453 mParams.getSupportedPreviewSizes()); 454 455 // Use smallest preview dimension with same aspect ratio as sensor that is >= than all 456 // of the configured output dimensions. If none exists, fall back to using the largest 457 // supported preview size. 458 long largestOutputArea = largestOutput.getHeight() * (long) largestOutput.getWidth(); 459 Size bestPreviewDimen = SizeAreaComparator.findLargestByArea(supportedPreviewSizes); 460 for (Size s : supportedPreviewSizes) { 461 long currArea = s.getWidth() * s.getHeight(); 462 long bestArea = bestPreviewDimen.getWidth() * bestPreviewDimen.getHeight(); 463 if (checkAspectRatiosMatch(chosenJpegDimen, s) && (currArea < bestArea && 464 currArea >= largestOutputArea)) { 465 bestPreviewDimen = s; 466 } 467 } 468 469 mIntermediateBufferSize = bestPreviewDimen; 470 mParams.setPreviewSize(mIntermediateBufferSize.getWidth(), 471 mIntermediateBufferSize.getHeight()); 472 473 if (DEBUG) { 474 Log.d(TAG, "Intermediate buffer selected with dimens: " + 475 bestPreviewDimen.toString()); 476 } 477 } else { 478 mIntermediateBufferSize = null; 479 if (DEBUG) { 480 Log.d(TAG, "No Intermediate buffer selected, no preview outputs were configured"); 481 } 482 } 483 484 if (smallestSupportedJpegSize != null) { 485 /* 486 * Set takePicture size to the smallest supported JPEG size large enough 487 * to scale/crop out of for the bounding rectangle of the configured JPEG sizes. 488 */ 489 490 Log.i(TAG, "configureOutputs - set take picture size to " + smallestSupportedJpegSize); 491 mParams.setPictureSize( 492 smallestSupportedJpegSize.getWidth(), smallestSupportedJpegSize.getHeight()); 493 } 494 495 // TODO: Detect and optimize single-output paths here to skip stream teeing. 496 if (mGLThreadManager == null) { 497 mGLThreadManager = new GLThreadManager(mCameraId, facing, mDeviceState); 498 mGLThreadManager.start(); 499 } 500 mGLThreadManager.waitUntilStarted(); 501 List<Pair<Surface, Size>> previews = new ArrayList<>(); 502 Iterator<Size> previewSizeIter = previewOutputSizes.iterator(); 503 for (Surface p : mPreviewOutputs) { 504 previews.add(new Pair<>(p, previewSizeIter.next())); 505 } 506 mGLThreadManager.setConfigurationAndWait(previews, mCaptureCollector); 507 508 for (Surface p : mPreviewOutputs) { 509 try { 510 LegacyCameraDevice.setSurfaceOrientation(p, facing, orientation); 511 } catch (LegacyExceptionUtils.BufferQueueAbandonedException e) { 512 Log.e(TAG, "Surface abandoned, skipping setSurfaceOrientation()", e); 513 } 514 } 515 516 mGLThreadManager.allowNewFrames(); 517 mPreviewTexture = mGLThreadManager.getCurrentSurfaceTexture(); 518 if (mPreviewTexture != null) { 519 mPreviewTexture.setOnFrameAvailableListener(mPreviewCallback); 520 } 521 522 try { 523 mCamera.setParameters(mParams); 524 } catch (RuntimeException e) { 525 Log.e(TAG, "Received device exception while configuring: ", e); 526 mDeviceState.setError( 527 CameraDeviceImpl.CameraDeviceCallbacks.ERROR_CAMERA_DEVICE); 528 529 } 530 } 531 532 private void resetJpegSurfaceFormats(Collection<Surface> surfaces) { 533 if (!USE_BLOB_FORMAT_OVERRIDE || surfaces == null) { 534 return; 535 } 536 for(Surface s : surfaces) { 537 if (s == null || !s.isValid()) { 538 Log.w(TAG, "Jpeg surface is invalid, skipping..."); 539 continue; 540 } 541 try { 542 LegacyCameraDevice.setSurfaceFormat(s, LegacyMetadataMapper.HAL_PIXEL_FORMAT_BLOB); 543 } catch (LegacyExceptionUtils.BufferQueueAbandonedException e) { 544 Log.w(TAG, "Surface abandoned, skipping...", e); 545 } 546 } 547 } 548 549 /** 550 * Find a JPEG size (that is supported by the legacy camera device) which is equal to or larger 551 * than all of the configured {@code JPEG} outputs (by both width and height). 552 * 553 * <p>If multiple supported JPEG sizes are larger, select the smallest of them which 554 * still satisfies the above constraint.</p> 555 * 556 * <p>As a result, the returned size is guaranteed to be usable without needing 557 * to upscale any of the outputs. If only one {@code JPEG} surface is used, 558 * then no scaling/cropping is necessary between the taken picture and 559 * the {@code JPEG} output surface.</p> 560 * 561 * @param callbackOutputs a non-{@code null} list of {@code Surface}s with any image formats 562 * @param params api1 parameters (used for reading only) 563 * 564 * @return a size large enough to fit all of the configured {@code JPEG} outputs, or 565 * {@code null} if the {@code callbackOutputs} did not have any {@code JPEG} 566 * surfaces. 567 */ 568 private Size calculatePictureSize( List<Surface> callbackOutputs, 569 List<Size> callbackSizes, Camera.Parameters params) { 570 /* 571 * Find the largest JPEG size (if any), from the configured outputs: 572 * - the api1 picture size should be set to the smallest legal size that's at least as large 573 * as the largest configured JPEG size 574 */ 575 if (callbackOutputs.size() != callbackSizes.size()) { 576 throw new IllegalStateException("Input collections must be same length"); 577 } 578 List<Size> configuredJpegSizes = new ArrayList<>(); 579 Iterator<Size> sizeIterator = callbackSizes.iterator(); 580 for (Surface callbackSurface : callbackOutputs) { 581 Size jpegSize = sizeIterator.next(); 582 if (!LegacyCameraDevice.containsSurfaceId(callbackSurface, mJpegSurfaceIds)) { 583 continue; // Ignore non-JPEG callback formats 584 } 585 586 configuredJpegSizes.add(jpegSize); 587 } 588 if (!configuredJpegSizes.isEmpty()) { 589 /* 590 * Find the largest configured JPEG width, and height, independently 591 * of the rest. 592 * 593 * The rest of the JPEG streams can be cropped out of this smallest bounding 594 * rectangle. 595 */ 596 int maxConfiguredJpegWidth = -1; 597 int maxConfiguredJpegHeight = -1; 598 for (Size jpegSize : configuredJpegSizes) { 599 maxConfiguredJpegWidth = jpegSize.getWidth() > maxConfiguredJpegWidth ? 600 jpegSize.getWidth() : maxConfiguredJpegWidth; 601 maxConfiguredJpegHeight = jpegSize.getHeight() > maxConfiguredJpegHeight ? 602 jpegSize.getHeight() : maxConfiguredJpegHeight; 603 } 604 Size smallestBoundJpegSize = new Size(maxConfiguredJpegWidth, maxConfiguredJpegHeight); 605 606 List<Size> supportedJpegSizes = ParameterUtils.convertSizeList( 607 params.getSupportedPictureSizes()); 608 609 /* 610 * Find the smallest supported JPEG size that can fit the smallest bounding 611 * rectangle for the configured JPEG sizes. 612 */ 613 List<Size> candidateSupportedJpegSizes = new ArrayList<>(); 614 for (Size supportedJpegSize : supportedJpegSizes) { 615 if (supportedJpegSize.getWidth() >= maxConfiguredJpegWidth && 616 supportedJpegSize.getHeight() >= maxConfiguredJpegHeight) { 617 candidateSupportedJpegSizes.add(supportedJpegSize); 618 } 619 } 620 621 if (candidateSupportedJpegSizes.isEmpty()) { 622 throw new AssertionError( 623 "Could not find any supported JPEG sizes large enough to fit " + 624 smallestBoundJpegSize); 625 } 626 627 Size smallestSupportedJpegSize = Collections.min(candidateSupportedJpegSizes, 628 new SizeAreaComparator()); 629 630 if (!smallestSupportedJpegSize.equals(smallestBoundJpegSize)) { 631 Log.w(TAG, 632 String.format( 633 "configureOutputs - Will need to crop picture %s into " 634 + "smallest bound size %s", 635 smallestSupportedJpegSize, smallestBoundJpegSize)); 636 } 637 638 return smallestSupportedJpegSize; 639 } 640 641 return null; 642 } 643 644 private static boolean checkAspectRatiosMatch(Size a, Size b) { 645 float aAspect = a.getWidth() / (float) a.getHeight(); 646 float bAspect = b.getWidth() / (float) b.getHeight(); 647 648 return Math.abs(aAspect - bAspect) < ASPECT_RATIO_TOLERANCE; 649 } 650 651 // Calculate the highest FPS range supported 652 private int[] getPhotoPreviewFpsRange(List<int[]> frameRates) { 653 if (frameRates.size() == 0) { 654 Log.e(TAG, "No supported frame rates returned!"); 655 return null; 656 } 657 658 int bestMin = 0; 659 int bestMax = 0; 660 int bestIndex = 0; 661 int index = 0; 662 for (int[] rate : frameRates) { 663 int minFps = rate[Camera.Parameters.PREVIEW_FPS_MIN_INDEX]; 664 int maxFps = rate[Camera.Parameters.PREVIEW_FPS_MAX_INDEX]; 665 if (maxFps > bestMax || (maxFps == bestMax && minFps > bestMin)) { 666 bestMin = minFps; 667 bestMax = maxFps; 668 bestIndex = index; 669 } 670 index++; 671 } 672 673 return frameRates.get(bestIndex); 674 } 675 676 private final Handler.Callback mRequestHandlerCb = new Handler.Callback() { 677 private boolean mCleanup = false; 678 private final LegacyResultMapper mMapper = new LegacyResultMapper(); 679 680 @Override 681 public boolean handleMessage(Message msg) { 682 if (mCleanup) { 683 return true; 684 } 685 686 if (DEBUG) { 687 Log.d(TAG, "Request thread handling message:" + msg.what); 688 } 689 long startTime = 0; 690 if (DEBUG) { 691 startTime = SystemClock.elapsedRealtimeNanos(); 692 } 693 switch (msg.what) { 694 case MSG_CONFIGURE_OUTPUTS: 695 ConfigureHolder config = (ConfigureHolder) msg.obj; 696 int sizes = config.surfaces != null ? config.surfaces.size() : 0; 697 Log.i(TAG, "Configure outputs: " + sizes + " surfaces configured."); 698 699 try { 700 boolean success = mCaptureCollector.waitForEmpty(JPEG_FRAME_TIMEOUT, 701 TimeUnit.MILLISECONDS); 702 if (!success) { 703 Log.e(TAG, "Timed out while queueing configure request."); 704 mCaptureCollector.failAll(); 705 } 706 } catch (InterruptedException e) { 707 Log.e(TAG, "Interrupted while waiting for requests to complete."); 708 mDeviceState.setError( 709 CameraDeviceImpl.CameraDeviceCallbacks.ERROR_CAMERA_DEVICE); 710 break; 711 } 712 713 configureOutputs(config.surfaces); 714 config.condition.open(); 715 if (DEBUG) { 716 long totalTime = SystemClock.elapsedRealtimeNanos() - startTime; 717 Log.d(TAG, "Configure took " + totalTime + " ns"); 718 } 719 break; 720 case MSG_SUBMIT_CAPTURE_REQUEST: 721 Handler handler = RequestThreadManager.this.mRequestThread.getHandler(); 722 boolean anyRequestOutputAbandoned = false; 723 724 // Get the next burst from the request queue. 725 RequestQueue.RequestQueueEntry nextBurst = mRequestQueue.getNext(); 726 727 if (nextBurst == null) { 728 // If there are no further requests queued, wait for any currently executing 729 // requests to complete, then switch to idle state. 730 try { 731 boolean success = mCaptureCollector.waitForEmpty(JPEG_FRAME_TIMEOUT, 732 TimeUnit.MILLISECONDS); 733 if (!success) { 734 Log.e(TAG, 735 "Timed out while waiting for prior requests to complete."); 736 mCaptureCollector.failAll(); 737 } 738 } catch (InterruptedException e) { 739 Log.e(TAG, "Interrupted while waiting for requests to complete: ", e); 740 mDeviceState.setError( 741 CameraDeviceImpl.CameraDeviceCallbacks.ERROR_CAMERA_DEVICE); 742 break; 743 } 744 745 synchronized (mIdleLock) { 746 // Retry the the request queue. 747 nextBurst = mRequestQueue.getNext(); 748 749 // If we still have no queued requests, go idle. 750 if (nextBurst == null) { 751 mDeviceState.setIdle(); 752 break; 753 } 754 } 755 } 756 757 if (nextBurst != null) { 758 // Queue another capture if we did not get the last burst. 759 handler.sendEmptyMessage(MSG_SUBMIT_CAPTURE_REQUEST); 760 761 // Check whether capture queue becomes empty 762 if (nextBurst.isQueueEmpty()) { 763 mDeviceState.setRequestQueueEmpty(); 764 } 765 } 766 767 // Complete each request in the burst 768 BurstHolder burstHolder = nextBurst.getBurstHolder(); 769 List<RequestHolder> requests = 770 burstHolder.produceRequestHolders(nextBurst.getFrameNumber()); 771 for (RequestHolder holder : requests) { 772 CaptureRequest request = holder.getRequest(); 773 774 boolean paramsChanged = false; 775 776 // Only update parameters if the request has changed 777 if (mLastRequest == null || mLastRequest.captureRequest != request) { 778 779 // The intermediate buffer is sometimes null, but we always need 780 // the Camera1 API configured preview size 781 Size previewSize = ParameterUtils.convertSize(mParams.getPreviewSize()); 782 783 LegacyRequest legacyRequest = new LegacyRequest(mCharacteristics, 784 request, previewSize, mParams); // params are copied 785 786 787 // Parameters are mutated as a side-effect 788 LegacyMetadataMapper.convertRequestMetadata(/*inout*/legacyRequest); 789 790 // If the parameters have changed, set them in the Camera1 API. 791 if (!mParams.same(legacyRequest.parameters)) { 792 try { 793 mCamera.setParameters(legacyRequest.parameters); 794 } catch (RuntimeException e) { 795 // If setting the parameters failed, report a request error to 796 // the camera client, and skip any further work for this request 797 Log.e(TAG, "Exception while setting camera parameters: ", e); 798 holder.failRequest(); 799 mDeviceState.setCaptureStart(holder, /*timestamp*/0, 800 CameraDeviceImpl.CameraDeviceCallbacks. 801 ERROR_CAMERA_REQUEST); 802 continue; 803 } 804 paramsChanged = true; 805 mParams = legacyRequest.parameters; 806 } 807 808 mLastRequest = legacyRequest; 809 } 810 811 try { 812 boolean success = mCaptureCollector.queueRequest(holder, 813 mLastRequest, JPEG_FRAME_TIMEOUT, TimeUnit.MILLISECONDS); 814 815 if (!success) { 816 // Report a request error if we timed out while queuing this. 817 Log.e(TAG, "Timed out while queueing capture request."); 818 holder.failRequest(); 819 mDeviceState.setCaptureStart(holder, /*timestamp*/0, 820 CameraDeviceImpl.CameraDeviceCallbacks. 821 ERROR_CAMERA_REQUEST); 822 continue; 823 } 824 825 // Starting the preview needs to happen before enabling 826 // face detection or auto focus 827 if (holder.hasPreviewTargets()) { 828 doPreviewCapture(holder); 829 } 830 if (holder.hasJpegTargets()) { 831 while(!mCaptureCollector.waitForPreviewsEmpty(PREVIEW_FRAME_TIMEOUT, 832 TimeUnit.MILLISECONDS)) { 833 // Fail preview requests until the queue is empty. 834 Log.e(TAG, "Timed out while waiting for preview requests to " + 835 "complete."); 836 mCaptureCollector.failNextPreview(); 837 } 838 mReceivedJpeg.close(); 839 doJpegCapturePrepare(holder); 840 } 841 842 /* 843 * Do all the actions that require a preview to have been started 844 */ 845 846 // Toggle face detection on/off 847 // - do this before AF to give AF a chance to use faces 848 mFaceDetectMapper.processFaceDetectMode(request, /*in*/mParams); 849 850 // Unconditionally process AF triggers, since they're non-idempotent 851 // - must be done after setting the most-up-to-date AF mode 852 mFocusStateMapper.processRequestTriggers(request, mParams); 853 854 if (holder.hasJpegTargets()) { 855 doJpegCapture(holder); 856 if (!mReceivedJpeg.block(JPEG_FRAME_TIMEOUT)) { 857 Log.e(TAG, "Hit timeout for jpeg callback!"); 858 mCaptureCollector.failNextJpeg(); 859 } 860 } 861 862 } catch (IOException e) { 863 Log.e(TAG, "Received device exception during capture call: ", e); 864 mDeviceState.setError( 865 CameraDeviceImpl.CameraDeviceCallbacks.ERROR_CAMERA_DEVICE); 866 break; 867 } catch (InterruptedException e) { 868 Log.e(TAG, "Interrupted during capture: ", e); 869 mDeviceState.setError( 870 CameraDeviceImpl.CameraDeviceCallbacks.ERROR_CAMERA_DEVICE); 871 break; 872 } catch (RuntimeException e) { 873 Log.e(TAG, "Received device exception during capture call: ", e); 874 mDeviceState.setError( 875 CameraDeviceImpl.CameraDeviceCallbacks.ERROR_CAMERA_DEVICE); 876 break; 877 } 878 879 if (paramsChanged) { 880 if (DEBUG) { 881 Log.d(TAG, "Params changed -- getting new Parameters from HAL."); 882 } 883 try { 884 mParams = mCamera.getParameters(); 885 } catch (RuntimeException e) { 886 Log.e(TAG, "Received device exception: ", e); 887 mDeviceState.setError( 888 CameraDeviceImpl.CameraDeviceCallbacks.ERROR_CAMERA_DEVICE); 889 break; 890 } 891 892 // Update parameters to the latest that we think the camera is using 893 mLastRequest.setParameters(mParams); 894 } 895 896 MutableLong timestampMutable = new MutableLong(/*value*/0L); 897 try { 898 boolean success = mCaptureCollector.waitForRequestCompleted(holder, 899 REQUEST_COMPLETE_TIMEOUT, TimeUnit.MILLISECONDS, 900 /*out*/timestampMutable); 901 902 if (!success) { 903 Log.e(TAG, "Timed out while waiting for request to complete."); 904 mCaptureCollector.failAll(); 905 } 906 } catch (InterruptedException e) { 907 Log.e(TAG, "Interrupted waiting for request completion: ", e); 908 mDeviceState.setError( 909 CameraDeviceImpl.CameraDeviceCallbacks.ERROR_CAMERA_DEVICE); 910 break; 911 } 912 913 CameraMetadataNative result = mMapper.cachedConvertResultMetadata( 914 mLastRequest, timestampMutable.value); 915 /* 916 * Order matters: The default result mapper is state-less; the 917 * other mappers carry state and may override keys set by the default 918 * mapper with their own values. 919 */ 920 921 // Update AF state 922 mFocusStateMapper.mapResultTriggers(result); 923 // Update face-related results 924 mFaceDetectMapper.mapResultFaces(result, mLastRequest); 925 926 if (!holder.requestFailed()) { 927 mDeviceState.setCaptureResult(holder, result); 928 } 929 930 if (holder.isOutputAbandoned()) { 931 anyRequestOutputAbandoned = true; 932 } 933 } 934 935 // Stop the repeating request if any of its output surfaces is abandoned. 936 if (anyRequestOutputAbandoned && burstHolder.isRepeating()) { 937 long lastFrameNumber = cancelRepeating(burstHolder.getRequestId()); 938 if (DEBUG) { 939 Log.d(TAG, "Stopped repeating request. Last frame number is " + 940 lastFrameNumber); 941 } 942 mDeviceState.setRepeatingRequestError(lastFrameNumber, 943 burstHolder.getRequestId()); 944 } 945 946 if (DEBUG) { 947 long totalTime = SystemClock.elapsedRealtimeNanos() - startTime; 948 Log.d(TAG, "Capture request took " + totalTime + " ns"); 949 mRequestCounter.countAndLog(); 950 } 951 break; 952 case MSG_CLEANUP: 953 mCleanup = true; 954 try { 955 boolean success = mCaptureCollector.waitForEmpty(JPEG_FRAME_TIMEOUT, 956 TimeUnit.MILLISECONDS); 957 if (!success) { 958 Log.e(TAG, "Timed out while queueing cleanup request."); 959 mCaptureCollector.failAll(); 960 } 961 } catch (InterruptedException e) { 962 Log.e(TAG, "Interrupted while waiting for requests to complete: ", e); 963 mDeviceState.setError( 964 CameraDeviceImpl.CameraDeviceCallbacks.ERROR_CAMERA_DEVICE); 965 } 966 if (mGLThreadManager != null) { 967 mGLThreadManager.quit(); 968 mGLThreadManager = null; 969 } 970 if (mCamera != null) { 971 mCamera.release(); 972 mCamera = null; 973 } 974 resetJpegSurfaceFormats(mCallbackOutputs); 975 break; 976 case RequestHandlerThread.MSG_POKE_IDLE_HANDLER: 977 // OK: Ignore message. 978 break; 979 default: 980 throw new AssertionError("Unhandled message " + msg.what + 981 " on RequestThread."); 982 } 983 return true; 984 } 985 }; 986 987 /** 988 * Create a new RequestThreadManager. 989 * 990 * @param cameraId the id of the camera to use. 991 * @param camera an open camera object. The RequestThreadManager takes ownership of this camera 992 * object, and is responsible for closing it. 993 * @param characteristics the static camera characteristics corresponding to this camera device 994 * @param deviceState a {@link CameraDeviceState} state machine. 995 */ 996 public RequestThreadManager(int cameraId, Camera camera, CameraCharacteristics characteristics, 997 CameraDeviceState deviceState) { 998 mCamera = checkNotNull(camera, "camera must not be null"); 999 mCameraId = cameraId; 1000 mCharacteristics = checkNotNull(characteristics, "characteristics must not be null"); 1001 String name = String.format("RequestThread-%d", cameraId); 1002 TAG = name; 1003 mDeviceState = checkNotNull(deviceState, "deviceState must not be null"); 1004 mFocusStateMapper = new LegacyFocusStateMapper(mCamera); 1005 mFaceDetectMapper = new LegacyFaceDetectMapper(mCamera, mCharacteristics); 1006 mCaptureCollector = new CaptureCollector(MAX_IN_FLIGHT_REQUESTS, mDeviceState); 1007 mRequestThread = new RequestHandlerThread(name, mRequestHandlerCb); 1008 mCamera.setErrorCallback(mErrorCallback); 1009 } 1010 1011 /** 1012 * Start the request thread. 1013 */ 1014 public void start() { 1015 mRequestThread.start(); 1016 } 1017 1018 /** 1019 * Flush any pending requests. 1020 * 1021 * @return the last frame number. 1022 */ 1023 public long flush() { 1024 Log.i(TAG, "Flushing all pending requests."); 1025 long lastFrame = mRequestQueue.stopRepeating(); 1026 mCaptureCollector.failAll(); 1027 return lastFrame; 1028 } 1029 1030 /** 1031 * Quit the request thread, and clean up everything. 1032 */ 1033 public void quit() { 1034 if (!mQuit.getAndSet(true)) { // Avoid sending messages on dead thread's handler. 1035 Handler handler = mRequestThread.waitAndGetHandler(); 1036 handler.sendMessageAtFrontOfQueue(handler.obtainMessage(MSG_CLEANUP)); 1037 mRequestThread.quitSafely(); 1038 try { 1039 mRequestThread.join(); 1040 } catch (InterruptedException e) { 1041 Log.e(TAG, String.format("Thread %s (%d) interrupted while quitting.", 1042 mRequestThread.getName(), mRequestThread.getId())); 1043 } 1044 } 1045 } 1046 1047 /** 1048 * Submit the given burst of requests to be captured. 1049 * 1050 * <p>If the burst is repeating, replace the current repeating burst.</p> 1051 * 1052 * @param requests the burst of requests to add to the queue. 1053 * @param repeating true if the burst is repeating. 1054 * @return the submission info, including the new request id, and the last frame number, which 1055 * contains either the frame number of the last frame that will be returned for this request, 1056 * or the frame number of the last frame that will be returned for the current repeating 1057 * request if this burst is set to be repeating. 1058 */ 1059 public SubmitInfo submitCaptureRequests(CaptureRequest[] requests, boolean repeating) { 1060 Handler handler = mRequestThread.waitAndGetHandler(); 1061 SubmitInfo info; 1062 synchronized (mIdleLock) { 1063 info = mRequestQueue.submit(requests, repeating); 1064 handler.sendEmptyMessage(MSG_SUBMIT_CAPTURE_REQUEST); 1065 } 1066 return info; 1067 } 1068 1069 /** 1070 * Cancel a repeating request. 1071 * 1072 * @param requestId the id of the repeating request to cancel. 1073 * @return the last frame to be returned from the HAL for the given repeating request, or 1074 * {@code INVALID_FRAME} if none exists. 1075 */ 1076 public long cancelRepeating(int requestId) { 1077 return mRequestQueue.stopRepeating(requestId); 1078 } 1079 1080 /** 1081 * Configure with the current list of output Surfaces. 1082 * 1083 * <p> 1084 * This operation blocks until the configuration is complete. 1085 * </p> 1086 * 1087 * <p>Using a {@code null} or empty {@code outputs} list is the equivalent of unconfiguring.</p> 1088 * 1089 * @param outputs a {@link java.util.Collection} of outputs to configure. 1090 */ 1091 public void configure(Collection<Pair<Surface, Size>> outputs) { 1092 Handler handler = mRequestThread.waitAndGetHandler(); 1093 final ConditionVariable condition = new ConditionVariable(/*closed*/false); 1094 ConfigureHolder holder = new ConfigureHolder(condition, outputs); 1095 handler.sendMessage(handler.obtainMessage(MSG_CONFIGURE_OUTPUTS, 0, 0, holder)); 1096 condition.block(); 1097 } 1098 } 1099
