1 /* 2 * Copyright (C) 2014 The Android Open Source Project Licensed under the Apache 3 * License, Version 2.0 (the "License"); you may not use this file except in 4 * compliance with the License. You may obtain a copy of the License at 5 * http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law 6 * or agreed to in writing, software distributed under the License is 7 * distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 8 * KIND, either express or implied. See the License for the specific language 9 * governing permissions and limitations under the License. 10 */ 11 12 package android.hardware.camera2.cts; 13 14 import static android.hardware.camera2.cts.CameraTestUtils.*; 15 import static com.android.ex.camera2.blocking.BlockingSessionCallback.*; 16 17 import android.cts.util.MediaUtils; 18 import android.graphics.ImageFormat; 19 import android.hardware.camera2.CameraCharacteristics; 20 import android.hardware.camera2.CameraCaptureSession; 21 import android.hardware.camera2.CameraConstrainedHighSpeedCaptureSession; 22 import android.hardware.camera2.CameraDevice; 23 import android.hardware.camera2.CaptureRequest; 24 import android.hardware.camera2.CaptureResult; 25 import android.hardware.camera2.params.StreamConfigurationMap; 26 import android.util.Size; 27 import android.hardware.camera2.cts.testcases.Camera2SurfaceViewTestCase; 28 import android.media.CamcorderProfile; 29 import android.media.MediaCodec; 30 import android.media.MediaCodecInfo; 31 import android.media.MediaCodecInfo.CodecCapabilities; 32 import android.media.MediaCodecInfo.CodecProfileLevel; 33 import android.media.Image; 34 import android.media.ImageReader; 35 import android.media.MediaCodecList; 36 import android.media.MediaExtractor; 37 import android.media.MediaFormat; 38 import android.media.MediaRecorder; 39 import android.os.Environment; 40 import android.os.SystemClock; 41 import android.test.suitebuilder.annotation.LargeTest; 42 import android.util.Log; 43 import android.util.Range; 44 import android.view.Surface; 45 46 import com.android.ex.camera2.blocking.BlockingSessionCallback; 47 48 import junit.framework.AssertionFailedError; 49 50 import java.io.File; 51 import java.util.ArrayList; 52 import java.util.Arrays; 53 import java.util.Collections; 54 import java.util.List; 55 import java.util.HashMap; 56 57 /** 58 * CameraDevice video recording use case tests by using MediaRecorder and 59 * MediaCodec. 60 */ 61 @LargeTest 62 public class RecordingTest extends Camera2SurfaceViewTestCase { 63 private static final String TAG = "RecordingTest"; 64 private static final boolean VERBOSE = Log.isLoggable(TAG, Log.VERBOSE); 65 private static final boolean DEBUG_DUMP = Log.isLoggable(TAG, Log.DEBUG); 66 private static final int RECORDING_DURATION_MS = 3000; 67 private static final float DURATION_MARGIN = 0.2f; 68 private static final double FRAME_DURATION_ERROR_TOLERANCE_MS = 3.0; 69 private static final float FRAMEDURATION_MARGIN = 0.2f; 70 private static final float VID_SNPSHT_FRMDRP_RATE_TOLERANCE = 10.0f; 71 private static final float FRMDRP_RATE_TOLERANCE = 5.0f; 72 private static final int BIT_RATE_1080P = 16000000; 73 private static final int BIT_RATE_MIN = 64000; 74 private static final int BIT_RATE_MAX = 40000000; 75 private static final int VIDEO_FRAME_RATE = 30; 76 private final String VIDEO_FILE_PATH = Environment.getExternalStorageDirectory().getPath(); 77 private static final int[] mCamcorderProfileList = { 78 CamcorderProfile.QUALITY_HIGH, 79 CamcorderProfile.QUALITY_2160P, 80 CamcorderProfile.QUALITY_1080P, 81 CamcorderProfile.QUALITY_720P, 82 CamcorderProfile.QUALITY_480P, 83 CamcorderProfile.QUALITY_CIF, 84 CamcorderProfile.QUALITY_QCIF, 85 CamcorderProfile.QUALITY_QVGA, 86 CamcorderProfile.QUALITY_LOW, 87 }; 88 private static final int MAX_VIDEO_SNAPSHOT_IMAGES = 5; 89 private static final int BURST_VIDEO_SNAPSHOT_NUM = 3; 90 private static final int SLOWMO_SLOW_FACTOR = 4; 91 private static final int MAX_NUM_FRAME_DROP_INTERVAL_ALLOWED = 4; 92 private List<Size> mSupportedVideoSizes; 93 private Surface mRecordingSurface; 94 private Surface mPersistentSurface; 95 private MediaRecorder mMediaRecorder; 96 private String mOutMediaFileName; 97 private int mVideoFrameRate; 98 private Size mVideoSize; 99 private long mRecordingStartTime; 100 101 @Override 102 protected void setUp() throws Exception { 103 super.setUp(); 104 } 105 106 @Override 107 protected void tearDown() throws Exception { 108 super.tearDown(); 109 } 110 111 private void doBasicRecording(boolean useVideoStab) throws Exception { 112 for (int i = 0; i < mCameraIds.length; i++) { 113 try { 114 Log.i(TAG, "Testing basic recording for camera " + mCameraIds[i]); 115 // Re-use the MediaRecorder object for the same camera device. 116 mMediaRecorder = new MediaRecorder(); 117 openDevice(mCameraIds[i]); 118 if (!mStaticInfo.isColorOutputSupported()) { 119 Log.i(TAG, "Camera " + mCameraIds[i] + 120 " does not support color outputs, skipping"); 121 continue; 122 } 123 124 if (!mStaticInfo.isVideoStabilizationSupported() && useVideoStab) { 125 Log.i(TAG, "Camera " + mCameraIds[i] + 126 " does not support video stabilization, skipping the stabilization" 127 + " test"); 128 continue; 129 } 130 131 initSupportedVideoSize(mCameraIds[i]); 132 133 basicRecordingTestByCamera(mCamcorderProfileList, useVideoStab); 134 } finally { 135 closeDevice(); 136 releaseRecorder(); 137 } 138 } 139 } 140 141 /** 142 * <p> 143 * Test basic video stabilitzation camera recording. 144 * </p> 145 * <p> 146 * This test covers the typical basic use case of camera recording with video 147 * stabilization is enabled, if video stabilization is supported. 148 * MediaRecorder is used to record the audio and video, CamcorderProfile is 149 * used to configure the MediaRecorder. It goes through the pre-defined 150 * CamcorderProfile list, test each profile configuration and validate the 151 * recorded video. Preview is set to the video size. 152 * </p> 153 */ 154 public void testBasicVideoStabilizationRecording() throws Exception { 155 doBasicRecording(/*useVideoStab*/true); 156 } 157 158 /** 159 * <p> 160 * Test basic camera recording. 161 * </p> 162 * <p> 163 * This test covers the typical basic use case of camera recording. 164 * MediaRecorder is used to record the audio and video, CamcorderProfile is 165 * used to configure the MediaRecorder. It goes through the pre-defined 166 * CamcorderProfile list, test each profile configuration and validate the 167 * recorded video. Preview is set to the video size. 168 * </p> 169 */ 170 public void testBasicRecording() throws Exception { 171 doBasicRecording(/*useVideoStab*/false); 172 } 173 174 /** 175 * <p> 176 * Test basic camera recording from a persistent input surface. 177 * </p> 178 * <p> 179 * This test is similar to testBasicRecording except that MediaRecorder records 180 * from a persistent input surface that's used across multiple recording sessions. 181 * </p> 182 */ 183 public void testRecordingFromPersistentSurface() throws Exception { 184 if (!MediaUtils.checkCodecForDomain(true /* encoder */, "video")) { 185 return; // skipped 186 } 187 mPersistentSurface = MediaCodec.createPersistentInputSurface(); 188 assertNotNull("Failed to create persistent input surface!", mPersistentSurface); 189 190 try { 191 doBasicRecording(/*useVideoStab*/false); 192 } finally { 193 mPersistentSurface.release(); 194 mPersistentSurface = null; 195 } 196 } 197 198 /** 199 * <p> 200 * Test camera recording for all supported sizes by using MediaRecorder. 201 * </p> 202 * <p> 203 * This test covers camera recording for all supported sizes by camera. MediaRecorder 204 * is used to encode the video. Preview is set to the video size. Recorded videos are 205 * validated according to the recording configuration. 206 * </p> 207 */ 208 public void testSupportedVideoSizes() throws Exception { 209 for (int i = 0; i < mCameraIds.length; i++) { 210 try { 211 Log.i(TAG, "Testing supported video size recording for camera " + mCameraIds[i]); 212 // Re-use the MediaRecorder object for the same camera device. 213 mMediaRecorder = new MediaRecorder(); 214 openDevice(mCameraIds[i]); 215 if (!mStaticInfo.isColorOutputSupported()) { 216 Log.i(TAG, "Camera " + mCameraIds[i] + 217 " does not support color outputs, skipping"); 218 continue; 219 } 220 initSupportedVideoSize(mCameraIds[i]); 221 222 recordingSizeTestByCamera(); 223 } finally { 224 closeDevice(); 225 releaseRecorder(); 226 } 227 } 228 } 229 230 /** 231 * Test different start/stop orders of Camera and Recorder. 232 * 233 * <p>The recording should be working fine for any kind of start/stop orders.</p> 234 */ 235 public void testCameraRecorderOrdering() { 236 // TODO: need implement 237 } 238 239 /** 240 * <p> 241 * Test camera recording for all supported sizes by using MediaCodec. 242 * </p> 243 * <p> 244 * This test covers video only recording for all supported sizes (camera and 245 * encoder). MediaCodec is used to encode the video. The recorded videos are 246 * validated according to the recording configuration. 247 * </p> 248 */ 249 public void testMediaCodecRecording() throws Exception { 250 // TODO. Need implement. 251 } 252 253 /** 254 * <p> 255 * Test video snapshot for each camera. 256 * </p> 257 * <p> 258 * This test covers video snapshot typical use case. The MediaRecorder is used to record the 259 * video for each available video size. The largest still capture size is selected to 260 * capture the JPEG image. The still capture images are validated according to the capture 261 * configuration. The timestamp of capture result before and after video snapshot is also 262 * checked to make sure no frame drop caused by video snapshot. 263 * </p> 264 */ 265 public void testVideoSnapshot() throws Exception { 266 videoSnapshotHelper(/*burstTest*/false); 267 } 268 269 /** 270 * <p> 271 * Test burst video snapshot for each camera. 272 * </p> 273 * <p> 274 * This test covers burst video snapshot capture. The MediaRecorder is used to record the 275 * video for each available video size. The largest still capture size is selected to 276 * capture the JPEG image. {@value #BURST_VIDEO_SNAPSHOT_NUM} video snapshot requests will be 277 * sent during the test. The still capture images are validated according to the capture 278 * configuration. 279 * </p> 280 */ 281 public void testBurstVideoSnapshot() throws Exception { 282 videoSnapshotHelper(/*burstTest*/true); 283 } 284 285 /** 286 * Test timelapse recording, where capture rate is slower than video (playback) frame rate. 287 */ 288 public void testTimelapseRecording() throws Exception { 289 // TODO. Need implement. 290 } 291 292 public void testSlowMotionRecording() throws Exception { 293 slowMotionRecording(); 294 } 295 296 public void testConstrainedHighSpeedRecording() throws Exception { 297 constrainedHighSpeedRecording(); 298 } 299 300 /** 301 * <p> 302 * Test recording framerate accuracy when switching from low FPS to high FPS. 303 * </p> 304 * <p> 305 * This test first record a video with profile of lowest framerate then record a video with 306 * profile of highest framerate. Make sure that the video framerate are still accurate. 307 * </p> 308 */ 309 public void testRecordingFramerateLowToHigh() throws Exception { 310 for (int i = 0; i < mCameraIds.length; i++) { 311 try { 312 Log.i(TAG, "Testing basic recording for camera " + mCameraIds[i]); 313 // Re-use the MediaRecorder object for the same camera device. 314 mMediaRecorder = new MediaRecorder(); 315 openDevice(mCameraIds[i]); 316 if (!mStaticInfo.isColorOutputSupported()) { 317 Log.i(TAG, "Camera " + mCameraIds[i] + 318 " does not support color outputs, skipping"); 319 continue; 320 } 321 initSupportedVideoSize(mCameraIds[i]); 322 323 int minFpsProfileId = -1, minFps = 1000; 324 int maxFpsProfileId = -1, maxFps = 0; 325 int cameraId = Integer.valueOf(mCamera.getId()); 326 327 for (int profileId : mCamcorderProfileList) { 328 if (!CamcorderProfile.hasProfile(cameraId, profileId)) { 329 continue; 330 } 331 CamcorderProfile profile = CamcorderProfile.get(cameraId, profileId); 332 if (profile.videoFrameRate < minFps) { 333 minFpsProfileId = profileId; 334 minFps = profile.videoFrameRate; 335 } 336 if (profile.videoFrameRate > maxFps) { 337 maxFpsProfileId = profileId; 338 maxFps = profile.videoFrameRate; 339 } 340 } 341 342 int camcorderProfileList[] = new int[] {minFpsProfileId, maxFpsProfileId}; 343 basicRecordingTestByCamera(camcorderProfileList, /*useVideoStab*/false); 344 } finally { 345 closeDevice(); 346 releaseRecorder(); 347 } 348 } 349 } 350 351 /** 352 * Test slow motion recording where capture rate (camera output) is different with 353 * video (playback) frame rate for each camera if high speed recording is supported 354 * by both camera and encoder. 355 * 356 * <p> 357 * Normal recording use cases make the capture rate (camera output frame 358 * rate) the same as the video (playback) frame rate. This guarantees that 359 * the motions in the scene play at the normal speed. If the capture rate is 360 * faster than video frame rate, for a given time duration, more number of 361 * frames are captured than it can be played in the same time duration. This 362 * generates "slow motion" effect during playback. 363 * </p> 364 */ 365 private void slowMotionRecording() throws Exception { 366 for (String id : mCameraIds) { 367 try { 368 Log.i(TAG, "Testing slow motion recording for camera " + id); 369 // Re-use the MediaRecorder object for the same camera device. 370 mMediaRecorder = new MediaRecorder(); 371 openDevice(id); 372 if (!mStaticInfo.isColorOutputSupported()) { 373 Log.i(TAG, "Camera " + id + 374 " does not support color outputs, skipping"); 375 continue; 376 } 377 if (!mStaticInfo.isHighSpeedVideoSupported()) { 378 continue; 379 } 380 381 StreamConfigurationMap config = 382 mStaticInfo.getValueFromKeyNonNull( 383 CameraCharacteristics.SCALER_STREAM_CONFIGURATION_MAP); 384 Size[] highSpeedVideoSizes = config.getHighSpeedVideoSizes(); 385 for (Size size : highSpeedVideoSizes) { 386 Range<Integer> fpsRange = getHighestHighSpeedFixedFpsRangeForSize(config, size); 387 mCollector.expectNotNull("Unable to find the fixed frame rate fps range for " + 388 "size " + size, fpsRange); 389 if (fpsRange == null) { 390 continue; 391 } 392 393 int captureRate = fpsRange.getLower(); 394 int videoFramerate = captureRate / SLOWMO_SLOW_FACTOR; 395 // Skip the test if the highest recording FPS supported by CamcorderProfile 396 if (fpsRange.getUpper() > getFpsFromHighSpeedProfileForSize(size)) { 397 Log.w(TAG, "high speed recording " + size + "@" + captureRate + "fps" 398 + " is not supported by CamcorderProfile"); 399 continue; 400 } 401 402 mOutMediaFileName = VIDEO_FILE_PATH + "/test_slowMo_video.mp4"; 403 if (DEBUG_DUMP) { 404 mOutMediaFileName = VIDEO_FILE_PATH + "/test_slowMo_video_" + id + "_" 405 + size.toString() + ".mp4"; 406 } 407 408 prepareRecording(size, videoFramerate, captureRate); 409 410 // prepare preview surface by using video size. 411 updatePreviewSurfaceWithVideo(size, captureRate); 412 413 // Start recording 414 SimpleCaptureCallback resultListener = new SimpleCaptureCallback(); 415 startSlowMotionRecording(/*useMediaRecorder*/true, videoFramerate, captureRate, 416 fpsRange, resultListener, /*useHighSpeedSession*/false); 417 418 // Record certain duration. 419 SystemClock.sleep(RECORDING_DURATION_MS); 420 421 // Stop recording and preview 422 stopRecording(/*useMediaRecorder*/true); 423 // Convert number of frames camera produced into the duration in unit of ms. 424 float frameDurationMs = 1000.0f / videoFramerate; 425 float durationMs = resultListener.getTotalNumFrames() * frameDurationMs; 426 427 // Validation. 428 validateRecording(size, durationMs, frameDurationMs, FRMDRP_RATE_TOLERANCE); 429 } 430 431 } finally { 432 closeDevice(); 433 releaseRecorder(); 434 } 435 } 436 } 437 438 private void constrainedHighSpeedRecording() throws Exception { 439 for (String id : mCameraIds) { 440 try { 441 Log.i(TAG, "Testing constrained high speed recording for camera " + id); 442 // Re-use the MediaRecorder object for the same camera device. 443 mMediaRecorder = new MediaRecorder(); 444 openDevice(id); 445 446 if (!mStaticInfo.isConstrainedHighSpeedVideoSupported()) { 447 Log.i(TAG, "Camera " + id + " doesn't support high speed recording, skipping."); 448 continue; 449 } 450 451 StreamConfigurationMap config = 452 mStaticInfo.getValueFromKeyNonNull( 453 CameraCharacteristics.SCALER_STREAM_CONFIGURATION_MAP); 454 Size[] highSpeedVideoSizes = config.getHighSpeedVideoSizes(); 455 for (Size size : highSpeedVideoSizes) { 456 List<Range<Integer>> fixedFpsRanges = 457 getHighSpeedFixedFpsRangeForSize(config, size); 458 mCollector.expectTrue("Unable to find the fixed frame rate fps range for " + 459 "size " + size, fixedFpsRanges.size() > 0); 460 // Test recording for each FPS range 461 for (Range<Integer> fpsRange : fixedFpsRanges) { 462 int captureRate = fpsRange.getLower(); 463 final int VIDEO_FRAME_RATE = 30; 464 // Skip the test if the highest recording FPS supported by CamcorderProfile 465 if (fpsRange.getUpper() > getFpsFromHighSpeedProfileForSize(size)) { 466 Log.w(TAG, "high speed recording " + size + "@" + captureRate + "fps" 467 + " is not supported by CamcorderProfile"); 468 continue; 469 } 470 471 mOutMediaFileName = VIDEO_FILE_PATH + "/test_cslowMo_video_" + captureRate + 472 "fps_" + id + "_" + size.toString() + ".mp4"; 473 474 prepareRecording(size, VIDEO_FRAME_RATE, captureRate); 475 476 // prepare preview surface by using video size. 477 updatePreviewSurfaceWithVideo(size, captureRate); 478 479 // Start recording 480 SimpleCaptureCallback resultListener = new SimpleCaptureCallback(); 481 startSlowMotionRecording(/*useMediaRecorder*/true, VIDEO_FRAME_RATE, 482 captureRate, fpsRange, resultListener, 483 /*useHighSpeedSession*/true); 484 485 // Record certain duration. 486 SystemClock.sleep(RECORDING_DURATION_MS); 487 488 // Stop recording and preview 489 stopRecording(/*useMediaRecorder*/true); 490 // Convert number of frames camera produced into the duration in unit of ms. 491 float frameDurationMs = 1000.0f / VIDEO_FRAME_RATE; 492 float durationMs = resultListener.getTotalNumFrames() * frameDurationMs; 493 494 // Validation. 495 validateRecording(size, durationMs, frameDurationMs, FRMDRP_RATE_TOLERANCE); 496 } 497 } 498 499 } finally { 500 closeDevice(); 501 releaseRecorder(); 502 } 503 } 504 } 505 506 /** 507 * Get high speed FPS from CamcorderProfiles for a given size. 508 * 509 * @param size The size used to search the CamcorderProfiles for the FPS. 510 * @return high speed video FPS, 0 if the given size is not supported by the CamcorderProfiles. 511 */ 512 private int getFpsFromHighSpeedProfileForSize(Size size) { 513 for (int quality = CamcorderProfile.QUALITY_HIGH_SPEED_480P; 514 quality <= CamcorderProfile.QUALITY_HIGH_SPEED_2160P; quality++) { 515 if (CamcorderProfile.hasProfile(quality)) { 516 CamcorderProfile profile = CamcorderProfile.get(quality); 517 if (size.equals(new Size(profile.videoFrameWidth, profile.videoFrameHeight))){ 518 return profile.videoFrameRate; 519 } 520 } 521 } 522 523 return 0; 524 } 525 526 private Range<Integer> getHighestHighSpeedFixedFpsRangeForSize(StreamConfigurationMap config, 527 Size size) { 528 Range<Integer>[] availableFpsRanges = config.getHighSpeedVideoFpsRangesFor(size); 529 Range<Integer> maxRange = availableFpsRanges[0]; 530 boolean foundRange = false; 531 for (Range<Integer> range : availableFpsRanges) { 532 if (range.getLower().equals(range.getUpper()) && range.getLower() >= maxRange.getLower()) { 533 foundRange = true; 534 maxRange = range; 535 } 536 } 537 538 if (!foundRange) { 539 return null; 540 } 541 return maxRange; 542 } 543 544 private List<Range<Integer>> getHighSpeedFixedFpsRangeForSize(StreamConfigurationMap config, 545 Size size) { 546 Range<Integer>[] availableFpsRanges = config.getHighSpeedVideoFpsRangesFor(size); 547 List<Range<Integer>> fixedRanges = new ArrayList<Range<Integer>>(); 548 for (Range<Integer> range : availableFpsRanges) { 549 if (range.getLower().equals(range.getUpper())) { 550 fixedRanges.add(range); 551 } 552 } 553 return fixedRanges; 554 } 555 556 private void startSlowMotionRecording(boolean useMediaRecorder, int videoFrameRate, 557 int captureRate, Range<Integer> fpsRange, 558 CameraCaptureSession.CaptureCallback listener, boolean useHighSpeedSession) throws Exception { 559 List<Surface> outputSurfaces = new ArrayList<Surface>(2); 560 assertTrue("Both preview and recording surfaces should be valid", 561 mPreviewSurface.isValid() && mRecordingSurface.isValid()); 562 outputSurfaces.add(mPreviewSurface); 563 outputSurfaces.add(mRecordingSurface); 564 // Video snapshot surface 565 if (mReaderSurface != null) { 566 outputSurfaces.add(mReaderSurface); 567 } 568 mSessionListener = new BlockingSessionCallback(); 569 mSession = configureCameraSession(mCamera, outputSurfaces, useHighSpeedSession, 570 mSessionListener, mHandler); 571 572 // Create slow motion request list 573 List<CaptureRequest> slowMoRequests = null; 574 if (useHighSpeedSession) { 575 CaptureRequest.Builder requestBuilder = 576 mCamera.createCaptureRequest(CameraDevice.TEMPLATE_RECORD); 577 requestBuilder.set(CaptureRequest.CONTROL_AE_TARGET_FPS_RANGE, fpsRange); 578 requestBuilder.addTarget(mPreviewSurface); 579 requestBuilder.addTarget(mRecordingSurface); 580 slowMoRequests = ((CameraConstrainedHighSpeedCaptureSession) mSession). 581 createHighSpeedRequestList(requestBuilder.build()); 582 } else { 583 CaptureRequest.Builder recordingRequestBuilder = 584 mCamera.createCaptureRequest(CameraDevice.TEMPLATE_RECORD); 585 recordingRequestBuilder.set(CaptureRequest.CONTROL_MODE, 586 CaptureRequest.CONTROL_MODE_USE_SCENE_MODE); 587 recordingRequestBuilder.set(CaptureRequest.CONTROL_SCENE_MODE, 588 CaptureRequest.CONTROL_SCENE_MODE_HIGH_SPEED_VIDEO); 589 590 CaptureRequest.Builder recordingOnlyBuilder = 591 mCamera.createCaptureRequest(CameraDevice.TEMPLATE_RECORD); 592 recordingOnlyBuilder.set(CaptureRequest.CONTROL_MODE, 593 CaptureRequest.CONTROL_MODE_USE_SCENE_MODE); 594 recordingOnlyBuilder.set(CaptureRequest.CONTROL_SCENE_MODE, 595 CaptureRequest.CONTROL_SCENE_MODE_HIGH_SPEED_VIDEO); 596 int slowMotionFactor = captureRate / videoFrameRate; 597 598 // Make sure camera output frame rate is set to correct value. 599 recordingRequestBuilder.set(CaptureRequest.CONTROL_AE_TARGET_FPS_RANGE, fpsRange); 600 recordingRequestBuilder.addTarget(mRecordingSurface); 601 recordingRequestBuilder.addTarget(mPreviewSurface); 602 recordingOnlyBuilder.set(CaptureRequest.CONTROL_AE_TARGET_FPS_RANGE, fpsRange); 603 recordingOnlyBuilder.addTarget(mRecordingSurface); 604 605 slowMoRequests = new ArrayList<CaptureRequest>(); 606 slowMoRequests.add(recordingRequestBuilder.build());// Preview + recording. 607 608 for (int i = 0; i < slowMotionFactor - 1; i++) { 609 slowMoRequests.add(recordingOnlyBuilder.build()); // Recording only. 610 } 611 } 612 613 mSession.setRepeatingBurst(slowMoRequests, listener, mHandler); 614 615 if (useMediaRecorder) { 616 mMediaRecorder.start(); 617 } else { 618 // TODO: need implement MediaCodec path. 619 } 620 621 } 622 623 /** 624 * Test camera recording by using each available CamcorderProfile for a 625 * given camera. preview size is set to the video size. 626 */ 627 private void basicRecordingTestByCamera(int[] camcorderProfileList, boolean useVideoStab) 628 throws Exception { 629 Size maxPreviewSize = mOrderedPreviewSizes.get(0); 630 List<Range<Integer> > fpsRanges = Arrays.asList( 631 mStaticInfo.getAeAvailableTargetFpsRangesChecked()); 632 int cameraId = Integer.valueOf(mCamera.getId()); 633 int maxVideoFrameRate = -1; 634 for (int profileId : camcorderProfileList) { 635 if (!CamcorderProfile.hasProfile(cameraId, profileId)) { 636 continue; 637 } 638 639 CamcorderProfile profile = CamcorderProfile.get(cameraId, profileId); 640 Size videoSz = new Size(profile.videoFrameWidth, profile.videoFrameHeight); 641 Range<Integer> fpsRange = new Range(profile.videoFrameRate, profile.videoFrameRate); 642 if (maxVideoFrameRate < profile.videoFrameRate) { 643 maxVideoFrameRate = profile.videoFrameRate; 644 } 645 646 if (allowedUnsupported(cameraId, profileId)) { 647 continue; 648 } 649 650 if (mStaticInfo.isHardwareLevelLegacy() && 651 (videoSz.getWidth() > maxPreviewSize.getWidth() || 652 videoSz.getHeight() > maxPreviewSize.getHeight())) { 653 // Skip. Legacy mode can only do recording up to max preview size 654 continue; 655 } 656 assertTrue("Video size " + videoSz.toString() + " for profile ID " + profileId + 657 " must be one of the camera device supported video size!", 658 mSupportedVideoSizes.contains(videoSz)); 659 assertTrue("Frame rate range " + fpsRange + " (for profile ID " + profileId + 660 ") must be one of the camera device available FPS range!", 661 fpsRanges.contains(fpsRange)); 662 663 if (VERBOSE) { 664 Log.v(TAG, "Testing camera recording with video size " + videoSz.toString()); 665 } 666 667 // Configure preview and recording surfaces. 668 mOutMediaFileName = VIDEO_FILE_PATH + "/test_video.mp4"; 669 if (DEBUG_DUMP) { 670 mOutMediaFileName = VIDEO_FILE_PATH + "/test_video_" + cameraId + "_" 671 + videoSz.toString() + ".mp4"; 672 } 673 674 prepareRecordingWithProfile(profile); 675 676 // prepare preview surface by using video size. 677 updatePreviewSurfaceWithVideo(videoSz, profile.videoFrameRate); 678 679 // Start recording 680 SimpleCaptureCallback resultListener = new SimpleCaptureCallback(); 681 startRecording(/* useMediaRecorder */true, resultListener, useVideoStab); 682 683 // Record certain duration. 684 SystemClock.sleep(RECORDING_DURATION_MS); 685 686 // Stop recording and preview 687 stopRecording(/* useMediaRecorder */true); 688 // Convert number of frames camera produced into the duration in unit of ms. 689 float frameDurationMs = 1000.0f / profile.videoFrameRate; 690 float durationMs = resultListener.getTotalNumFrames() * frameDurationMs; 691 692 if (VERBOSE) { 693 Log.v(TAG, "video frame rate: " + profile.videoFrameRate + 694 ", num of frames produced: " + resultListener.getTotalNumFrames()); 695 } 696 697 // Validation. 698 validateRecording(videoSz, durationMs, frameDurationMs, FRMDRP_RATE_TOLERANCE); 699 } 700 if (maxVideoFrameRate != -1) { 701 // At least one CamcorderProfile is present, check FPS 702 assertTrue("At least one CamcorderProfile must support >= 24 FPS", 703 maxVideoFrameRate >= 24); 704 } 705 } 706 707 /** 708 * Test camera recording for each supported video size by camera, preview 709 * size is set to the video size. 710 */ 711 private void recordingSizeTestByCamera() throws Exception { 712 for (Size sz : mSupportedVideoSizes) { 713 if (!isSupported(sz, VIDEO_FRAME_RATE, VIDEO_FRAME_RATE)) { 714 continue; 715 } 716 717 if (VERBOSE) { 718 Log.v(TAG, "Testing camera recording with video size " + sz.toString()); 719 } 720 721 // Configure preview and recording surfaces. 722 mOutMediaFileName = VIDEO_FILE_PATH + "/test_video.mp4"; 723 if (DEBUG_DUMP) { 724 mOutMediaFileName = VIDEO_FILE_PATH + "/test_video_" + mCamera.getId() + "_" 725 + sz.toString() + ".mp4"; 726 } 727 728 // Use AVC and AAC a/v compression format. 729 prepareRecording(sz, VIDEO_FRAME_RATE, VIDEO_FRAME_RATE); 730 731 // prepare preview surface by using video size. 732 updatePreviewSurfaceWithVideo(sz, VIDEO_FRAME_RATE); 733 734 // Start recording 735 SimpleCaptureCallback resultListener = new SimpleCaptureCallback(); 736 startRecording(/* useMediaRecorder */true, resultListener, /*useVideoStab*/false); 737 738 // Record certain duration. 739 SystemClock.sleep(RECORDING_DURATION_MS); 740 741 // Stop recording and preview 742 stopRecording(/* useMediaRecorder */true); 743 // Convert number of frames camera produced into the duration in unit of ms. 744 float frameDurationMs = 1000.0f / VIDEO_FRAME_RATE; 745 float durationMs = resultListener.getTotalNumFrames() * frameDurationMs; 746 747 // Validation. 748 validateRecording(sz, durationMs, frameDurationMs, FRMDRP_RATE_TOLERANCE); 749 } 750 } 751 752 /** 753 * Initialize the supported video sizes. 754 */ 755 private void initSupportedVideoSize(String cameraId) throws Exception { 756 Size maxVideoSize = SIZE_BOUND_1080P; 757 if (CamcorderProfile.hasProfile(CamcorderProfile.QUALITY_2160P)) { 758 maxVideoSize = SIZE_BOUND_2160P; 759 } 760 mSupportedVideoSizes = 761 getSupportedVideoSizes(cameraId, mCameraManager, maxVideoSize); 762 } 763 764 /** 765 * Simple wrapper to wrap normal/burst video snapshot tests 766 */ 767 private void videoSnapshotHelper(boolean burstTest) throws Exception { 768 for (String id : mCameraIds) { 769 try { 770 Log.i(TAG, "Testing video snapshot for camera " + id); 771 // Re-use the MediaRecorder object for the same camera device. 772 mMediaRecorder = new MediaRecorder(); 773 774 openDevice(id); 775 776 if (!mStaticInfo.isColorOutputSupported()) { 777 Log.i(TAG, "Camera " + id + 778 " does not support color outputs, skipping"); 779 continue; 780 } 781 782 initSupportedVideoSize(id); 783 784 videoSnapshotTestByCamera(burstTest); 785 } finally { 786 closeDevice(); 787 releaseRecorder(); 788 } 789 } 790 } 791 792 /** 793 * Returns {@code true} if the {@link CamcorderProfile} ID is allowed to be unsupported. 794 * 795 * <p>This only allows unsupported profiles when using the LEGACY mode of the Camera API.</p> 796 * 797 * @param profileId a {@link CamcorderProfile} ID to check. 798 * @return {@code true} if supported. 799 */ 800 private boolean allowedUnsupported(int cameraId, int profileId) { 801 if (!mStaticInfo.isHardwareLevelLegacy()) { 802 return false; 803 } 804 805 switch(profileId) { 806 case CamcorderProfile.QUALITY_2160P: 807 case CamcorderProfile.QUALITY_1080P: 808 case CamcorderProfile.QUALITY_HIGH: 809 return !CamcorderProfile.hasProfile(cameraId, profileId) || 810 CamcorderProfile.get(cameraId, profileId).videoFrameWidth >= 1080; 811 } 812 return false; 813 } 814 815 /** 816 * Test video snapshot for each available CamcorderProfile for a given camera. 817 * 818 * <p> 819 * Preview size is set to the video size. For the burst test, frame drop and jittering 820 * is not checked. 821 * </p> 822 * 823 * @param burstTest Perform burst capture or single capture. For burst capture 824 * {@value #BURST_VIDEO_SNAPSHOT_NUM} capture requests will be sent. 825 */ 826 private void videoSnapshotTestByCamera(boolean burstTest) 827 throws Exception { 828 final int NUM_SINGLE_SHOT_TEST = 5; 829 final int FRAMEDROP_TOLERANCE = 8; 830 final int FRAME_SIZE_15M = 15000000; 831 final float FRAME_DROP_TOLERENCE_FACTOR = 1.5f; 832 int kFrameDrop_Tolerence = FRAMEDROP_TOLERANCE; 833 834 for (int profileId : mCamcorderProfileList) { 835 int cameraId = Integer.valueOf(mCamera.getId()); 836 if (!CamcorderProfile.hasProfile(cameraId, profileId) || 837 allowedUnsupported(cameraId, profileId)) { 838 continue; 839 } 840 841 CamcorderProfile profile = CamcorderProfile.get(cameraId, profileId); 842 Size videoSz = new Size(profile.videoFrameWidth, profile.videoFrameHeight); 843 Size maxPreviewSize = mOrderedPreviewSizes.get(0); 844 845 if (mStaticInfo.isHardwareLevelLegacy() && 846 (videoSz.getWidth() > maxPreviewSize.getWidth() || 847 videoSz.getHeight() > maxPreviewSize.getHeight())) { 848 // Skip. Legacy mode can only do recording up to max preview size 849 continue; 850 } 851 852 if (!mSupportedVideoSizes.contains(videoSz)) { 853 mCollector.addMessage("Video size " + videoSz.toString() + " for profile ID " + 854 profileId + " must be one of the camera device supported video size!"); 855 continue; 856 } 857 858 // For LEGACY, find closest supported smaller or equal JPEG size to the current video 859 // size; if no size is smaller than the video, pick the smallest JPEG size. The assert 860 // for video size above guarantees that for LIMITED or FULL, we select videoSz here. 861 // Also check for minFrameDuration here to make sure jpeg stream won't slow down 862 // video capture 863 Size videoSnapshotSz = mOrderedStillSizes.get(mOrderedStillSizes.size() - 1); 864 // Allow a bit tolerance so we don't fail for a few nano seconds of difference 865 final float FRAME_DURATION_TOLERANCE = 0.01f; 866 long videoFrameDuration = (long) (1e9 / profile.videoFrameRate * 867 (1.0 + FRAME_DURATION_TOLERANCE)); 868 HashMap<Size, Long> minFrameDurationMap = mStaticInfo. 869 getAvailableMinFrameDurationsForFormatChecked(ImageFormat.JPEG); 870 for (int i = mOrderedStillSizes.size() - 2; i >= 0; i--) { 871 Size candidateSize = mOrderedStillSizes.get(i); 872 if (mStaticInfo.isHardwareLevelLegacy()) { 873 // Legacy level doesn't report min frame duration 874 if (candidateSize.getWidth() <= videoSz.getWidth() && 875 candidateSize.getHeight() <= videoSz.getHeight()) { 876 videoSnapshotSz = candidateSize; 877 } 878 } else { 879 Long jpegFrameDuration = minFrameDurationMap.get(candidateSize); 880 assertTrue("Cannot find minimum frame duration for jpeg size " + candidateSize, 881 jpegFrameDuration != null); 882 if (candidateSize.getWidth() <= videoSz.getWidth() && 883 candidateSize.getHeight() <= videoSz.getHeight() && 884 jpegFrameDuration <= videoFrameDuration) { 885 videoSnapshotSz = candidateSize; 886 } 887 } 888 } 889 Size defaultvideoSnapshotSz = videoSnapshotSz; 890 891 /** 892 * Only test full res snapshot when below conditions are all true. 893 * 1. Camera is at least a LIMITED device. 894 * 2. video size is up to max preview size, which will be bounded by 1080p. 895 * 3. Full resolution jpeg stream can keep up to video stream speed. 896 * When full res jpeg stream cannot keep up to video stream speed, search 897 * the largest jpeg size that can susptain video speed instead. 898 */ 899 if (mStaticInfo.isHardwareLevelAtLeastLimited() && 900 videoSz.getWidth() <= maxPreviewSize.getWidth() && 901 videoSz.getHeight() <= maxPreviewSize.getHeight()) { 902 for (Size jpegSize : mOrderedStillSizes) { 903 Long jpegFrameDuration = minFrameDurationMap.get(jpegSize); 904 assertTrue("Cannot find minimum frame duration for jpeg size " + jpegSize, 905 jpegFrameDuration != null); 906 if (jpegFrameDuration <= videoFrameDuration) { 907 videoSnapshotSz = jpegSize; 908 break; 909 } 910 if (jpegSize.equals(videoSz)) { 911 throw new AssertionFailedError( 912 "Cannot find adequate video snapshot size for video size" + 913 videoSz); 914 } 915 } 916 } 917 918 Log.i(TAG, "Testing video snapshot size " + videoSnapshotSz + 919 " for video size " + videoSz); 920 if (videoSnapshotSz.getWidth() * videoSnapshotSz.getHeight() > FRAME_SIZE_15M) 921 kFrameDrop_Tolerence = (int)(FRAMEDROP_TOLERANCE * FRAME_DROP_TOLERENCE_FACTOR); 922 923 createImageReader( 924 videoSnapshotSz, ImageFormat.JPEG, 925 MAX_VIDEO_SNAPSHOT_IMAGES, /*listener*/null); 926 927 // Full or better devices should support whatever video snapshot size calculated above. 928 // Limited devices may only be able to support the default one. 929 if (mStaticInfo.isHardwareLevelLimited()) { 930 List<Surface> outputs = new ArrayList<Surface>(); 931 outputs.add(mPreviewSurface); 932 outputs.add(mRecordingSurface); 933 outputs.add(mReaderSurface); 934 boolean isSupported = isStreamConfigurationSupported( 935 mCamera, outputs, mSessionListener, mHandler); 936 if (!isSupported) { 937 videoSnapshotSz = defaultvideoSnapshotSz; 938 createImageReader( 939 videoSnapshotSz, ImageFormat.JPEG, 940 MAX_VIDEO_SNAPSHOT_IMAGES, /*listener*/null); 941 } 942 } 943 944 if (VERBOSE) { 945 Log.v(TAG, "Testing camera recording with video size " + videoSz.toString()); 946 } 947 948 // Configure preview and recording surfaces. 949 mOutMediaFileName = VIDEO_FILE_PATH + "/test_video.mp4"; 950 if (DEBUG_DUMP) { 951 mOutMediaFileName = VIDEO_FILE_PATH + "/test_video_" + cameraId + "_" 952 + videoSz.toString() + ".mp4"; 953 } 954 955 int numTestIterations = burstTest ? 1 : NUM_SINGLE_SHOT_TEST; 956 int totalDroppedFrames = 0; 957 958 for (int numTested = 0; numTested < numTestIterations; numTested++) { 959 prepareRecordingWithProfile(profile); 960 961 // prepare video snapshot 962 SimpleCaptureCallback resultListener = new SimpleCaptureCallback(); 963 SimpleImageReaderListener imageListener = new SimpleImageReaderListener(); 964 CaptureRequest.Builder videoSnapshotRequestBuilder = 965 mCamera.createCaptureRequest((mStaticInfo.isHardwareLevelLegacy()) ? 966 CameraDevice.TEMPLATE_RECORD : 967 CameraDevice.TEMPLATE_VIDEO_SNAPSHOT); 968 969 // prepare preview surface by using video size. 970 updatePreviewSurfaceWithVideo(videoSz, profile.videoFrameRate); 971 972 prepareVideoSnapshot(videoSnapshotRequestBuilder, imageListener); 973 CaptureRequest request = videoSnapshotRequestBuilder.build(); 974 975 // Start recording 976 startRecording(/* useMediaRecorder */true, resultListener, 977 /*useVideoStab*/mStaticInfo.isVideoStabilizationSupported()); 978 long startTime = SystemClock.elapsedRealtime(); 979 980 // Record certain duration. 981 SystemClock.sleep(RECORDING_DURATION_MS / 2); 982 983 // take video snapshot 984 if (burstTest) { 985 List<CaptureRequest> requests = 986 new ArrayList<CaptureRequest>(BURST_VIDEO_SNAPSHOT_NUM); 987 for (int i = 0; i < BURST_VIDEO_SNAPSHOT_NUM; i++) { 988 requests.add(request); 989 } 990 mSession.captureBurst(requests, resultListener, mHandler); 991 } else { 992 mSession.capture(request, resultListener, mHandler); 993 } 994 995 // make sure recording is still going after video snapshot 996 SystemClock.sleep(RECORDING_DURATION_MS / 2); 997 998 // Stop recording and preview 999 float durationMs = (float) stopRecording(/* useMediaRecorder */true); 1000 // For non-burst test, use number of frames to also double check video frame rate. 1001 // Burst video snapshot is allowed to cause frame rate drop, so do not use number 1002 // of frames to estimate duration 1003 if (!burstTest) { 1004 durationMs = resultListener.getTotalNumFrames() * 1000.0f / 1005 profile.videoFrameRate; 1006 } 1007 1008 float frameDurationMs = 1000.0f / profile.videoFrameRate; 1009 // Validation recorded video 1010 validateRecording(videoSz, durationMs, 1011 frameDurationMs, VID_SNPSHT_FRMDRP_RATE_TOLERANCE); 1012 1013 if (burstTest) { 1014 for (int i = 0; i < BURST_VIDEO_SNAPSHOT_NUM; i++) { 1015 Image image = imageListener.getImage(CAPTURE_IMAGE_TIMEOUT_MS); 1016 validateVideoSnapshotCapture(image, videoSnapshotSz); 1017 image.close(); 1018 } 1019 } else { 1020 // validate video snapshot image 1021 Image image = imageListener.getImage(CAPTURE_IMAGE_TIMEOUT_MS); 1022 validateVideoSnapshotCapture(image, videoSnapshotSz); 1023 1024 // validate if there is framedrop around video snapshot 1025 totalDroppedFrames += validateFrameDropAroundVideoSnapshot( 1026 resultListener, image.getTimestamp()); 1027 1028 //TODO: validate jittering. Should move to PTS 1029 //validateJittering(resultListener); 1030 1031 image.close(); 1032 } 1033 } 1034 1035 if (!burstTest) { 1036 Log.w(TAG, String.format("Camera %d Video size %s: Number of dropped frames " + 1037 "detected in %d trials is %d frames.", cameraId, videoSz.toString(), 1038 numTestIterations, totalDroppedFrames)); 1039 mCollector.expectLessOrEqual( 1040 String.format( 1041 "Camera %d Video size %s: Number of dropped frames %d must not" 1042 + " be larger than %d", 1043 cameraId, videoSz.toString(), totalDroppedFrames, 1044 kFrameDrop_Tolerence), 1045 kFrameDrop_Tolerence, totalDroppedFrames); 1046 } 1047 closeImageReader(); 1048 } 1049 } 1050 1051 /** 1052 * Configure video snapshot request according to the still capture size 1053 */ 1054 private void prepareVideoSnapshot( 1055 CaptureRequest.Builder requestBuilder, 1056 ImageReader.OnImageAvailableListener imageListener) 1057 throws Exception { 1058 mReader.setOnImageAvailableListener(imageListener, mHandler); 1059 assertNotNull("Recording surface must be non-null!", mRecordingSurface); 1060 requestBuilder.addTarget(mRecordingSurface); 1061 assertNotNull("Preview surface must be non-null!", mPreviewSurface); 1062 requestBuilder.addTarget(mPreviewSurface); 1063 assertNotNull("Reader surface must be non-null!", mReaderSurface); 1064 requestBuilder.addTarget(mReaderSurface); 1065 } 1066 1067 /** 1068 * Update preview size with video size. 1069 * 1070 * <p>Preview size will be capped with max preview size.</p> 1071 * 1072 * @param videoSize The video size used for preview. 1073 * @param videoFrameRate The video frame rate 1074 * 1075 */ 1076 private void updatePreviewSurfaceWithVideo(Size videoSize, int videoFrameRate) { 1077 if (mOrderedPreviewSizes == null) { 1078 throw new IllegalStateException("supported preview size list is not initialized yet"); 1079 } 1080 final float FRAME_DURATION_TOLERANCE = 0.01f; 1081 long videoFrameDuration = (long) (1e9 / videoFrameRate * 1082 (1.0 + FRAME_DURATION_TOLERANCE)); 1083 HashMap<Size, Long> minFrameDurationMap = mStaticInfo. 1084 getAvailableMinFrameDurationsForFormatChecked(ImageFormat.PRIVATE); 1085 Size maxPreviewSize = mOrderedPreviewSizes.get(0); 1086 Size previewSize = null; 1087 if (videoSize.getWidth() > maxPreviewSize.getWidth() || 1088 videoSize.getHeight() > maxPreviewSize.getHeight()) { 1089 for (Size s : mOrderedPreviewSizes) { 1090 Long frameDuration = minFrameDurationMap.get(s); 1091 if (mStaticInfo.isHardwareLevelLegacy()) { 1092 // Legacy doesn't report min frame duration 1093 frameDuration = new Long(0); 1094 } 1095 assertTrue("Cannot find minimum frame duration for private size" + s, 1096 frameDuration != null); 1097 if (frameDuration <= videoFrameDuration && 1098 s.getWidth() <= videoSize.getWidth() && 1099 s.getHeight() <= videoSize.getHeight()) { 1100 Log.w(TAG, "Overwrite preview size from " + videoSize.toString() + 1101 " to " + s.toString()); 1102 previewSize = s; 1103 break; 1104 // If all preview size doesn't work then we fallback to video size 1105 } 1106 } 1107 } 1108 if (previewSize == null) { 1109 previewSize = videoSize; 1110 } 1111 updatePreviewSurface(previewSize); 1112 } 1113 1114 /** 1115 * Configure MediaRecorder recording session with CamcorderProfile, prepare 1116 * the recording surface. 1117 */ 1118 private void prepareRecordingWithProfile(CamcorderProfile profile) 1119 throws Exception { 1120 // Prepare MediaRecorder. 1121 mMediaRecorder.setAudioSource(MediaRecorder.AudioSource.CAMCORDER); 1122 mMediaRecorder.setVideoSource(MediaRecorder.VideoSource.SURFACE); 1123 mMediaRecorder.setProfile(profile); 1124 mMediaRecorder.setOutputFile(mOutMediaFileName); 1125 if (mPersistentSurface != null) { 1126 mMediaRecorder.setInputSurface(mPersistentSurface); 1127 mRecordingSurface = mPersistentSurface; 1128 } 1129 mMediaRecorder.prepare(); 1130 if (mPersistentSurface == null) { 1131 mRecordingSurface = mMediaRecorder.getSurface(); 1132 } 1133 assertNotNull("Recording surface must be non-null!", mRecordingSurface); 1134 mVideoFrameRate = profile.videoFrameRate; 1135 mVideoSize = new Size(profile.videoFrameWidth, profile.videoFrameHeight); 1136 } 1137 1138 /** 1139 * Configure MediaRecorder recording session with CamcorderProfile, prepare 1140 * the recording surface. Use AVC for video compression, AAC for audio compression. 1141 * Both are required for android devices by android CDD. 1142 */ 1143 private void prepareRecording(Size sz, int videoFrameRate, int captureRate) 1144 throws Exception { 1145 // Prepare MediaRecorder. 1146 mMediaRecorder.setAudioSource(MediaRecorder.AudioSource.CAMCORDER); 1147 mMediaRecorder.setVideoSource(MediaRecorder.VideoSource.SURFACE); 1148 mMediaRecorder.setOutputFormat(MediaRecorder.OutputFormat.THREE_GPP); 1149 mMediaRecorder.setOutputFile(mOutMediaFileName); 1150 mMediaRecorder.setVideoEncodingBitRate(getVideoBitRate(sz)); 1151 mMediaRecorder.setVideoFrameRate(videoFrameRate); 1152 mMediaRecorder.setCaptureRate(captureRate); 1153 mMediaRecorder.setVideoSize(sz.getWidth(), sz.getHeight()); 1154 mMediaRecorder.setVideoEncoder(MediaRecorder.VideoEncoder.H264); 1155 mMediaRecorder.setAudioEncoder(MediaRecorder.AudioEncoder.AAC); 1156 if (mPersistentSurface != null) { 1157 mMediaRecorder.setInputSurface(mPersistentSurface); 1158 mRecordingSurface = mPersistentSurface; 1159 } 1160 mMediaRecorder.prepare(); 1161 if (mPersistentSurface == null) { 1162 mRecordingSurface = mMediaRecorder.getSurface(); 1163 } 1164 assertNotNull("Recording surface must be non-null!", mRecordingSurface); 1165 mVideoFrameRate = videoFrameRate; 1166 mVideoSize = sz; 1167 } 1168 1169 private void startRecording(boolean useMediaRecorder, 1170 CameraCaptureSession.CaptureCallback listener, boolean useVideoStab) throws Exception { 1171 if (!mStaticInfo.isVideoStabilizationSupported() && useVideoStab) { 1172 throw new IllegalArgumentException("Video stabilization is not supported"); 1173 } 1174 1175 List<Surface> outputSurfaces = new ArrayList<Surface>(2); 1176 assertTrue("Both preview and recording surfaces should be valid", 1177 mPreviewSurface.isValid() && mRecordingSurface.isValid()); 1178 outputSurfaces.add(mPreviewSurface); 1179 outputSurfaces.add(mRecordingSurface); 1180 // Video snapshot surface 1181 if (mReaderSurface != null) { 1182 outputSurfaces.add(mReaderSurface); 1183 } 1184 mSessionListener = new BlockingSessionCallback(); 1185 mSession = configureCameraSession(mCamera, outputSurfaces, mSessionListener, mHandler); 1186 1187 CaptureRequest.Builder recordingRequestBuilder = 1188 mCamera.createCaptureRequest(CameraDevice.TEMPLATE_RECORD); 1189 // Make sure camera output frame rate is set to correct value. 1190 Range<Integer> fpsRange = Range.create(mVideoFrameRate, mVideoFrameRate); 1191 recordingRequestBuilder.set(CaptureRequest.CONTROL_AE_TARGET_FPS_RANGE, fpsRange); 1192 if (useVideoStab) { 1193 recordingRequestBuilder.set(CaptureRequest.CONTROL_VIDEO_STABILIZATION_MODE, 1194 CaptureRequest.CONTROL_VIDEO_STABILIZATION_MODE_ON); 1195 } 1196 recordingRequestBuilder.addTarget(mRecordingSurface); 1197 recordingRequestBuilder.addTarget(mPreviewSurface); 1198 mSession.setRepeatingRequest(recordingRequestBuilder.build(), listener, mHandler); 1199 1200 if (useMediaRecorder) { 1201 mMediaRecorder.start(); 1202 } else { 1203 // TODO: need implement MediaCodec path. 1204 } 1205 mRecordingStartTime = SystemClock.elapsedRealtime(); 1206 } 1207 1208 private void stopCameraStreaming() throws Exception { 1209 if (VERBOSE) { 1210 Log.v(TAG, "Stopping camera streaming and waiting for idle"); 1211 } 1212 // Stop repeating, wait for captures to complete, and disconnect from 1213 // surfaces 1214 mSession.close(); 1215 mSessionListener.getStateWaiter().waitForState(SESSION_CLOSED, SESSION_CLOSE_TIMEOUT_MS); 1216 } 1217 1218 // Stop recording and return the estimated video duration in milliseconds. 1219 private int stopRecording(boolean useMediaRecorder) throws Exception { 1220 long stopRecordingTime = SystemClock.elapsedRealtime(); 1221 if (useMediaRecorder) { 1222 stopCameraStreaming(); 1223 1224 mMediaRecorder.stop(); 1225 // Can reuse the MediaRecorder object after reset. 1226 mMediaRecorder.reset(); 1227 } else { 1228 // TODO: need implement MediaCodec path. 1229 } 1230 if (mPersistentSurface == null && mRecordingSurface != null) { 1231 mRecordingSurface.release(); 1232 mRecordingSurface = null; 1233 } 1234 return (int) (stopRecordingTime - mRecordingStartTime); 1235 } 1236 1237 private void releaseRecorder() { 1238 if (mMediaRecorder != null) { 1239 mMediaRecorder.release(); 1240 mMediaRecorder = null; 1241 } 1242 } 1243 1244 private void validateRecording( 1245 Size sz, float expectedDurationMs, float expectedFrameDurationMs, 1246 float frameDropTolerance) throws Exception { 1247 File outFile = new File(mOutMediaFileName); 1248 assertTrue("No video is recorded", outFile.exists()); 1249 float maxFrameDuration = expectedFrameDurationMs * (1.0f + FRAMEDURATION_MARGIN); 1250 MediaExtractor extractor = new MediaExtractor(); 1251 try { 1252 extractor.setDataSource(mOutMediaFileName); 1253 long durationUs = 0; 1254 int width = -1, height = -1; 1255 int numTracks = extractor.getTrackCount(); 1256 int selectedTrack = -1; 1257 final String VIDEO_MIME_TYPE = "video"; 1258 for (int i = 0; i < numTracks; i++) { 1259 MediaFormat format = extractor.getTrackFormat(i); 1260 String mime = format.getString(MediaFormat.KEY_MIME); 1261 if (mime.contains(VIDEO_MIME_TYPE)) { 1262 Log.i(TAG, "video format is: " + format.toString()); 1263 durationUs = format.getLong(MediaFormat.KEY_DURATION); 1264 width = format.getInteger(MediaFormat.KEY_WIDTH); 1265 height = format.getInteger(MediaFormat.KEY_HEIGHT); 1266 selectedTrack = i; 1267 extractor.selectTrack(i); 1268 break; 1269 } 1270 } 1271 if (selectedTrack < 0) { 1272 throw new AssertionFailedError( 1273 "Cannot find video track!"); 1274 } 1275 1276 Size videoSz = new Size(width, height); 1277 assertTrue("Video size doesn't match, expected " + sz.toString() + 1278 " got " + videoSz.toString(), videoSz.equals(sz)); 1279 float duration = (float) (durationUs / 1000); 1280 if (VERBOSE) { 1281 Log.v(TAG, String.format("Video duration: recorded %fms, expected %fms", 1282 duration, expectedDurationMs)); 1283 } 1284 1285 // TODO: Don't skip this for video snapshot 1286 if (!mStaticInfo.isHardwareLevelLegacy()) { 1287 assertTrue(String.format( 1288 "Camera %s: Video duration doesn't match: recorded %fms, expected %fms.", 1289 mCamera.getId(), duration, expectedDurationMs), 1290 Math.abs(duration - expectedDurationMs) < 1291 DURATION_MARGIN * expectedDurationMs); 1292 } 1293 1294 // Check for framedrop 1295 long lastSampleUs = 0; 1296 int frameDropCount = 0; 1297 int expectedFrameCount = (int) (expectedDurationMs / expectedFrameDurationMs); 1298 ArrayList<Long> timestamps = new ArrayList<Long>(expectedFrameCount); 1299 while (true) { 1300 timestamps.add(extractor.getSampleTime()); 1301 if (!extractor.advance()) { 1302 break; 1303 } 1304 } 1305 Collections.sort(timestamps); 1306 long prevSampleUs = timestamps.get(0); 1307 for (int i = 1; i < timestamps.size(); i++) { 1308 long currentSampleUs = timestamps.get(i); 1309 float frameDurationMs = (float) (currentSampleUs - prevSampleUs) / 1000; 1310 if (frameDurationMs > maxFrameDuration) { 1311 Log.w(TAG, String.format( 1312 "Frame drop at %d: expectation %f, observed %f", 1313 i, expectedFrameDurationMs, frameDurationMs)); 1314 frameDropCount++; 1315 } 1316 prevSampleUs = currentSampleUs; 1317 } 1318 float frameDropRate = 100.f * frameDropCount / expectedFrameCount; 1319 Log.i(TAG, String.format("Frame drop rate %d/%d (%f%%)", 1320 frameDropCount, expectedFrameCount, frameDropRate)); 1321 assertTrue(String.format( 1322 "Camera %s: Video frame drop rate too high: %f%%, tolerance %f%%.", 1323 mCamera.getId(), frameDropRate, frameDropTolerance), 1324 frameDropRate < frameDropTolerance); 1325 } finally { 1326 extractor.release(); 1327 if (!DEBUG_DUMP) { 1328 outFile.delete(); 1329 } 1330 } 1331 } 1332 1333 /** 1334 * Validate video snapshot capture image object sanity and test. 1335 * 1336 * <p> Check for size, format and jpeg decoding</p> 1337 * 1338 * @param image The JPEG image to be verified. 1339 * @param size The JPEG capture size to be verified against. 1340 */ 1341 private void validateVideoSnapshotCapture(Image image, Size size) { 1342 CameraTestUtils.validateImage(image, size.getWidth(), size.getHeight(), 1343 ImageFormat.JPEG, /*filePath*/null); 1344 } 1345 1346 /** 1347 * Validate if video snapshot causes frame drop. 1348 * Here frame drop is defined as frame duration >= 2 * expected frame duration. 1349 * Return the estimated number of frames dropped during video snapshot 1350 */ 1351 private int validateFrameDropAroundVideoSnapshot( 1352 SimpleCaptureCallback resultListener, long imageTimeStamp) { 1353 double expectedDurationMs = 1000.0 / mVideoFrameRate; 1354 CaptureResult prevResult = resultListener.getCaptureResult(WAIT_FOR_RESULT_TIMEOUT_MS); 1355 long prevTS = getValueNotNull(prevResult, CaptureResult.SENSOR_TIMESTAMP); 1356 while (resultListener.hasMoreResults()) { 1357 CaptureResult currentResult = 1358 resultListener.getCaptureResult(WAIT_FOR_RESULT_TIMEOUT_MS); 1359 long currentTS = getValueNotNull(currentResult, CaptureResult.SENSOR_TIMESTAMP); 1360 if (currentTS == imageTimeStamp) { 1361 // validate the timestamp before and after, then return 1362 CaptureResult nextResult = 1363 resultListener.getCaptureResult(WAIT_FOR_RESULT_TIMEOUT_MS); 1364 long nextTS = getValueNotNull(nextResult, CaptureResult.SENSOR_TIMESTAMP); 1365 double durationMs = (currentTS - prevTS) / 1000000.0; 1366 int totalFramesDropped = 0; 1367 1368 // Snapshots in legacy mode pause the preview briefly. Skip the duration 1369 // requirements for legacy mode unless this is fixed. 1370 if (!mStaticInfo.isHardwareLevelLegacy()) { 1371 mCollector.expectTrue( 1372 String.format( 1373 "Video %dx%d Frame drop detected before video snapshot: " + 1374 "duration %.2fms (expected %.2fms)", 1375 mVideoSize.getWidth(), mVideoSize.getHeight(), 1376 durationMs, expectedDurationMs 1377 ), 1378 durationMs <= (expectedDurationMs * MAX_NUM_FRAME_DROP_INTERVAL_ALLOWED) 1379 ); 1380 // Log a warning is there is any frame drop detected. 1381 if (durationMs >= expectedDurationMs * 2) { 1382 Log.w(TAG, String.format( 1383 "Video %dx%d Frame drop detected before video snapshot: " + 1384 "duration %.2fms (expected %.2fms)", 1385 mVideoSize.getWidth(), mVideoSize.getHeight(), 1386 durationMs, expectedDurationMs 1387 )); 1388 } 1389 1390 durationMs = (nextTS - currentTS) / 1000000.0; 1391 mCollector.expectTrue( 1392 String.format( 1393 "Video %dx%d Frame drop detected after video snapshot: " + 1394 "duration %.2fms (expected %.2fms)", 1395 mVideoSize.getWidth(), mVideoSize.getHeight(), 1396 durationMs, expectedDurationMs 1397 ), 1398 durationMs <= (expectedDurationMs * MAX_NUM_FRAME_DROP_INTERVAL_ALLOWED) 1399 ); 1400 // Log a warning is there is any frame drop detected. 1401 if (durationMs >= expectedDurationMs * 2) { 1402 Log.w(TAG, String.format( 1403 "Video %dx%d Frame drop detected after video snapshot: " + 1404 "duration %fms (expected %fms)", 1405 mVideoSize.getWidth(), mVideoSize.getHeight(), 1406 durationMs, expectedDurationMs 1407 )); 1408 } 1409 1410 double totalDurationMs = (nextTS - prevTS) / 1000000.0; 1411 // Minus 2 for the expected 2 frames interval 1412 totalFramesDropped = (int) (totalDurationMs / expectedDurationMs) - 2; 1413 if (totalFramesDropped < 0) { 1414 Log.w(TAG, "totalFrameDropped is " + totalFramesDropped + 1415 ". Video frame rate might be too fast."); 1416 } 1417 totalFramesDropped = Math.max(0, totalFramesDropped); 1418 } 1419 return totalFramesDropped; 1420 } 1421 prevTS = currentTS; 1422 } 1423 throw new AssertionFailedError( 1424 "Video snapshot timestamp does not match any of capture results!"); 1425 } 1426 1427 /** 1428 * Validate frame jittering from the input simple listener's buffered results 1429 */ 1430 private void validateJittering(SimpleCaptureCallback resultListener) { 1431 double expectedDurationMs = 1000.0 / mVideoFrameRate; 1432 CaptureResult prevResult = resultListener.getCaptureResult(WAIT_FOR_RESULT_TIMEOUT_MS); 1433 long prevTS = getValueNotNull(prevResult, CaptureResult.SENSOR_TIMESTAMP); 1434 while (resultListener.hasMoreResults()) { 1435 CaptureResult currentResult = 1436 resultListener.getCaptureResult(WAIT_FOR_RESULT_TIMEOUT_MS); 1437 long currentTS = getValueNotNull(currentResult, CaptureResult.SENSOR_TIMESTAMP); 1438 double durationMs = (currentTS - prevTS) / 1000000.0; 1439 double durationError = Math.abs(durationMs - expectedDurationMs); 1440 long frameNumber = currentResult.getFrameNumber(); 1441 mCollector.expectTrue( 1442 String.format( 1443 "Resolution %dx%d Frame %d: jittering (%.2fms) exceeds bound [%.2fms,%.2fms]", 1444 mVideoSize.getWidth(), mVideoSize.getHeight(), 1445 frameNumber, durationMs, 1446 expectedDurationMs - FRAME_DURATION_ERROR_TOLERANCE_MS, 1447 expectedDurationMs + FRAME_DURATION_ERROR_TOLERANCE_MS), 1448 durationError <= FRAME_DURATION_ERROR_TOLERANCE_MS); 1449 prevTS = currentTS; 1450 } 1451 } 1452 1453 /** 1454 * Calculate a video bit rate based on the size. The bit rate is scaled 1455 * based on ratio of video size to 1080p size. 1456 */ 1457 private int getVideoBitRate(Size sz) { 1458 int rate = BIT_RATE_1080P; 1459 float scaleFactor = sz.getHeight() * sz.getWidth() / (float)(1920 * 1080); 1460 rate = (int)(rate * scaleFactor); 1461 1462 // Clamp to the MIN, MAX range. 1463 return Math.max(BIT_RATE_MIN, Math.min(BIT_RATE_MAX, rate)); 1464 } 1465 1466 /** 1467 * Check if the encoder and camera are able to support this size and frame rate. 1468 * Assume the video compression format is AVC. 1469 */ 1470 private boolean isSupported(Size sz, int captureRate, int encodingRate) throws Exception { 1471 // Check camera capability. 1472 if (!isSupportedByCamera(sz, captureRate)) { 1473 return false; 1474 } 1475 1476 // Check encode capability. 1477 if (!isSupportedByAVCEncoder(sz, encodingRate)){ 1478 return false; 1479 } 1480 1481 if(VERBOSE) { 1482 Log.v(TAG, "Both encoder and camera support " + sz.toString() + "@" + encodingRate + "@" 1483 + getVideoBitRate(sz) / 1000 + "Kbps"); 1484 } 1485 1486 return true; 1487 } 1488 1489 private boolean isSupportedByCamera(Size sz, int frameRate) { 1490 // Check if camera can support this sz and frame rate combination. 1491 StreamConfigurationMap config = mStaticInfo. 1492 getValueFromKeyNonNull(CameraCharacteristics.SCALER_STREAM_CONFIGURATION_MAP); 1493 1494 long minDuration = config.getOutputMinFrameDuration(MediaRecorder.class, sz); 1495 if (minDuration == 0) { 1496 return false; 1497 } 1498 1499 int maxFrameRate = (int) (1e9f / minDuration); 1500 return maxFrameRate >= frameRate; 1501 } 1502 1503 /** 1504 * Check if encoder can support this size and frame rate combination by querying 1505 * MediaCodec capability. Check is based on size and frame rate. Ignore the bit rate 1506 * as the bit rates targeted in this test are well below the bit rate max value specified 1507 * by AVC specification for certain level. 1508 */ 1509 private static boolean isSupportedByAVCEncoder(Size sz, int frameRate) { 1510 MediaFormat format = MediaFormat.createVideoFormat( 1511 MediaFormat.MIMETYPE_VIDEO_AVC, sz.getWidth(), sz.getHeight()); 1512 format.setInteger(MediaFormat.KEY_FRAME_RATE, frameRate); 1513 MediaCodecList mcl = new MediaCodecList(MediaCodecList.REGULAR_CODECS); 1514 return mcl.findEncoderForFormat(format) != null; 1515 } 1516 } 1517