1 /* 2 * Copyright (C) 2014 The Android Open Source Project Licensed under the Apache 3 * License, Version 2.0 (the "License"); you may not use this file except in 4 * compliance with the License. You may obtain a copy of the License at 5 * http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law 6 * or agreed to in writing, software distributed under the License is 7 * distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 8 * KIND, either express or implied. See the License for the specific language 9 * governing permissions and limitations under the License. 10 */ 11 12 package android.hardware.camera2.cts; 13 14 import static android.hardware.camera2.cts.CameraTestUtils.*; 15 import static com.android.ex.camera2.blocking.BlockingSessionCallback.*; 16 17 import android.graphics.ImageFormat; 18 import android.hardware.camera2.CameraCharacteristics; 19 import android.hardware.camera2.CameraCaptureSession; 20 import android.hardware.camera2.CameraDevice; 21 import android.hardware.camera2.CaptureRequest; 22 import android.hardware.camera2.CaptureResult; 23 import android.hardware.camera2.params.StreamConfigurationMap; 24 import android.util.Size; 25 import android.hardware.camera2.cts.testcases.Camera2SurfaceViewTestCase; 26 import android.media.CamcorderProfile; 27 import android.media.MediaCodecInfo; 28 import android.media.MediaCodecInfo.CodecCapabilities; 29 import android.media.MediaCodecInfo.CodecProfileLevel; 30 import android.media.Image; 31 import android.media.ImageReader; 32 import android.media.MediaCodecList; 33 import android.media.MediaPlayer; 34 import android.media.MediaRecorder; 35 import android.os.Environment; 36 import android.os.SystemClock; 37 import android.test.suitebuilder.annotation.LargeTest; 38 import android.util.Log; 39 import android.util.Range; 40 import android.view.Surface; 41 42 import com.android.ex.camera2.blocking.BlockingSessionCallback; 43 44 import junit.framework.AssertionFailedError; 45 46 import java.io.File; 47 import java.util.ArrayList; 48 import java.util.List; 49 50 /** 51 * CameraDevice video recording use case tests by using MediaRecorder and 52 * MediaCodec. 53 */ 54 @LargeTest 55 public class RecordingTest extends Camera2SurfaceViewTestCase { 56 private static final String TAG = "RecordingTest"; 57 private static final boolean VERBOSE = Log.isLoggable(TAG, Log.VERBOSE); 58 private static final boolean DEBUG_DUMP = Log.isLoggable(TAG, Log.DEBUG); 59 private static final int RECORDING_DURATION_MS = 3000; 60 private static final int DURATION_MARGIN_MS = 600; 61 private static final int FRAME_DURATION_ERROR_TOLERANCE_MS = 3; 62 private static final int BIT_RATE_1080P = 16000000; 63 private static final int BIT_RATE_MIN = 64000; 64 private static final int BIT_RATE_MAX = 40000000; 65 private static final int VIDEO_FRAME_RATE = 30; 66 private final String VIDEO_FILE_PATH = Environment.getExternalStorageDirectory().getPath(); 67 private static final int[] mCamcorderProfileList = { 68 CamcorderProfile.QUALITY_2160P, 69 CamcorderProfile.QUALITY_1080P, 70 CamcorderProfile.QUALITY_480P, 71 CamcorderProfile.QUALITY_720P, 72 CamcorderProfile.QUALITY_CIF, 73 CamcorderProfile.QUALITY_LOW, 74 CamcorderProfile.QUALITY_HIGH, 75 CamcorderProfile.QUALITY_QCIF, 76 CamcorderProfile.QUALITY_QVGA, 77 }; 78 private static final int MAX_VIDEO_SNAPSHOT_IMAGES = 5; 79 private static final int BURST_VIDEO_SNAPSHOT_NUM = 3; 80 private static final int SLOWMO_SLOW_FACTOR = 4; 81 private List<Size> mSupportedVideoSizes; 82 private Surface mRecordingSurface; 83 private MediaRecorder mMediaRecorder; 84 private String mOutMediaFileName; 85 private int mVideoFrameRate; 86 private Size mVideoSize; 87 88 @Override 89 protected void setUp() throws Exception { 90 super.setUp(); 91 } 92 93 @Override 94 protected void tearDown() throws Exception { 95 super.tearDown(); 96 } 97 98 /** 99 * <p> 100 * Test basic camera recording. 101 * </p> 102 * <p> 103 * This test covers the typical basic use case of camera recording. 104 * MediaRecorder is used to record the audio and video, CamcorderProfile is 105 * used to configure the MediaRecorder. It goes through the pre-defined 106 * CamcorderProfile list, test each profile configuration and validate the 107 * recorded video. Preview is set to the video size. 108 * </p> 109 */ 110 public void testBasicRecording() throws Exception { 111 for (int i = 0; i < mCameraIds.length; i++) { 112 try { 113 Log.i(TAG, "Testing basic recording for camera " + mCameraIds[i]); 114 // Re-use the MediaRecorder object for the same camera device. 115 mMediaRecorder = new MediaRecorder(); 116 openDevice(mCameraIds[i]); 117 118 initSupportedVideoSize(mCameraIds[i]); 119 120 basicRecordingTestByCamera(); 121 } finally { 122 closeDevice(); 123 releaseRecorder(); 124 } 125 } 126 } 127 128 /** 129 * <p> 130 * Test camera recording for all supported sizes by using MediaRecorder. 131 * </p> 132 * <p> 133 * This test covers camera recording for all supported sizes by camera. MediaRecorder 134 * is used to encode the video. Preview is set to the video size. Recorded videos are 135 * validated according to the recording configuration. 136 * </p> 137 */ 138 public void testSupportedVideoSizes() throws Exception { 139 for (int i = 0; i < mCameraIds.length; i++) { 140 try { 141 Log.i(TAG, "Testing supported video size recording for camera " + mCameraIds[i]); 142 // Re-use the MediaRecorder object for the same camera device. 143 mMediaRecorder = new MediaRecorder(); 144 openDevice(mCameraIds[i]); 145 146 initSupportedVideoSize(mCameraIds[i]); 147 148 recordingSizeTestByCamera(); 149 } finally { 150 closeDevice(); 151 releaseRecorder(); 152 } 153 } 154 } 155 156 /** 157 * Test different start/stop orders of Camera and Recorder. 158 * 159 * <p>The recording should be working fine for any kind of start/stop orders.</p> 160 */ 161 public void testCameraRecorderOrdering() { 162 // TODO: need implement 163 } 164 165 /** 166 * <p> 167 * Test camera recording for all supported sizes by using MediaCodec. 168 * </p> 169 * <p> 170 * This test covers video only recording for all supported sizes (camera and 171 * encoder). MediaCodec is used to encode the video. The recorded videos are 172 * validated according to the recording configuration. 173 * </p> 174 */ 175 public void testMediaCodecRecording() throws Exception { 176 // TODO. Need implement. 177 } 178 179 /** 180 * <p> 181 * Test video snapshot for each camera. 182 * </p> 183 * <p> 184 * This test covers video snapshot typical use case. The MediaRecorder is used to record the 185 * video for each available video size. The largest still capture size is selected to 186 * capture the JPEG image. The still capture images are validated according to the capture 187 * configuration. The timestamp of capture result before and after video snapshot is also 188 * checked to make sure no frame drop caused by video snapshot. 189 * </p> 190 */ 191 public void testVideoSnapshot() throws Exception { 192 videoSnapshotHelper(/*burstTest*/false); 193 } 194 195 /** 196 * <p> 197 * Test burst video snapshot for each camera. 198 * </p> 199 * <p> 200 * This test covers burst video snapshot capture. The MediaRecorder is used to record the 201 * video for each available video size. The largest still capture size is selected to 202 * capture the JPEG image. {@value #BURST_VIDEO_SNAPSHOT_NUM} video snapshot requests will be 203 * sent during the test. The still capture images are validated according to the capture 204 * configuration. 205 * </p> 206 */ 207 public void testBurstVideoSnapshot() throws Exception { 208 videoSnapshotHelper(/*burstTest*/true); 209 } 210 211 /** 212 * Test timelapse recording, where capture rate is slower than video (playback) frame rate. 213 */ 214 public void testTimelapseRecording() throws Exception { 215 // TODO. Need implement. 216 } 217 218 public void testSlowMotionRecording() throws Exception { 219 slowMotionRecording(); 220 } 221 222 /** 223 * Test slow motion recording where capture rate (camera output) is different with 224 * video (playback) frame rate for each camera if high speed recording is supported 225 * by both camera and encoder. 226 * 227 * <p> 228 * Normal recording use cases make the capture rate (camera output frame 229 * rate) the same as the video (playback) frame rate. This guarantees that 230 * the motions in the scene play at the normal speed. If the capture rate is 231 * faster than video frame rate, for a given time duration, more number of 232 * frames are captured than it can be played in the same time duration. This 233 * generates "slow motion" effect during playback. 234 * </p> 235 */ 236 private void slowMotionRecording() throws Exception { 237 for (String id : mCameraIds) { 238 try { 239 Log.i(TAG, "Testing slow motion recording for camera " + id); 240 // Re-use the MediaRecorder object for the same camera device. 241 mMediaRecorder = new MediaRecorder(); 242 openDevice(id); 243 244 if (!mStaticInfo.isHighSpeedVideoSupported()) { 245 continue; 246 } 247 248 StreamConfigurationMap config = 249 mStaticInfo.getValueFromKeyNonNull( 250 CameraCharacteristics.SCALER_STREAM_CONFIGURATION_MAP); 251 Size[] highSpeedVideoSizes = config.getHighSpeedVideoSizes(); 252 for (Size size : highSpeedVideoSizes) { 253 Range<Integer> fpsRange = getHighestHighSpeedFixedFpsRangeForSize(config, size); 254 mCollector.expectNotNull("Unable to find the fixed frame rate fps range for " + 255 "size " + size, fpsRange); 256 if (fpsRange == null) { 257 continue; 258 } 259 260 int captureRate = fpsRange.getLower(); 261 int videoFramerate = captureRate / SLOWMO_SLOW_FACTOR; 262 /** 263 * Check if encoder support this. TODO: use HIGH_SPEED_720p 264 * CamCorderProfile to get the performance guarantee. Also 265 * add the test in StaticMetadataTest to check: 1. Camera 266 * high speed recording metadata is correctly reported 2. 267 * Encoder profile/level info is correctly reported. After 268 * that, we only need check the CamcorderProfile before 269 * skipping the test. 270 */ 271 if (!isSupportedByAVCEncoder(size, captureRate)) { 272 Log.i(TAG, "high speed recording " + size + "@" + captureRate + "fps" 273 + " is not supported by AVC encoder"); 274 continue; 275 } 276 277 mOutMediaFileName = VIDEO_FILE_PATH + "/test_slowMo_video.mp4"; 278 if (DEBUG_DUMP) { 279 mOutMediaFileName = VIDEO_FILE_PATH + "/test_slowMo_video_" + id + "_" 280 + size.toString() + ".mp4"; 281 } 282 283 prepareRecording(size, videoFramerate, captureRate); 284 285 // prepare preview surface by using video size. 286 updatePreviewSurfaceWithVideoSize(size); 287 288 // Start recording 289 startSlowMotionRecording(/*useMediaRecorder*/true, videoFramerate, captureRate, 290 fpsRange); 291 long startTime = SystemClock.elapsedRealtime(); 292 293 // Record certain duration. 294 SystemClock.sleep(RECORDING_DURATION_MS); 295 296 // Stop recording and preview 297 stopRecording(/*useMediaRecorder*/true); 298 int duration = (int) (SystemClock.elapsedRealtime() - startTime); 299 300 // Validation. 301 validateRecording(size, duration * SLOWMO_SLOW_FACTOR); 302 303 } 304 305 } finally { 306 closeDevice(); 307 releaseRecorder(); 308 } 309 } 310 } 311 312 private Range<Integer> getHighestHighSpeedFixedFpsRangeForSize(StreamConfigurationMap config, 313 Size size) { 314 Range<Integer>[] availableFpsRanges = config.getHighSpeedVideoFpsRangesFor(size); 315 Range<Integer> maxRange = availableFpsRanges[0]; 316 boolean foundRange = false; 317 for (Range<Integer> range : availableFpsRanges) { 318 if (range.getLower() == range.getUpper() && range.getLower() >= maxRange.getLower()) { 319 foundRange = true; 320 maxRange = range; 321 } 322 } 323 324 if (!foundRange) { 325 return null; 326 } 327 return maxRange; 328 } 329 330 private void startSlowMotionRecording(boolean useMediaRecorder, int videoFrameRate, 331 int captureRate, Range<Integer> fpsRange) throws Exception { 332 List<Surface> outputSurfaces = new ArrayList<Surface>(2); 333 assertTrue("Both preview and recording surfaces should be valid", 334 mPreviewSurface.isValid() && mRecordingSurface.isValid()); 335 outputSurfaces.add(mPreviewSurface); 336 outputSurfaces.add(mRecordingSurface); 337 // Video snapshot surface 338 if (mReaderSurface != null) { 339 outputSurfaces.add(mReaderSurface); 340 } 341 mSessionListener = new BlockingSessionCallback(); 342 mSession = configureCameraSession(mCamera, outputSurfaces, mSessionListener, mHandler); 343 344 CaptureRequest.Builder recordingRequestBuilder = 345 mCamera.createCaptureRequest(CameraDevice.TEMPLATE_RECORD); 346 recordingRequestBuilder.set(CaptureRequest.CONTROL_MODE, 347 CaptureRequest.CONTROL_MODE_USE_SCENE_MODE); 348 recordingRequestBuilder.set(CaptureRequest.CONTROL_SCENE_MODE, 349 CaptureRequest.CONTROL_SCENE_MODE_HIGH_SPEED_VIDEO); 350 351 CaptureRequest.Builder recordingOnlyBuilder = 352 mCamera.createCaptureRequest(CameraDevice.TEMPLATE_RECORD); 353 recordingOnlyBuilder.set(CaptureRequest.CONTROL_MODE, 354 CaptureRequest.CONTROL_MODE_USE_SCENE_MODE); 355 recordingOnlyBuilder.set(CaptureRequest.CONTROL_SCENE_MODE, 356 CaptureRequest.CONTROL_SCENE_MODE_HIGH_SPEED_VIDEO); 357 int slowMotionFactor = captureRate / videoFrameRate; 358 359 // Make sure camera output frame rate is set to correct value. 360 recordingRequestBuilder.set(CaptureRequest.CONTROL_AE_TARGET_FPS_RANGE, fpsRange); 361 recordingRequestBuilder.addTarget(mRecordingSurface); 362 recordingRequestBuilder.addTarget(mPreviewSurface); 363 recordingOnlyBuilder.set(CaptureRequest.CONTROL_AE_TARGET_FPS_RANGE, fpsRange); 364 recordingOnlyBuilder.addTarget(mRecordingSurface); 365 366 List<CaptureRequest> slowMoRequests = new ArrayList<CaptureRequest>(); 367 slowMoRequests.add(recordingRequestBuilder.build());// Preview + recording. 368 369 for (int i = 0; i < slowMotionFactor - 1; i++) { 370 slowMoRequests.add(recordingOnlyBuilder.build()); // Recording only. 371 } 372 mSession.setRepeatingBurst(slowMoRequests, null, null); 373 374 if (useMediaRecorder) { 375 mMediaRecorder.start(); 376 } else { 377 // TODO: need implement MediaCodec path. 378 } 379 380 } 381 382 /** 383 * Test camera recording by using each available CamcorderProfile for a 384 * given camera. preview size is set to the video size. 385 */ 386 private void basicRecordingTestByCamera() throws Exception { 387 for (int profileId : mCamcorderProfileList) { 388 int cameraId = Integer.valueOf(mCamera.getId()); 389 if (!CamcorderProfile.hasProfile(cameraId, profileId) || 390 allowedUnsupported(cameraId, profileId)) { 391 continue; 392 } 393 394 CamcorderProfile profile = CamcorderProfile.get(cameraId, profileId); 395 Size videoSz = new Size(profile.videoFrameWidth, profile.videoFrameHeight); 396 assertTrue("Video size " + videoSz.toString() + " for profile ID " + profileId + 397 " must be one of the camera device supported video size!", 398 mSupportedVideoSizes.contains(videoSz)); 399 400 if (VERBOSE) { 401 Log.v(TAG, "Testing camera recording with video size " + videoSz.toString()); 402 } 403 404 // Configure preview and recording surfaces. 405 mOutMediaFileName = VIDEO_FILE_PATH + "/test_video.mp4"; 406 if (DEBUG_DUMP) { 407 mOutMediaFileName = VIDEO_FILE_PATH + "/test_video_" + cameraId + "_" 408 + videoSz.toString() + ".mp4"; 409 } 410 411 prepareRecordingWithProfile(profile); 412 413 // prepare preview surface by using video size. 414 updatePreviewSurfaceWithVideoSize(videoSz); 415 416 // Start recording 417 startRecording(/* useMediaRecorder */true); 418 long startTime = SystemClock.elapsedRealtime(); 419 420 // Record certain duration. 421 SystemClock.sleep(RECORDING_DURATION_MS); 422 423 // Stop recording and preview 424 stopRecording(/* useMediaRecorder */true); 425 int duration = (int) (SystemClock.elapsedRealtime() - startTime); 426 427 // Validation. 428 validateRecording(videoSz, duration); 429 } 430 } 431 432 /** 433 * Test camera recording for each supported video size by camera, preview 434 * size is set to the video size. 435 */ 436 private void recordingSizeTestByCamera() throws Exception { 437 for (Size sz : mSupportedVideoSizes) { 438 if (!isSupported(sz, VIDEO_FRAME_RATE, VIDEO_FRAME_RATE)) { 439 continue; 440 } 441 442 if (VERBOSE) { 443 Log.v(TAG, "Testing camera recording with video size " + sz.toString()); 444 } 445 446 // Configure preview and recording surfaces. 447 mOutMediaFileName = VIDEO_FILE_PATH + "/test_video.mp4"; 448 if (DEBUG_DUMP) { 449 mOutMediaFileName = VIDEO_FILE_PATH + "/test_video_" + mCamera.getId() + "_" 450 + sz.toString() + ".mp4"; 451 } 452 453 // Use AVC and AAC a/v compression format. 454 prepareRecording(sz, VIDEO_FRAME_RATE, VIDEO_FRAME_RATE); 455 456 // prepare preview surface by using video size. 457 updatePreviewSurfaceWithVideoSize(sz); 458 459 // Start recording 460 startRecording(/* useMediaRecorder */true); 461 long startTime = SystemClock.elapsedRealtime(); 462 463 // Record certain duration. 464 SystemClock.sleep(RECORDING_DURATION_MS); 465 466 // Stop recording and preview 467 stopRecording(/* useMediaRecorder */true); 468 int duration = (int) (SystemClock.elapsedRealtime() - startTime); 469 470 // Validation. 471 validateRecording(sz, duration); 472 } 473 } 474 475 /** 476 * Initialize the supported video sizes. 477 */ 478 private void initSupportedVideoSize(String cameraId) throws Exception { 479 Size maxVideoSize = SIZE_BOUND_1080P; 480 if (CamcorderProfile.hasProfile(CamcorderProfile.QUALITY_2160P)) { 481 maxVideoSize = SIZE_BOUND_2160P; 482 } 483 mSupportedVideoSizes = 484 getSupportedVideoSizes(cameraId, mCameraManager, maxVideoSize); 485 } 486 487 /** 488 * Simple wrapper to wrap normal/burst video snapshot tests 489 */ 490 private void videoSnapshotHelper(boolean burstTest) throws Exception { 491 for (String id : mCameraIds) { 492 try { 493 Log.i(TAG, "Testing video snapshot for camera " + id); 494 // Re-use the MediaRecorder object for the same camera device. 495 mMediaRecorder = new MediaRecorder(); 496 497 openDevice(id); 498 499 if (mStaticInfo.isHardwareLevelLegacy()) { 500 Log.i(TAG, "Skipping test on legacy devices"); 501 continue; 502 } 503 504 initSupportedVideoSize(id); 505 506 videoSnapshotTestByCamera(burstTest); 507 } finally { 508 closeDevice(); 509 releaseRecorder(); 510 } 511 } 512 } 513 514 /** 515 * Returns {@code true} if the {@link CamcorderProfile} ID is allowed to be unsupported. 516 * 517 * <p>This only allows unsupported profiles when using the LEGACY mode of the Camera API.</p> 518 * 519 * @param profileId a {@link CamcorderProfile} ID to check. 520 * @return {@code true} if supported. 521 */ 522 private boolean allowedUnsupported(int cameraId, int profileId) { 523 if (!mStaticInfo.isHardwareLevelLegacy()) { 524 return false; 525 } 526 527 switch(profileId) { 528 case CamcorderProfile.QUALITY_2160P: 529 case CamcorderProfile.QUALITY_1080P: 530 case CamcorderProfile.QUALITY_HIGH: 531 return !CamcorderProfile.hasProfile(cameraId, profileId) || 532 CamcorderProfile.get(cameraId, profileId).videoFrameWidth >= 1080; 533 } 534 return false; 535 } 536 537 /** 538 * Test video snapshot for each available CamcorderProfile for a given camera. 539 * 540 * <p> 541 * Preview size is set to the video size. For the burst test, frame drop and jittering 542 * is not checked. 543 * </p> 544 * 545 * @param burstTest Perform burst capture or single capture. For burst capture 546 * {@value #BURST_VIDEO_SNAPSHOT_NUM} capture requests will be sent. 547 */ 548 private void videoSnapshotTestByCamera(boolean burstTest) 549 throws Exception { 550 final int NUM_SINGLE_SHOT_TEST = 5; 551 final int FRAMEDROP_TOLERANCE = 8; 552 for (int profileId : mCamcorderProfileList) { 553 int cameraId = Integer.valueOf(mCamera.getId()); 554 if (!CamcorderProfile.hasProfile(cameraId, profileId) || 555 allowedUnsupported(cameraId, profileId)) { 556 continue; 557 } 558 559 CamcorderProfile profile = CamcorderProfile.get(cameraId, profileId); 560 Size videoSz = new Size(profile.videoFrameWidth, profile.videoFrameHeight); 561 if (!mSupportedVideoSizes.contains(videoSz)) { 562 mCollector.addMessage("Video size " + videoSz.toString() + " for profile ID " + 563 profileId + " must be one of the camera device supported video size!"); 564 continue; 565 } 566 567 Size maxPreviewSize = mOrderedPreviewSizes.get(0); 568 569 // For LEGACY, find closest supported smaller or equal JPEG size to the current video 570 // size; if no size is smaller than the video, pick the smallest JPEG size. The assert 571 // for video size above guarantees that for LIMITED or FULL, we select videoSz here. 572 Size videoSnapshotSz = mOrderedStillSizes.get(mOrderedStillSizes.size() - 1); 573 for (int i = mOrderedStillSizes.size() - 2; i >= 0; i--) { 574 Size candidateSize = mOrderedStillSizes.get(i); 575 if (candidateSize.getWidth() <= videoSz.getWidth() && 576 candidateSize.getHeight() <= videoSz.getHeight()) { 577 videoSnapshotSz = candidateSize; 578 } 579 } 580 581 /** 582 * Only test full res snapshot when below conditions are all true. 583 * 1. Camera is a FULL device 584 * 2. video size is up to max preview size, which will be bounded by 1080p. 585 */ 586 if (mStaticInfo.isHardwareLevelFull() && 587 videoSz.getWidth() <= maxPreviewSize.getWidth() && 588 videoSz.getHeight() <= maxPreviewSize.getHeight()) { 589 videoSnapshotSz = mOrderedStillSizes.get(0); 590 } 591 592 createImageReader( 593 videoSnapshotSz, ImageFormat.JPEG, 594 MAX_VIDEO_SNAPSHOT_IMAGES, /*listener*/null); 595 596 if (VERBOSE) { 597 Log.v(TAG, "Testing camera recording with video size " + videoSz.toString()); 598 } 599 600 // Configure preview and recording surfaces. 601 mOutMediaFileName = VIDEO_FILE_PATH + "/test_video.mp4"; 602 if (DEBUG_DUMP) { 603 mOutMediaFileName = VIDEO_FILE_PATH + "/test_video_" + cameraId + "_" 604 + videoSz.toString() + ".mp4"; 605 } 606 607 int numTestIterations = burstTest ? 1 : NUM_SINGLE_SHOT_TEST; 608 int totalDroppedFrames = 0; 609 610 for (int numTested = 0; numTested < numTestIterations; numTested++) { 611 prepareRecordingWithProfile(profile); 612 613 // prepare video snapshot 614 SimpleCaptureCallback resultListener = new SimpleCaptureCallback(); 615 SimpleImageReaderListener imageListener = new SimpleImageReaderListener(); 616 CaptureRequest.Builder videoSnapshotRequestBuilder = 617 mCamera.createCaptureRequest((mStaticInfo.isHardwareLevelLegacy()) ? 618 CameraDevice.TEMPLATE_RECORD : 619 CameraDevice.TEMPLATE_VIDEO_SNAPSHOT); 620 621 // prepare preview surface by using video size. 622 updatePreviewSurfaceWithVideoSize(videoSz); 623 624 prepareVideoSnapshot(videoSnapshotRequestBuilder, imageListener); 625 CaptureRequest request = videoSnapshotRequestBuilder.build(); 626 627 // Start recording 628 startRecording(/* useMediaRecorder */true, resultListener); 629 long startTime = SystemClock.elapsedRealtime(); 630 631 // Record certain duration. 632 SystemClock.sleep(RECORDING_DURATION_MS / 2); 633 634 // take video snapshot 635 if (burstTest) { 636 List<CaptureRequest> requests = 637 new ArrayList<CaptureRequest>(BURST_VIDEO_SNAPSHOT_NUM); 638 for (int i = 0; i < BURST_VIDEO_SNAPSHOT_NUM; i++) { 639 requests.add(request); 640 } 641 mSession.captureBurst(requests, resultListener, mHandler); 642 } else { 643 mSession.capture(request, resultListener, mHandler); 644 } 645 646 // make sure recording is still going after video snapshot 647 SystemClock.sleep(RECORDING_DURATION_MS / 2); 648 649 // Stop recording and preview 650 stopRecording(/* useMediaRecorder */true); 651 int duration = (int) (SystemClock.elapsedRealtime() - startTime); 652 653 // Validation recorded video 654 validateRecording(videoSz, duration); 655 656 if (burstTest) { 657 for (int i = 0; i < BURST_VIDEO_SNAPSHOT_NUM; i++) { 658 Image image = imageListener.getImage(CAPTURE_IMAGE_TIMEOUT_MS); 659 validateVideoSnapshotCapture(image, videoSnapshotSz); 660 image.close(); 661 } 662 } else { 663 // validate video snapshot image 664 Image image = imageListener.getImage(CAPTURE_IMAGE_TIMEOUT_MS); 665 validateVideoSnapshotCapture(image, videoSnapshotSz); 666 667 // validate if there is framedrop around video snapshot 668 totalDroppedFrames += validateFrameDropAroundVideoSnapshot( 669 resultListener, image.getTimestamp()); 670 671 //TODO: validate jittering. Should move to PTS 672 //validateJittering(resultListener); 673 674 image.close(); 675 } 676 } 677 678 if (!burstTest) { 679 Log.w(TAG, String.format("Camera %d Video size %s: Number of dropped frames " + 680 "detected in %d trials is %d frames.", cameraId, videoSz.toString(), 681 numTestIterations, totalDroppedFrames)); 682 mCollector.expectLessOrEqual( 683 String.format( 684 "Camera %d Video size %s: Number of dropped frames %d must not" 685 + " be larger than %d", 686 cameraId, videoSz.toString(), totalDroppedFrames, 687 FRAMEDROP_TOLERANCE), 688 FRAMEDROP_TOLERANCE, totalDroppedFrames); 689 } 690 closeImageReader(); 691 } 692 } 693 694 /** 695 * Configure video snapshot request according to the still capture size 696 */ 697 private void prepareVideoSnapshot( 698 CaptureRequest.Builder requestBuilder, 699 ImageReader.OnImageAvailableListener imageListener) 700 throws Exception { 701 mReader.setOnImageAvailableListener(imageListener, mHandler); 702 assertNotNull("Recording surface must be non-null!", mRecordingSurface); 703 requestBuilder.addTarget(mRecordingSurface); 704 assertNotNull("Preview surface must be non-null!", mPreviewSurface); 705 requestBuilder.addTarget(mPreviewSurface); 706 assertNotNull("Reader surface must be non-null!", mReaderSurface); 707 requestBuilder.addTarget(mReaderSurface); 708 } 709 710 /** 711 * Update preview size with video size. 712 * 713 * <p>Preview size will be capped with max preview size.</p> 714 * 715 * @param videoSize The video size used for preview. 716 */ 717 private void updatePreviewSurfaceWithVideoSize(Size videoSize) { 718 if (mOrderedPreviewSizes == null) { 719 throw new IllegalStateException("supported preview size list is not initialized yet"); 720 } 721 Size maxPreviewSize = mOrderedPreviewSizes.get(0); 722 Size previewSize = videoSize; 723 if (videoSize.getWidth() > maxPreviewSize.getWidth() || 724 videoSize.getHeight() > maxPreviewSize.getHeight()) { 725 Log.w(TAG, "Overwrite preview size from " + videoSize.toString() + 726 " to " + maxPreviewSize.toString()); 727 previewSize = maxPreviewSize; 728 } 729 730 updatePreviewSurface(previewSize); 731 } 732 733 /** 734 * Configure MediaRecorder recording session with CamcorderProfile, prepare 735 * the recording surface. 736 */ 737 private void prepareRecordingWithProfile(CamcorderProfile profile) 738 throws Exception { 739 // Prepare MediaRecorder. 740 mMediaRecorder.setAudioSource(MediaRecorder.AudioSource.CAMCORDER); 741 mMediaRecorder.setVideoSource(MediaRecorder.VideoSource.SURFACE); 742 mMediaRecorder.setProfile(profile); 743 mMediaRecorder.setOutputFile(mOutMediaFileName); 744 mMediaRecorder.prepare(); 745 mRecordingSurface = mMediaRecorder.getSurface(); 746 assertNotNull("Recording surface must be non-null!", mRecordingSurface); 747 mVideoFrameRate = profile.videoFrameRate; 748 mVideoSize = new Size(profile.videoFrameWidth, profile.videoFrameHeight); 749 } 750 751 /** 752 * Configure MediaRecorder recording session with CamcorderProfile, prepare 753 * the recording surface. Use AVC for video compression, AAC for audio compression. 754 * Both are required for android devices by android CDD. 755 */ 756 private void prepareRecording(Size sz, int videoFrameRate, int captureRate) 757 throws Exception { 758 // Prepare MediaRecorder. 759 mMediaRecorder.setAudioSource(MediaRecorder.AudioSource.CAMCORDER); 760 mMediaRecorder.setVideoSource(MediaRecorder.VideoSource.SURFACE); 761 mMediaRecorder.setOutputFormat(MediaRecorder.OutputFormat.THREE_GPP); 762 mMediaRecorder.setOutputFile(mOutMediaFileName); 763 mMediaRecorder.setVideoEncodingBitRate(getVideoBitRate(sz)); 764 mMediaRecorder.setVideoFrameRate(videoFrameRate); 765 mMediaRecorder.setCaptureRate(captureRate); 766 mMediaRecorder.setVideoSize(sz.getWidth(), sz.getHeight()); 767 mMediaRecorder.setVideoEncoder(MediaRecorder.VideoEncoder.H264); 768 mMediaRecorder.setAudioEncoder(MediaRecorder.AudioEncoder.AAC); 769 mMediaRecorder.prepare(); 770 mRecordingSurface = mMediaRecorder.getSurface(); 771 assertNotNull("Recording surface must be non-null!", mRecordingSurface); 772 mVideoFrameRate = videoFrameRate; 773 mVideoSize = sz; 774 } 775 776 private void startRecording(boolean useMediaRecorder, 777 CameraCaptureSession.CaptureCallback listener) throws Exception { 778 List<Surface> outputSurfaces = new ArrayList<Surface>(2); 779 assertTrue("Both preview and recording surfaces should be valid", 780 mPreviewSurface.isValid() && mRecordingSurface.isValid()); 781 outputSurfaces.add(mPreviewSurface); 782 outputSurfaces.add(mRecordingSurface); 783 // Video snapshot surface 784 if (mReaderSurface != null) { 785 outputSurfaces.add(mReaderSurface); 786 } 787 mSessionListener = new BlockingSessionCallback(); 788 mSession = configureCameraSession(mCamera, outputSurfaces, mSessionListener, mHandler); 789 790 CaptureRequest.Builder recordingRequestBuilder = 791 mCamera.createCaptureRequest(CameraDevice.TEMPLATE_RECORD); 792 // Make sure camera output frame rate is set to correct value. 793 Range<Integer> fpsRange = Range.create(mVideoFrameRate, mVideoFrameRate); 794 recordingRequestBuilder.set(CaptureRequest.CONTROL_AE_TARGET_FPS_RANGE, fpsRange); 795 recordingRequestBuilder.addTarget(mRecordingSurface); 796 recordingRequestBuilder.addTarget(mPreviewSurface); 797 mSession.setRepeatingRequest(recordingRequestBuilder.build(), listener, mHandler); 798 799 if (useMediaRecorder) { 800 mMediaRecorder.start(); 801 } else { 802 // TODO: need implement MediaCodec path. 803 } 804 } 805 806 private void startRecording(boolean useMediaRecorder) throws Exception { 807 startRecording(useMediaRecorder, null); 808 } 809 810 private void stopCameraStreaming() throws Exception { 811 if (VERBOSE) { 812 Log.v(TAG, "Stopping camera streaming and waiting for idle"); 813 } 814 // Stop repeating, wait for captures to complete, and disconnect from 815 // surfaces 816 mSession.close(); 817 mSessionListener.getStateWaiter().waitForState(SESSION_CLOSED, SESSION_CLOSE_TIMEOUT_MS); 818 } 819 820 private void stopRecording(boolean useMediaRecorder) throws Exception { 821 if (useMediaRecorder) { 822 stopCameraStreaming(); 823 824 mMediaRecorder.stop(); 825 // Can reuse the MediaRecorder object after reset. 826 mMediaRecorder.reset(); 827 } else { 828 // TODO: need implement MediaCodec path. 829 } 830 if (mRecordingSurface != null) { 831 mRecordingSurface.release(); 832 mRecordingSurface = null; 833 } 834 } 835 836 private void releaseRecorder() { 837 if (mMediaRecorder != null) { 838 mMediaRecorder.release(); 839 mMediaRecorder = null; 840 } 841 } 842 843 private void validateRecording(Size sz, int durationMs) throws Exception { 844 File outFile = new File(mOutMediaFileName); 845 assertTrue("No video is recorded", outFile.exists()); 846 847 MediaPlayer mediaPlayer = new MediaPlayer(); 848 try { 849 mediaPlayer.setDataSource(mOutMediaFileName); 850 mediaPlayer.prepare(); 851 Size videoSz = new Size(mediaPlayer.getVideoWidth(), mediaPlayer.getVideoHeight()); 852 assertTrue("Video size doesn't match, expected " + sz.toString() + 853 " got " + videoSz.toString(), videoSz.equals(sz)); 854 int duration = mediaPlayer.getDuration(); 855 if (VERBOSE) { 856 Log.v(TAG, String.format("Video duration: recorded %dms, expected %dms", 857 duration, durationMs)); 858 } 859 860 // TODO: Don't skip this for video snapshot 861 if (!mStaticInfo.isHardwareLevelLegacy()) { 862 assertTrue(String.format( 863 "Video duration doesn't match: recorded %dms, expected %dms", duration, 864 durationMs), Math.abs(duration - durationMs) < DURATION_MARGIN_MS); 865 } 866 } finally { 867 mediaPlayer.release(); 868 if (!DEBUG_DUMP) { 869 outFile.delete(); 870 } 871 } 872 } 873 874 /** 875 * Validate video snapshot capture image object sanity and test. 876 * 877 * <p> Check for size, format and jpeg decoding</p> 878 * 879 * @param image The JPEG image to be verified. 880 * @param size The JPEG capture size to be verified against. 881 */ 882 private void validateVideoSnapshotCapture(Image image, Size size) { 883 CameraTestUtils.validateImage(image, size.getWidth(), size.getHeight(), 884 ImageFormat.JPEG, /*filePath*/null); 885 } 886 887 /** 888 * Validate if video snapshot causes frame drop. 889 * Here frame drop is defined as frame duration >= 2 * expected frame duration. 890 * Return the estimated number of frames dropped during video snapshot 891 */ 892 private int validateFrameDropAroundVideoSnapshot( 893 SimpleCaptureCallback resultListener, long imageTimeStamp) { 894 int expectedDurationMs = 1000 / mVideoFrameRate; 895 CaptureResult prevResult = resultListener.getCaptureResult(WAIT_FOR_RESULT_TIMEOUT_MS); 896 long prevTS = getValueNotNull(prevResult, CaptureResult.SENSOR_TIMESTAMP); 897 while (!resultListener.hasMoreResults()) { 898 CaptureResult currentResult = 899 resultListener.getCaptureResult(WAIT_FOR_RESULT_TIMEOUT_MS); 900 long currentTS = getValueNotNull(currentResult, CaptureResult.SENSOR_TIMESTAMP); 901 if (currentTS == imageTimeStamp) { 902 // validate the timestamp before and after, then return 903 CaptureResult nextResult = 904 resultListener.getCaptureResult(WAIT_FOR_RESULT_TIMEOUT_MS); 905 long nextTS = getValueNotNull(nextResult, CaptureResult.SENSOR_TIMESTAMP); 906 int durationMs = (int) (currentTS - prevTS) / 1000000; 907 int totalFramesDropped = 0; 908 909 // Snapshots in legacy mode pause the preview briefly. Skip the duration 910 // requirements for legacy mode unless this is fixed. 911 if (!mStaticInfo.isHardwareLevelLegacy()) { 912 // Log a warning is there is any frame drop detected. 913 if (durationMs >= expectedDurationMs * 2) { 914 Log.w(TAG, String.format( 915 "Video %dx%d Frame drop detected before video snapshot: " + 916 "duration %dms (expected %dms)", 917 mVideoSize.getWidth(), mVideoSize.getHeight(), 918 durationMs, expectedDurationMs 919 )); 920 } 921 922 durationMs = (int) (nextTS - currentTS) / 1000000; 923 // Log a warning is there is any frame drop detected. 924 if (durationMs >= expectedDurationMs * 2) { 925 Log.w(TAG, String.format( 926 "Video %dx%d Frame drop detected after video snapshot: " + 927 "duration %dms (expected %dms)", 928 mVideoSize.getWidth(), mVideoSize.getHeight(), 929 durationMs, expectedDurationMs 930 )); 931 } 932 933 int totalDurationMs = (int) (nextTS - prevTS) / 1000000; 934 // Rounding and minus 2 for the expected 2 frames interval 935 totalFramesDropped = 936 (totalDurationMs + expectedDurationMs / 2) /expectedDurationMs - 2; 937 if (totalFramesDropped < 0) { 938 Log.w(TAG, "totalFrameDropped is " + totalFramesDropped + 939 ". Video frame rate might be too fast."); 940 } 941 totalFramesDropped = Math.max(0, totalFramesDropped); 942 } 943 return totalFramesDropped; 944 } 945 prevTS = currentTS; 946 } 947 throw new AssertionFailedError( 948 "Video snapshot timestamp does not match any of capture results!"); 949 } 950 951 /** 952 * Validate frame jittering from the input simple listener's buffered results 953 */ 954 private void validateJittering(SimpleCaptureCallback resultListener) { 955 int expectedDurationMs = 1000 / mVideoFrameRate; 956 CaptureResult prevResult = resultListener.getCaptureResult(WAIT_FOR_RESULT_TIMEOUT_MS); 957 long prevTS = getValueNotNull(prevResult, CaptureResult.SENSOR_TIMESTAMP); 958 while (!resultListener.hasMoreResults()) { 959 CaptureResult currentResult = 960 resultListener.getCaptureResult(WAIT_FOR_RESULT_TIMEOUT_MS); 961 long currentTS = getValueNotNull(currentResult, CaptureResult.SENSOR_TIMESTAMP); 962 int durationMs = (int) (currentTS - prevTS) / 1000000; 963 int durationError = Math.abs(durationMs - expectedDurationMs); 964 long frameNumber = currentResult.getFrameNumber(); 965 mCollector.expectTrue( 966 String.format( 967 "Resolution %dx%d Frame %d: jittering (%dms) exceeds bound [%dms,%dms]", 968 mVideoSize.getWidth(), mVideoSize.getHeight(), 969 frameNumber, durationMs, 970 expectedDurationMs - FRAME_DURATION_ERROR_TOLERANCE_MS, 971 expectedDurationMs + FRAME_DURATION_ERROR_TOLERANCE_MS), 972 durationError <= FRAME_DURATION_ERROR_TOLERANCE_MS); 973 prevTS = currentTS; 974 } 975 } 976 977 /** 978 * Calculate a video bit rate based on the size. The bit rate is scaled 979 * based on ratio of video size to 1080p size. 980 */ 981 private int getVideoBitRate(Size sz) { 982 int rate = BIT_RATE_1080P; 983 float scaleFactor = sz.getHeight() * sz.getWidth() / (float)(1920 * 1080); 984 rate = (int)(rate * scaleFactor); 985 986 // Clamp to the MIN, MAX range. 987 return Math.max(BIT_RATE_MIN, Math.min(BIT_RATE_MAX, rate)); 988 } 989 990 /** 991 * Check if the encoder and camera are able to support this size and frame rate. 992 * Assume the video compression format is AVC. 993 */ 994 private boolean isSupported(Size sz, int captureRate, int encodingRate) throws Exception { 995 // Check camera capability. 996 if (!isSupportedByCamera(sz, captureRate)) { 997 return false; 998 } 999 1000 // Check encode capability. 1001 if (!isSupportedByAVCEncoder(sz, encodingRate)){ 1002 return false; 1003 } 1004 1005 if(VERBOSE) { 1006 Log.v(TAG, "Both encoder and camera support " + sz.toString() + "@" + encodingRate + "@" 1007 + getVideoBitRate(sz) / 1000 + "Kbps"); 1008 } 1009 1010 return true; 1011 } 1012 1013 private boolean isSupportedByCamera(Size sz, int frameRate) { 1014 // Check if camera can support this sz and frame rate combination. 1015 StreamConfigurationMap config = mStaticInfo. 1016 getValueFromKeyNonNull(CameraCharacteristics.SCALER_STREAM_CONFIGURATION_MAP); 1017 1018 long minDuration = config.getOutputMinFrameDuration(MediaRecorder.class, sz); 1019 if (minDuration == 0) { 1020 return false; 1021 } 1022 1023 int maxFrameRate = (int) (1e9f / minDuration); 1024 return maxFrameRate >= frameRate; 1025 } 1026 1027 /** 1028 * Check if encoder can support this size and frame rate combination by querying 1029 * MediaCodec capability. Check is based on size and frame rate. Ignore the bit rate 1030 * as the bit rates targeted in this test are well below the bit rate max value specified 1031 * by AVC specification for certain level. 1032 */ 1033 private static boolean isSupportedByAVCEncoder(Size sz, int frameRate) { 1034 String mimeType = "video/avc"; 1035 MediaCodecInfo codecInfo = getEncoderInfo(mimeType); 1036 if (codecInfo == null) { 1037 return false; 1038 } 1039 CodecCapabilities cap = codecInfo.getCapabilitiesForType(mimeType); 1040 if (cap == null) { 1041 return false; 1042 } 1043 1044 int highestLevel = 0; 1045 for (CodecProfileLevel lvl : cap.profileLevels) { 1046 if (lvl.level > highestLevel) { 1047 highestLevel = lvl.level; 1048 } 1049 } 1050 // Don't support anything meaningful for level 1 or 2. 1051 if (highestLevel <= CodecProfileLevel.AVCLevel2) { 1052 return false; 1053 } 1054 1055 if(VERBOSE) { 1056 Log.v(TAG, "The highest level supported by encoder is: " + highestLevel); 1057 } 1058 1059 // Put bitRate here for future use. 1060 int maxW, maxH, bitRate; 1061 // Max encoding speed. 1062 int maxMacroblocksPerSecond = 0; 1063 switch(highestLevel) { 1064 case CodecProfileLevel.AVCLevel21: 1065 maxW = 352; 1066 maxH = 576; 1067 bitRate = 4000000; 1068 maxMacroblocksPerSecond = 19800; 1069 break; 1070 case CodecProfileLevel.AVCLevel22: 1071 maxW = 720; 1072 maxH = 480; 1073 bitRate = 4000000; 1074 maxMacroblocksPerSecond = 20250; 1075 break; 1076 case CodecProfileLevel.AVCLevel3: 1077 maxW = 720; 1078 maxH = 480; 1079 bitRate = 10000000; 1080 maxMacroblocksPerSecond = 40500; 1081 break; 1082 case CodecProfileLevel.AVCLevel31: 1083 maxW = 1280; 1084 maxH = 720; 1085 bitRate = 14000000; 1086 maxMacroblocksPerSecond = 108000; 1087 break; 1088 case CodecProfileLevel.AVCLevel32: 1089 maxW = 1280; 1090 maxH = 720; 1091 bitRate = 20000000; 1092 maxMacroblocksPerSecond = 216000; 1093 break; 1094 case CodecProfileLevel.AVCLevel4: 1095 maxW = 1920; 1096 maxH = 1088; // It should be 1088 in terms of AVC capability. 1097 bitRate = 20000000; 1098 maxMacroblocksPerSecond = 245760; 1099 break; 1100 case CodecProfileLevel.AVCLevel41: 1101 maxW = 1920; 1102 maxH = 1088; // It should be 1088 in terms of AVC capability. 1103 bitRate = 50000000; 1104 maxMacroblocksPerSecond = 245760; 1105 break; 1106 case CodecProfileLevel.AVCLevel42: 1107 maxW = 2048; 1108 maxH = 1088; // It should be 1088 in terms of AVC capability. 1109 bitRate = 50000000; 1110 maxMacroblocksPerSecond = 522240; 1111 break; 1112 case CodecProfileLevel.AVCLevel5: 1113 maxW = 3672; 1114 maxH = 1536; 1115 bitRate = 135000000; 1116 maxMacroblocksPerSecond = 589824; 1117 break; 1118 case CodecProfileLevel.AVCLevel51: 1119 default: 1120 maxW = 4096; 1121 maxH = 2304; 1122 bitRate = 240000000; 1123 maxMacroblocksPerSecond = 983040; 1124 break; 1125 } 1126 1127 // Check size limit. 1128 if (sz.getWidth() > maxW || sz.getHeight() > maxH) { 1129 Log.i(TAG, "Requested resolution " + sz.toString() + " exceeds (" + 1130 maxW + "," + maxH + ")"); 1131 return false; 1132 } 1133 1134 // Check frame rate limit. 1135 Size sizeInMb = new Size((sz.getWidth() + 15) / 16, (sz.getHeight() + 15) / 16); 1136 int maxFps = maxMacroblocksPerSecond / (sizeInMb.getWidth() * sizeInMb.getHeight()); 1137 if (frameRate > maxFps) { 1138 Log.i(TAG, "Requested frame rate " + frameRate + " exceeds " + maxFps); 1139 return false; 1140 } 1141 1142 return true; 1143 } 1144 1145 private static MediaCodecInfo getEncoderInfo(String mimeType) { 1146 int numCodecs = MediaCodecList.getCodecCount(); 1147 for (int i = 0; i < numCodecs; i++) { 1148 MediaCodecInfo codecInfo = MediaCodecList.getCodecInfoAt(i); 1149 1150 if (!codecInfo.isEncoder()) { 1151 continue; 1152 } 1153 1154 String[] types = codecInfo.getSupportedTypes(); 1155 for (int j = 0; j < types.length; j++) { 1156 if (types[j].equalsIgnoreCase(mimeType)) { 1157 return codecInfo; 1158 } 1159 } 1160 } 1161 return null; 1162 } 1163 } 1164
