1 /* 2 * Copyright (C) 2013-2018 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17 #ifndef ANDROID_SERVERS_CAMERA3DEVICE_H 18 #define ANDROID_SERVERS_CAMERA3DEVICE_H 19 20 #include <utility> 21 #include <unordered_map> 22 #include <set> 23 24 #include <utils/Condition.h> 25 #include <utils/Errors.h> 26 #include <utils/List.h> 27 #include <utils/Mutex.h> 28 #include <utils/Thread.h> 29 #include <utils/KeyedVector.h> 30 #include <utils/Timers.h> 31 32 #include <android/hardware/camera/device/3.2/ICameraDevice.h> 33 #include <android/hardware/camera/device/3.2/ICameraDeviceSession.h> 34 #include <android/hardware/camera/device/3.3/ICameraDeviceSession.h> 35 #include <android/hardware/camera/device/3.4/ICameraDeviceSession.h> 36 #include <android/hardware/camera/device/3.2/ICameraDeviceCallback.h> 37 #include <android/hardware/camera/device/3.4/ICameraDeviceCallback.h> 38 #include <fmq/MessageQueue.h> 39 #include <hardware/camera3.h> 40 41 #include <camera/CaptureResult.h> 42 43 #include "common/CameraDeviceBase.h" 44 #include "device3/StatusTracker.h" 45 #include "device3/Camera3BufferManager.h" 46 #include "device3/DistortionMapper.h" 47 #include "utils/TagMonitor.h" 48 #include "utils/LatencyHistogram.h" 49 #include <camera_metadata_hidden.h> 50 51 using android::camera3::OutputStreamInfo; 52 53 /** 54 * Function pointer types with C calling convention to 55 * use for HAL callback functions. 56 */ 57 extern "C" { 58 typedef void (callbacks_process_capture_result_t)( 59 const struct camera3_callback_ops *, 60 const camera3_capture_result_t *); 61 62 typedef void (callbacks_notify_t)( 63 const struct camera3_callback_ops *, 64 const camera3_notify_msg_t *); 65 } 66 67 namespace android { 68 69 namespace camera3 { 70 71 class Camera3Stream; 72 class Camera3ZslStream; 73 class Camera3OutputStreamInterface; 74 class Camera3StreamInterface; 75 76 } // namespace camera3 77 78 /** 79 * CameraDevice for HAL devices with version CAMERA_DEVICE_API_VERSION_3_0 or higher. 80 */ 81 class Camera3Device : 82 public CameraDeviceBase, 83 virtual public hardware::camera::device::V3_4::ICameraDeviceCallback, 84 private camera3_callback_ops { 85 public: 86 87 explicit Camera3Device(const String8& id); 88 89 virtual ~Camera3Device(); 90 91 /** 92 * CameraDeviceBase interface 93 */ 94 95 const String8& getId() const override; 96 97 metadata_vendor_id_t getVendorTagId() const override { return mVendorTagId; } 98 99 // Transitions to idle state on success. 100 status_t initialize(sp<CameraProviderManager> manager, const String8& monitorTags) override; 101 status_t disconnect() override; 102 status_t dump(int fd, const Vector<String16> &args) override; 103 const CameraMetadata& info() const override; 104 105 // Capture and setStreamingRequest will configure streams if currently in 106 // idle state 107 status_t capture(CameraMetadata &request, int64_t *lastFrameNumber = NULL) override; 108 status_t captureList(const List<const PhysicalCameraSettingsList> &requestsList, 109 const std::list<const SurfaceMap> &surfaceMaps, 110 int64_t *lastFrameNumber = NULL) override; 111 status_t setStreamingRequest(const CameraMetadata &request, 112 int64_t *lastFrameNumber = NULL) override; 113 status_t setStreamingRequestList(const List<const PhysicalCameraSettingsList> &requestsList, 114 const std::list<const SurfaceMap> &surfaceMaps, 115 int64_t *lastFrameNumber = NULL) override; 116 status_t clearStreamingRequest(int64_t *lastFrameNumber = NULL) override; 117 118 status_t waitUntilRequestReceived(int32_t requestId, nsecs_t timeout) override; 119 120 // Actual stream creation/deletion is delayed until first request is submitted 121 // If adding streams while actively capturing, will pause device before adding 122 // stream, reconfiguring device, and unpausing. If the client create a stream 123 // with nullptr consumer surface, the client must then call setConsumers() 124 // and finish the stream configuration before starting output streaming. 125 status_t createStream(sp<Surface> consumer, 126 uint32_t width, uint32_t height, int format, 127 android_dataspace dataSpace, camera3_stream_rotation_t rotation, int *id, 128 const String8& physicalCameraId, 129 std::vector<int> *surfaceIds = nullptr, 130 int streamSetId = camera3::CAMERA3_STREAM_SET_ID_INVALID, 131 bool isShared = false, uint64_t consumerUsage = 0) override; 132 status_t createStream(const std::vector<sp<Surface>>& consumers, 133 bool hasDeferredConsumer, uint32_t width, uint32_t height, int format, 134 android_dataspace dataSpace, camera3_stream_rotation_t rotation, int *id, 135 const String8& physicalCameraId, 136 std::vector<int> *surfaceIds = nullptr, 137 int streamSetId = camera3::CAMERA3_STREAM_SET_ID_INVALID, 138 bool isShared = false, uint64_t consumerUsage = 0) override; 139 140 status_t createInputStream( 141 uint32_t width, uint32_t height, int format, 142 int *id) override; 143 144 status_t getStreamInfo(int id, StreamInfo *streamInfo) override; 145 status_t setStreamTransform(int id, int transform) override; 146 147 status_t deleteStream(int id) override; 148 149 status_t configureStreams(const CameraMetadata& sessionParams, 150 int operatingMode = 151 static_cast<int>(hardware::camera::device::V3_2::StreamConfigurationMode::NORMAL_MODE)) 152 override; 153 status_t getInputBufferProducer( 154 sp<IGraphicBufferProducer> *producer) override; 155 156 status_t createDefaultRequest(int templateId, CameraMetadata *request) override; 157 158 // Transitions to the idle state on success 159 status_t waitUntilDrained() override; 160 161 status_t setNotifyCallback(wp<NotificationListener> listener) override; 162 bool willNotify3A() override; 163 status_t waitForNextFrame(nsecs_t timeout) override; 164 status_t getNextResult(CaptureResult *frame) override; 165 166 status_t triggerAutofocus(uint32_t id) override; 167 status_t triggerCancelAutofocus(uint32_t id) override; 168 status_t triggerPrecaptureMetering(uint32_t id) override; 169 170 status_t flush(int64_t *lastFrameNumber = NULL) override; 171 172 status_t prepare(int streamId) override; 173 174 status_t tearDown(int streamId) override; 175 176 status_t addBufferListenerForStream(int streamId, 177 wp<camera3::Camera3StreamBufferListener> listener) override; 178 179 status_t prepare(int maxCount, int streamId) override; 180 181 ssize_t getJpegBufferSize(uint32_t width, uint32_t height) const override; 182 ssize_t getPointCloudBufferSize() const; 183 ssize_t getRawOpaqueBufferSize(int32_t width, int32_t height) const; 184 185 // Methods called by subclasses 186 void notifyStatus(bool idle); // updates from StatusTracker 187 188 /** 189 * Set the deferred consumer surfaces to the output stream and finish the deferred 190 * consumer configuration. 191 */ 192 status_t setConsumerSurfaces( 193 int streamId, const std::vector<sp<Surface>>& consumers, 194 std::vector<int> *surfaceIds /*out*/) override; 195 196 /** 197 * Update a given stream. 198 */ 199 status_t updateStream(int streamId, const std::vector<sp<Surface>> &newSurfaces, 200 const std::vector<OutputStreamInfo> &outputInfo, 201 const std::vector<size_t> &removedSurfaceIds, 202 KeyedVector<sp<Surface>, size_t> *outputMap/*out*/); 203 204 /** 205 * Drop buffers for stream of streamId if dropping is true. If dropping is false, do not 206 * drop buffers for stream of streamId. 207 */ 208 status_t dropStreamBuffers(bool dropping, int streamId) override; 209 210 private: 211 212 // internal typedefs 213 using RequestMetadataQueue = hardware::MessageQueue<uint8_t, hardware::kSynchronizedReadWrite>; 214 using ResultMetadataQueue = hardware::MessageQueue<uint8_t, hardware::kSynchronizedReadWrite>; 215 216 static const size_t kDumpLockAttempts = 10; 217 static const size_t kDumpSleepDuration = 100000; // 0.10 sec 218 static const nsecs_t kActiveTimeout = 500000000; // 500 ms 219 static const size_t kInFlightWarnLimit = 30; 220 static const size_t kInFlightWarnLimitHighSpeed = 256; // batch size 32 * pipe depth 8 221 static const nsecs_t kDefaultExpectedDuration = 100000000; // 100 ms 222 static const nsecs_t kMinInflightDuration = 5000000000; // 5 s 223 // SCHED_FIFO priority for request submission thread in HFR mode 224 static const int kRequestThreadPriority = 1; 225 226 struct RequestTrigger; 227 // minimal jpeg buffer size: 256KB + blob header 228 static const ssize_t kMinJpegBufferSize = 256 * 1024 + sizeof(camera3_jpeg_blob); 229 // Constant to use for stream ID when one doesn't exist 230 static const int NO_STREAM = -1; 231 232 // A lock to enforce serialization on the input/configure side 233 // of the public interface. 234 // Only locked by public methods inherited from CameraDeviceBase. 235 // Not locked by methods guarded by mOutputLock, since they may act 236 // concurrently to the input/configure side of the interface. 237 // Must be locked before mLock if both will be locked by a method 238 Mutex mInterfaceLock; 239 240 // The main lock on internal state 241 Mutex mLock; 242 243 // Camera device ID 244 const String8 mId; 245 246 // Current stream configuration mode; 247 int mOperatingMode; 248 // Current session wide parameters 249 hardware::camera2::impl::CameraMetadataNative mSessionParams; 250 251 // Constant to use for no set operating mode 252 static const int NO_MODE = -1; 253 254 // Flag indicating is the current active stream configuration is constrained high speed. 255 bool mIsConstrainedHighSpeedConfiguration; 256 257 // FMQ to write result on. Must be guarded by mProcessCaptureResultLock. 258 std::unique_ptr<ResultMetadataQueue> mResultMetadataQueue; 259 260 /**** Scope for mLock ****/ 261 262 /** 263 * Adapter for legacy HAL / HIDL HAL interface calls; calls either into legacy HALv3 or the 264 * HIDL HALv3 interfaces. 265 */ 266 class HalInterface : public camera3::Camera3StreamBufferFreedListener { 267 public: 268 HalInterface(sp<hardware::camera::device::V3_2::ICameraDeviceSession> &session, 269 std::shared_ptr<RequestMetadataQueue> queue); 270 HalInterface(const HalInterface &other); 271 HalInterface(); 272 273 // Returns true if constructed with a valid device or session, and not yet cleared 274 bool valid(); 275 276 // Reset this HalInterface object (does not call close()) 277 void clear(); 278 279 // Check if HalInterface support sending requests in batch 280 bool supportBatchRequest(); 281 282 // Calls into the HAL interface 283 284 // Caller takes ownership of requestTemplate 285 status_t constructDefaultRequestSettings(camera3_request_template_t templateId, 286 /*out*/ camera_metadata_t **requestTemplate); 287 status_t configureStreams(const camera_metadata_t *sessionParams, 288 /*inout*/ camera3_stream_configuration *config, 289 const std::vector<uint32_t>& bufferSizes); 290 status_t processCaptureRequest(camera3_capture_request_t *request); 291 status_t processBatchCaptureRequests( 292 std::vector<camera3_capture_request_t*>& requests, 293 /*out*/uint32_t* numRequestProcessed); 294 status_t flush(); 295 status_t dump(int fd); 296 status_t close(); 297 298 // Find a buffer_handle_t based on frame number and stream ID 299 status_t popInflightBuffer(int32_t frameNumber, int32_t streamId, 300 /*out*/ buffer_handle_t **buffer); 301 302 // Get a vector of (frameNumber, streamId) pair of currently inflight 303 // buffers 304 void getInflightBufferKeys(std::vector<std::pair<int32_t, int32_t>>* out); 305 306 private: 307 // Always valid 308 sp<hardware::camera::device::V3_2::ICameraDeviceSession> mHidlSession; 309 // Valid if ICameraDeviceSession is @3.3 or newer 310 sp<hardware::camera::device::V3_3::ICameraDeviceSession> mHidlSession_3_3; 311 // Valid if ICameraDeviceSession is @3.4 or newer 312 sp<hardware::camera::device::V3_4::ICameraDeviceSession> mHidlSession_3_4; 313 314 std::shared_ptr<RequestMetadataQueue> mRequestMetadataQueue; 315 316 std::mutex mInflightLock; 317 318 // The output HIDL request still depends on input camera3_capture_request_t 319 // Do not free input camera3_capture_request_t before output HIDL request 320 void wrapAsHidlRequest(camera3_capture_request_t* in, 321 /*out*/hardware::camera::device::V3_2::CaptureRequest* out, 322 /*out*/std::vector<native_handle_t*>* handlesCreated); 323 324 status_t pushInflightBufferLocked(int32_t frameNumber, int32_t streamId, 325 buffer_handle_t *buffer, int acquireFence); 326 // Cache of buffer handles keyed off (frameNumber << 32 | streamId) 327 // value is a pair of (buffer_handle_t*, acquire_fence FD) 328 std::unordered_map<uint64_t, std::pair<buffer_handle_t*, int>> mInflightBufferMap; 329 330 struct BufferHasher { 331 size_t operator()(const buffer_handle_t& buf) const { 332 if (buf == nullptr) 333 return 0; 334 335 size_t result = 1; 336 result = 31 * result + buf->numFds; 337 for (int i = 0; i < buf->numFds; i++) { 338 result = 31 * result + buf->data[i]; 339 } 340 return result; 341 } 342 }; 343 344 struct BufferComparator { 345 bool operator()(const buffer_handle_t& buf1, const buffer_handle_t& buf2) const { 346 if (buf1->numFds == buf2->numFds) { 347 for (int i = 0; i < buf1->numFds; i++) { 348 if (buf1->data[i] != buf2->data[i]) { 349 return false; 350 } 351 } 352 return true; 353 } 354 return false; 355 } 356 }; 357 358 std::mutex mBufferIdMapLock; // protecting mBufferIdMaps and mNextBufferId 359 typedef std::unordered_map<const buffer_handle_t, uint64_t, 360 BufferHasher, BufferComparator> BufferIdMap; 361 // stream ID -> per stream buffer ID map 362 std::unordered_map<int, BufferIdMap> mBufferIdMaps; 363 uint64_t mNextBufferId = 1; // 0 means no buffer 364 static const uint64_t BUFFER_ID_NO_BUFFER = 0; 365 366 // method to extract buffer's unique ID 367 // TODO: we should switch to use gralloc mapper's getBackingStore API 368 // once we ran in binderized gralloc mode, but before that is ready, 369 // we need to rely on the conventional buffer queue behavior where 370 // buffer_handle_t's FD won't change. 371 // return pair of (newlySeenBuffer?, bufferId) 372 std::pair<bool, uint64_t> getBufferId(const buffer_handle_t& buf, int streamId); 373 374 virtual void onBufferFreed(int streamId, const native_handle_t* handle) override; 375 376 std::vector<std::pair<int, uint64_t>> mFreedBuffers; 377 }; 378 379 sp<HalInterface> mInterface; 380 381 CameraMetadata mDeviceInfo; 382 383 CameraMetadata mRequestTemplateCache[CAMERA3_TEMPLATE_COUNT]; 384 385 struct Size { 386 uint32_t width; 387 uint32_t height; 388 explicit Size(uint32_t w = 0, uint32_t h = 0) : width(w), height(h){} 389 }; 390 // Map from format to size. 391 Vector<Size> mSupportedOpaqueInputSizes; 392 393 enum Status { 394 STATUS_ERROR, 395 STATUS_UNINITIALIZED, 396 STATUS_UNCONFIGURED, 397 STATUS_CONFIGURED, 398 STATUS_ACTIVE 399 } mStatus; 400 401 // Only clear mRecentStatusUpdates, mStatusWaiters from waitUntilStateThenRelock 402 Vector<Status> mRecentStatusUpdates; 403 int mStatusWaiters; 404 405 Condition mStatusChanged; 406 407 // Tracking cause of fatal errors when in STATUS_ERROR 408 String8 mErrorCause; 409 410 // Mapping of stream IDs to stream instances 411 typedef KeyedVector<int, sp<camera3::Camera3OutputStreamInterface> > 412 StreamSet; 413 414 StreamSet mOutputStreams; 415 sp<camera3::Camera3Stream> mInputStream; 416 int mNextStreamId; 417 bool mNeedConfig; 418 419 int mDummyStreamId; 420 421 // Whether to send state updates upstream 422 // Pause when doing transparent reconfiguration 423 bool mPauseStateNotify; 424 425 // Need to hold on to stream references until configure completes. 426 Vector<sp<camera3::Camera3StreamInterface> > mDeletedStreams; 427 428 // Whether the HAL will send partial result 429 bool mUsePartialResult; 430 431 // Number of partial results that will be delivered by the HAL. 432 uint32_t mNumPartialResults; 433 434 /**** End scope for mLock ****/ 435 436 // The offset converting from clock domain of other subsystem 437 // (video/hardware composer) to that of camera. Assumption is that this 438 // offset won't change during the life cycle of the camera device. In other 439 // words, camera device shouldn't be open during CPU suspend. 440 nsecs_t mTimestampOffset; 441 442 class CaptureRequest : public LightRefBase<CaptureRequest> { 443 public: 444 PhysicalCameraSettingsList mSettingsList; 445 sp<camera3::Camera3Stream> mInputStream; 446 camera3_stream_buffer_t mInputBuffer; 447 Vector<sp<camera3::Camera3OutputStreamInterface> > 448 mOutputStreams; 449 SurfaceMap mOutputSurfaces; 450 CaptureResultExtras mResultExtras; 451 // The number of requests that should be submitted to HAL at a time. 452 // For example, if batch size is 8, this request and the following 7 453 // requests will be submitted to HAL at a time. The batch size for 454 // the following 7 requests will be ignored by the request thread. 455 int mBatchSize; 456 // Whether this request is from a repeating or repeating burst. 457 bool mRepeating; 458 }; 459 typedef List<sp<CaptureRequest> > RequestList; 460 461 status_t checkStatusOkToCaptureLocked(); 462 463 status_t convertMetadataListToRequestListLocked( 464 const List<const PhysicalCameraSettingsList> &metadataList, 465 const std::list<const SurfaceMap> &surfaceMaps, 466 bool repeating, 467 /*out*/ 468 RequestList *requestList); 469 470 void convertToRequestList(List<const PhysicalCameraSettingsList>& requestsList, 471 std::list<const SurfaceMap>& surfaceMaps, 472 const CameraMetadata& request); 473 474 status_t submitRequestsHelper(const List<const PhysicalCameraSettingsList> &requestsList, 475 const std::list<const SurfaceMap> &surfaceMaps, 476 bool repeating, 477 int64_t *lastFrameNumber = NULL); 478 479 480 /** 481 * Implementation of android::hardware::camera::device::V3_4::ICameraDeviceCallback 482 */ 483 hardware::Return<void> processCaptureResult_3_4( 484 const hardware::hidl_vec< 485 hardware::camera::device::V3_4::CaptureResult>& results) override; 486 hardware::Return<void> processCaptureResult( 487 const hardware::hidl_vec< 488 hardware::camera::device::V3_2::CaptureResult>& results) override; 489 hardware::Return<void> notify( 490 const hardware::hidl_vec< 491 hardware::camera::device::V3_2::NotifyMsg>& msgs) override; 492 493 // Handle one capture result. Assume that mProcessCaptureResultLock is held. 494 void processOneCaptureResultLocked( 495 const hardware::camera::device::V3_2::CaptureResult& result, 496 const hardware::hidl_vec< 497 hardware::camera::device::V3_4::PhysicalCameraMetadata> physicalCameraMetadatas); 498 status_t readOneCameraMetadataLocked(uint64_t fmqResultSize, 499 hardware::camera::device::V3_2::CameraMetadata& resultMetadata, 500 const hardware::camera::device::V3_2::CameraMetadata& result); 501 502 // Handle one notify message 503 void notify(const hardware::camera::device::V3_2::NotifyMsg& msg); 504 505 // lock to ensure only one processCaptureResult is called at a time. 506 Mutex mProcessCaptureResultLock; 507 508 /** 509 * Common initialization code shared by both HAL paths 510 * 511 * Must be called with mLock and mInterfaceLock held. 512 */ 513 status_t initializeCommonLocked(); 514 515 /** 516 * Get the last request submitted to the hal by the request thread. 517 * 518 * Must be called with mLock held. 519 */ 520 virtual CameraMetadata getLatestRequestLocked(); 521 522 /** 523 * Update the current device status and wake all waiting threads. 524 * 525 * Must be called with mLock held. 526 */ 527 void internalUpdateStatusLocked(Status status); 528 529 /** 530 * Pause processing and flush everything, but don't tell the clients. 531 * This is for reconfiguring outputs transparently when according to the 532 * CameraDeviceBase interface we shouldn't need to. 533 * Must be called with mLock and mInterfaceLock both held. 534 */ 535 status_t internalPauseAndWaitLocked(nsecs_t maxExpectedDuration); 536 537 /** 538 * Resume work after internalPauseAndWaitLocked() 539 * Must be called with mLock and mInterfaceLock both held. 540 */ 541 status_t internalResumeLocked(); 542 543 /** 544 * Wait until status tracker tells us we've transitioned to the target state 545 * set, which is either ACTIVE when active==true or IDLE (which is any 546 * non-ACTIVE state) when active==false. 547 * 548 * Needs to be called with mLock and mInterfaceLock held. This means there 549 * can ever only be one waiter at most. 550 * 551 * During the wait mLock is released. 552 * 553 */ 554 status_t waitUntilStateThenRelock(bool active, nsecs_t timeout); 555 556 /** 557 * Implementation of waitUntilDrained. On success, will transition to IDLE state. 558 * 559 * Need to be called with mLock and mInterfaceLock held. 560 */ 561 status_t waitUntilDrainedLocked(nsecs_t maxExpectedDuration); 562 563 /** 564 * Do common work for setting up a streaming or single capture request. 565 * On success, will transition to ACTIVE if in IDLE. 566 */ 567 sp<CaptureRequest> setUpRequestLocked(const PhysicalCameraSettingsList &request, 568 const SurfaceMap &surfaceMap); 569 570 /** 571 * Build a CaptureRequest request from the CameraDeviceBase request 572 * settings. 573 */ 574 sp<CaptureRequest> createCaptureRequest(const PhysicalCameraSettingsList &request, 575 const SurfaceMap &surfaceMap); 576 577 /** 578 * Pause state updates to the client application. Needed to mask out idle/active 579 * transitions during internal reconfigure 580 */ 581 void pauseStateNotify(bool enable); 582 583 /** 584 * Internally re-configure camera device using new session parameters. 585 * This will get triggered by the request thread. Be sure to call 586 * pauseStateNotify(true) before going idle in the requesting location. 587 */ 588 bool reconfigureCamera(const CameraMetadata& sessionParams); 589 590 /** 591 * Filter stream session parameters and configure camera HAL. 592 */ 593 status_t filterParamsAndConfigureLocked(const CameraMetadata& sessionParams, 594 int operatingMode); 595 596 /** 597 * Take the currently-defined set of streams and configure the HAL to use 598 * them. This is a long-running operation (may be several hundered ms). 599 */ 600 status_t configureStreamsLocked(int operatingMode, 601 const CameraMetadata& sessionParams, bool notifyRequestThread = true); 602 603 /** 604 * Cancel stream configuration that did not finish successfully. 605 */ 606 void cancelStreamsConfigurationLocked(); 607 608 /** 609 * Add a dummy stream to the current stream set as a workaround for 610 * not allowing 0 streams in the camera HAL spec. 611 */ 612 status_t addDummyStreamLocked(); 613 614 /** 615 * Remove a dummy stream if the current config includes real streams. 616 */ 617 status_t tryRemoveDummyStreamLocked(); 618 619 /** 620 * Set device into an error state due to some fatal failure, and set an 621 * error message to indicate why. Only the first call's message will be 622 * used. The message is also sent to the log. 623 */ 624 void setErrorState(const char *fmt, ...); 625 void setErrorStateV(const char *fmt, va_list args); 626 void setErrorStateLocked(const char *fmt, ...); 627 void setErrorStateLockedV(const char *fmt, va_list args); 628 629 /** 630 * Debugging trylock/spin method 631 * Try to acquire a lock a few times with sleeps between before giving up. 632 */ 633 bool tryLockSpinRightRound(Mutex& lock); 634 635 /** 636 * Helper function to determine if an input size for implementation defined 637 * format is supported. 638 */ 639 bool isOpaqueInputSizeSupported(uint32_t width, uint32_t height); 640 641 /** 642 * Helper function to get the largest Jpeg resolution (in area) 643 * Return Size(0, 0) if static metatdata is invalid 644 */ 645 Size getMaxJpegResolution() const; 646 647 /** 648 * Helper function to get the offset between MONOTONIC and BOOTTIME 649 * timestamp. 650 */ 651 static nsecs_t getMonoToBoottimeOffset(); 652 653 /** 654 * Helper functions to map between framework and HIDL values 655 */ 656 static hardware::graphics::common::V1_0::PixelFormat mapToPixelFormat(int frameworkFormat); 657 static hardware::camera::device::V3_2::DataspaceFlags mapToHidlDataspace( 658 android_dataspace dataSpace); 659 static hardware::camera::device::V3_2::BufferUsageFlags mapToConsumerUsage(uint64_t usage); 660 static hardware::camera::device::V3_2::StreamRotation mapToStreamRotation( 661 camera3_stream_rotation_t rotation); 662 // Returns a negative error code if the passed-in operation mode is not valid. 663 static status_t mapToStreamConfigurationMode(camera3_stream_configuration_mode_t operationMode, 664 /*out*/ hardware::camera::device::V3_2::StreamConfigurationMode *mode); 665 static camera3_buffer_status_t mapHidlBufferStatus(hardware::camera::device::V3_2::BufferStatus status); 666 static int mapToFrameworkFormat(hardware::graphics::common::V1_0::PixelFormat pixelFormat); 667 static android_dataspace mapToFrameworkDataspace( 668 hardware::camera::device::V3_2::DataspaceFlags); 669 static uint64_t mapConsumerToFrameworkUsage( 670 hardware::camera::device::V3_2::BufferUsageFlags usage); 671 static uint64_t mapProducerToFrameworkUsage( 672 hardware::camera::device::V3_2::BufferUsageFlags usage); 673 674 struct RequestTrigger { 675 // Metadata tag number, e.g. android.control.aePrecaptureTrigger 676 uint32_t metadataTag; 677 // Metadata value, e.g. 'START' or the trigger ID 678 int32_t entryValue; 679 680 // The last part of the fully qualified path, e.g. afTrigger 681 const char *getTagName() const { 682 return get_camera_metadata_tag_name(metadataTag) ?: "NULL"; 683 } 684 685 // e.g. TYPE_BYTE, TYPE_INT32, etc. 686 int getTagType() const { 687 return get_camera_metadata_tag_type(metadataTag); 688 } 689 }; 690 691 /** 692 * Thread for managing capture request submission to HAL device. 693 */ 694 class RequestThread : public Thread { 695 696 public: 697 698 RequestThread(wp<Camera3Device> parent, 699 sp<camera3::StatusTracker> statusTracker, 700 sp<HalInterface> interface, const Vector<int32_t>& sessionParamKeys); 701 ~RequestThread(); 702 703 void setNotificationListener(wp<NotificationListener> listener); 704 705 /** 706 * Call after stream (re)-configuration is completed. 707 */ 708 void configurationComplete(bool isConstrainedHighSpeed, 709 const CameraMetadata& sessionParams); 710 711 /** 712 * Set or clear the list of repeating requests. Does not block 713 * on either. Use waitUntilPaused to wait until request queue 714 * has emptied out. 715 */ 716 status_t setRepeatingRequests(const RequestList& requests, 717 /*out*/ 718 int64_t *lastFrameNumber = NULL); 719 status_t clearRepeatingRequests(/*out*/ 720 int64_t *lastFrameNumber = NULL); 721 722 status_t queueRequestList(List<sp<CaptureRequest> > &requests, 723 /*out*/ 724 int64_t *lastFrameNumber = NULL); 725 726 /** 727 * Remove all queued and repeating requests, and pending triggers 728 */ 729 status_t clear(/*out*/int64_t *lastFrameNumber = NULL); 730 731 /** 732 * Flush all pending requests in HAL. 733 */ 734 status_t flush(); 735 736 /** 737 * Queue a trigger to be dispatched with the next outgoing 738 * process_capture_request. The settings for that request only 739 * will be temporarily rewritten to add the trigger tag/value. 740 * Subsequent requests will not be rewritten (for this tag). 741 */ 742 status_t queueTrigger(RequestTrigger trigger[], size_t count); 743 744 /** 745 * Pause/unpause the capture thread. Doesn't block, so use 746 * waitUntilPaused to wait until the thread is paused. 747 */ 748 void setPaused(bool paused); 749 750 /** 751 * Wait until thread processes the capture request with settings' 752 * android.request.id == requestId. 753 * 754 * Returns TIMED_OUT in case the thread does not process the request 755 * within the timeout. 756 */ 757 status_t waitUntilRequestProcessed(int32_t requestId, nsecs_t timeout); 758 759 /** 760 * Shut down the thread. Shutdown is asynchronous, so thread may 761 * still be running once this method returns. 762 */ 763 virtual void requestExit(); 764 765 /** 766 * Get the latest request that was sent to the HAL 767 * with process_capture_request. 768 */ 769 CameraMetadata getLatestRequest() const; 770 771 /** 772 * Returns true if the stream is a target of any queued or repeating 773 * capture request 774 */ 775 bool isStreamPending(sp<camera3::Camera3StreamInterface>& stream); 776 777 /** 778 * Returns true if the surface is a target of any queued or repeating 779 * capture request 780 */ 781 bool isOutputSurfacePending(int streamId, size_t surfaceId); 782 783 // dump processCaptureRequest latency 784 void dumpCaptureRequestLatency(int fd, const char* name) { 785 mRequestLatency.dump(fd, name); 786 } 787 788 protected: 789 790 virtual bool threadLoop(); 791 792 private: 793 static const String8& getId(const wp<Camera3Device> &device); 794 795 status_t queueTriggerLocked(RequestTrigger trigger); 796 // Mix-in queued triggers into this request 797 int32_t insertTriggers(const sp<CaptureRequest> &request); 798 // Purge the queued triggers from this request, 799 // restoring the old field values for those tags. 800 status_t removeTriggers(const sp<CaptureRequest> &request); 801 802 // HAL workaround: Make sure a trigger ID always exists if 803 // a trigger does 804 status_t addDummyTriggerIds(const sp<CaptureRequest> &request); 805 806 static const nsecs_t kRequestTimeout = 50e6; // 50 ms 807 808 // Used to prepare a batch of requests. 809 struct NextRequest { 810 sp<CaptureRequest> captureRequest; 811 camera3_capture_request_t halRequest; 812 Vector<camera3_stream_buffer_t> outputBuffers; 813 bool submitted; 814 }; 815 816 // Wait for the next batch of requests and put them in mNextRequests. mNextRequests will 817 // be empty if it times out. 818 void waitForNextRequestBatch(); 819 820 // Waits for a request, or returns NULL if times out. Must be called with mRequestLock hold. 821 sp<CaptureRequest> waitForNextRequestLocked(); 822 823 // Prepare HAL requests and output buffers in mNextRequests. Return TIMED_OUT if getting any 824 // output buffer timed out. If an error is returned, the caller should clean up the pending 825 // request batch. 826 status_t prepareHalRequests(); 827 828 // Return buffers, etc, for requests in mNextRequests that couldn't be fully constructed and 829 // send request errors if sendRequestError is true. The buffers will be returned in the 830 // ERROR state to mark them as not having valid data. mNextRequests will be cleared. 831 void cleanUpFailedRequests(bool sendRequestError); 832 833 // Stop the repeating request if any of its output streams is abandoned. 834 void checkAndStopRepeatingRequest(); 835 836 // Release physical camera settings and camera id resources. 837 void cleanupPhysicalSettings(sp<CaptureRequest> request, 838 /*out*/camera3_capture_request_t *halRequest); 839 840 // Pause handling 841 bool waitIfPaused(); 842 void unpauseForNewRequests(); 843 844 // Relay error to parent device object setErrorState 845 void setErrorState(const char *fmt, ...); 846 847 // If the input request is in mRepeatingRequests. Must be called with mRequestLock hold 848 bool isRepeatingRequestLocked(const sp<CaptureRequest>&); 849 850 // Clear repeating requests. Must be called with mRequestLock held. 851 status_t clearRepeatingRequestsLocked(/*out*/ int64_t *lastFrameNumber = NULL); 852 853 // send request in mNextRequests to HAL one by one. Return true = sucssess 854 bool sendRequestsOneByOne(); 855 856 // send request in mNextRequests to HAL in a batch. Return true = sucssess 857 bool sendRequestsBatch(); 858 859 // Calculate the expected maximum duration for a request 860 nsecs_t calculateMaxExpectedDuration(const camera_metadata_t *request); 861 862 // Check and update latest session parameters based on the current request settings. 863 bool updateSessionParameters(const CameraMetadata& settings); 864 865 // Check whether FPS range session parameter re-configuration is needed in constrained 866 // high speed recording camera sessions. 867 bool skipHFRTargetFPSUpdate(int32_t tag, const camera_metadata_ro_entry_t& newEntry, 868 const camera_metadata_entry_t& currentEntry); 869 870 // Re-configure camera using the latest session parameters. 871 bool reconfigureCamera(); 872 873 wp<Camera3Device> mParent; 874 wp<camera3::StatusTracker> mStatusTracker; 875 sp<HalInterface> mInterface; 876 877 wp<NotificationListener> mListener; 878 879 const String8& mId; // The camera ID 880 int mStatusId; // The RequestThread's component ID for 881 // status tracking 882 883 Mutex mRequestLock; 884 Condition mRequestSignal; 885 RequestList mRequestQueue; 886 RequestList mRepeatingRequests; 887 // The next batch of requests being prepped for submission to the HAL, no longer 888 // on the request queue. Read-only even with mRequestLock held, outside 889 // of threadLoop 890 Vector<NextRequest> mNextRequests; 891 892 // To protect flush() and sending a request batch to HAL. 893 Mutex mFlushLock; 894 895 bool mReconfigured; 896 897 // Used by waitIfPaused, waitForNextRequest, and waitUntilPaused 898 Mutex mPauseLock; 899 bool mDoPause; 900 Condition mDoPauseSignal; 901 bool mPaused; 902 Condition mPausedSignal; 903 904 sp<CaptureRequest> mPrevRequest; 905 int32_t mPrevTriggers; 906 907 uint32_t mFrameNumber; 908 909 mutable Mutex mLatestRequestMutex; 910 Condition mLatestRequestSignal; 911 // android.request.id for latest process_capture_request 912 int32_t mLatestRequestId; 913 CameraMetadata mLatestRequest; 914 915 typedef KeyedVector<uint32_t/*tag*/, RequestTrigger> TriggerMap; 916 Mutex mTriggerMutex; 917 TriggerMap mTriggerMap; 918 TriggerMap mTriggerRemovedMap; 919 TriggerMap mTriggerReplacedMap; 920 uint32_t mCurrentAfTriggerId; 921 uint32_t mCurrentPreCaptureTriggerId; 922 923 int64_t mRepeatingLastFrameNumber; 924 925 // Flag indicating if we should prepare video stream for video requests. 926 bool mPrepareVideoStream; 927 928 bool mConstrainedMode; 929 930 static const int32_t kRequestLatencyBinSize = 40; // in ms 931 CameraLatencyHistogram mRequestLatency; 932 933 Vector<int32_t> mSessionParamKeys; 934 CameraMetadata mLatestSessionParams; 935 }; 936 sp<RequestThread> mRequestThread; 937 938 /** 939 * In-flight queue for tracking completion of capture requests. 940 */ 941 942 struct InFlightRequest { 943 // Set by notify() SHUTTER call. 944 nsecs_t shutterTimestamp; 945 // Set by process_capture_result(). 946 nsecs_t sensorTimestamp; 947 int requestStatus; 948 // Set by process_capture_result call with valid metadata 949 bool haveResultMetadata; 950 // Decremented by calls to process_capture_result with valid output 951 // and input buffers 952 int numBuffersLeft; 953 CaptureResultExtras resultExtras; 954 // If this request has any input buffer 955 bool hasInputBuffer; 956 957 // The last metadata that framework receives from HAL and 958 // not yet send out because the shutter event hasn't arrived. 959 // It's added by process_capture_result and sent when framework 960 // receives the shutter event. 961 CameraMetadata pendingMetadata; 962 963 // The metadata of the partial results that framework receives from HAL so far 964 // and has sent out. 965 CameraMetadata collectedPartialResult; 966 967 // Buffers are added by process_capture_result when output buffers 968 // return from HAL but framework has not yet received the shutter 969 // event. They will be returned to the streams when framework receives 970 // the shutter event. 971 Vector<camera3_stream_buffer_t> pendingOutputBuffers; 972 973 // Whether this inflight request's shutter and result callback are to be 974 // called. The policy is that if the request is the last one in the constrained 975 // high speed recording request list, this flag will be true. If the request list 976 // is not for constrained high speed recording, this flag will also be true. 977 bool hasCallback; 978 979 // Maximum expected frame duration for this request. 980 // For manual captures, equal to the max of requested exposure time and frame duration 981 // For auto-exposure modes, equal to 1/(lower end of target FPS range) 982 nsecs_t maxExpectedDuration; 983 984 // Whether the result metadata for this request is to be skipped. The 985 // result metadata should be skipped in the case of 986 // REQUEST/RESULT error. 987 bool skipResultMetadata; 988 989 // The physical camera ids being requested. 990 std::set<String8> physicalCameraIds; 991 992 // Map of physicalCameraId <-> Metadata 993 std::vector<PhysicalCaptureResultInfo> physicalMetadatas; 994 995 // Default constructor needed by KeyedVector 996 InFlightRequest() : 997 shutterTimestamp(0), 998 sensorTimestamp(0), 999 requestStatus(OK), 1000 haveResultMetadata(false), 1001 numBuffersLeft(0), 1002 hasInputBuffer(false), 1003 hasCallback(true), 1004 maxExpectedDuration(kDefaultExpectedDuration), 1005 skipResultMetadata(false) { 1006 } 1007 1008 InFlightRequest(int numBuffers, CaptureResultExtras extras, bool hasInput, 1009 bool hasAppCallback, nsecs_t maxDuration, 1010 const std::set<String8>& physicalCameraIdSet) : 1011 shutterTimestamp(0), 1012 sensorTimestamp(0), 1013 requestStatus(OK), 1014 haveResultMetadata(false), 1015 numBuffersLeft(numBuffers), 1016 resultExtras(extras), 1017 hasInputBuffer(hasInput), 1018 hasCallback(hasAppCallback), 1019 maxExpectedDuration(maxDuration), 1020 skipResultMetadata(false), 1021 physicalCameraIds(physicalCameraIdSet) { 1022 } 1023 }; 1024 1025 // Map from frame number to the in-flight request state 1026 typedef KeyedVector<uint32_t, InFlightRequest> InFlightMap; 1027 1028 1029 Mutex mInFlightLock; // Protects mInFlightMap and 1030 // mExpectedInflightDuration 1031 InFlightMap mInFlightMap; 1032 nsecs_t mExpectedInflightDuration = 0; 1033 int mInFlightStatusId; 1034 1035 1036 status_t registerInFlight(uint32_t frameNumber, 1037 int32_t numBuffers, CaptureResultExtras resultExtras, bool hasInput, 1038 bool callback, nsecs_t maxExpectedDuration, std::set<String8>& physicalCameraIds); 1039 1040 /** 1041 * Returns the maximum expected time it'll take for all currently in-flight 1042 * requests to complete, based on their settings 1043 */ 1044 nsecs_t getExpectedInFlightDuration(); 1045 1046 /** 1047 * Tracking for idle detection 1048 */ 1049 sp<camera3::StatusTracker> mStatusTracker; 1050 1051 /** 1052 * Graphic buffer manager for output streams. Each device has a buffer manager, which is used 1053 * by the output streams to get and return buffers if these streams are registered to this 1054 * buffer manager. 1055 */ 1056 sp<camera3::Camera3BufferManager> mBufferManager; 1057 1058 /** 1059 * Thread for preparing streams 1060 */ 1061 class PreparerThread : private Thread, public virtual RefBase { 1062 public: 1063 PreparerThread(); 1064 ~PreparerThread(); 1065 1066 void setNotificationListener(wp<NotificationListener> listener); 1067 1068 /** 1069 * Queue up a stream to be prepared. Streams are processed by a background thread in FIFO 1070 * order. Pre-allocate up to maxCount buffers for the stream, or the maximum number needed 1071 * for the pipeline if maxCount is ALLOCATE_PIPELINE_MAX. 1072 */ 1073 status_t prepare(int maxCount, sp<camera3::Camera3StreamInterface>& stream); 1074 1075 /** 1076 * Cancel all current and pending stream preparation 1077 */ 1078 status_t clear(); 1079 1080 /** 1081 * Pause all preparation activities 1082 */ 1083 void pause(); 1084 1085 /** 1086 * Resume preparation activities 1087 */ 1088 status_t resume(); 1089 1090 private: 1091 Mutex mLock; 1092 Condition mThreadActiveSignal; 1093 1094 virtual bool threadLoop(); 1095 1096 // Guarded by mLock 1097 1098 wp<NotificationListener> mListener; 1099 std::unordered_map<int, sp<camera3::Camera3StreamInterface> > mPendingStreams; 1100 bool mActive; 1101 bool mCancelNow; 1102 1103 // Only accessed by threadLoop and the destructor 1104 1105 sp<camera3::Camera3StreamInterface> mCurrentStream; 1106 int mCurrentMaxCount; 1107 bool mCurrentPrepareComplete; 1108 }; 1109 sp<PreparerThread> mPreparerThread; 1110 1111 /** 1112 * Output result queue and current HAL device 3A state 1113 */ 1114 1115 // Lock for output side of device 1116 Mutex mOutputLock; 1117 1118 /**** Scope for mOutputLock ****/ 1119 // the minimal frame number of the next non-reprocess result 1120 uint32_t mNextResultFrameNumber; 1121 // the minimal frame number of the next reprocess result 1122 uint32_t mNextReprocessResultFrameNumber; 1123 // the minimal frame number of the next non-reprocess shutter 1124 uint32_t mNextShutterFrameNumber; 1125 // the minimal frame number of the next reprocess shutter 1126 uint32_t mNextReprocessShutterFrameNumber; 1127 List<CaptureResult> mResultQueue; 1128 Condition mResultSignal; 1129 wp<NotificationListener> mListener; 1130 1131 /**** End scope for mOutputLock ****/ 1132 1133 /** 1134 * Callback functions from HAL device 1135 */ 1136 void processCaptureResult(const camera3_capture_result *result); 1137 1138 void notify(const camera3_notify_msg *msg); 1139 1140 // Specific notify handlers 1141 void notifyError(const camera3_error_msg_t &msg, 1142 sp<NotificationListener> listener); 1143 void notifyShutter(const camera3_shutter_msg_t &msg, 1144 sp<NotificationListener> listener); 1145 1146 // helper function to return the output buffers to the streams. 1147 void returnOutputBuffers(const camera3_stream_buffer_t *outputBuffers, 1148 size_t numBuffers, nsecs_t timestamp); 1149 1150 // Send a partial capture result. 1151 void sendPartialCaptureResult(const camera_metadata_t * partialResult, 1152 const CaptureResultExtras &resultExtras, uint32_t frameNumber); 1153 1154 // Send a total capture result given the pending metadata and result extras, 1155 // partial results, and the frame number to the result queue. 1156 void sendCaptureResult(CameraMetadata &pendingMetadata, 1157 CaptureResultExtras &resultExtras, 1158 CameraMetadata &collectedPartialResult, uint32_t frameNumber, 1159 bool reprocess, const std::vector<PhysicalCaptureResultInfo>& physicalMetadatas); 1160 1161 bool isLastFullResult(const InFlightRequest& inFlightRequest); 1162 1163 // Insert the result to the result queue after updating frame number and overriding AE 1164 // trigger cancel. 1165 // mOutputLock must be held when calling this function. 1166 void insertResultLocked(CaptureResult *result, uint32_t frameNumber); 1167 1168 /**** Scope for mInFlightLock ****/ 1169 1170 // Remove the in-flight map entry of the given index from mInFlightMap. 1171 // It must only be called with mInFlightLock held. 1172 void removeInFlightMapEntryLocked(int idx); 1173 // Remove the in-flight request of the given index from mInFlightMap 1174 // if it's no longer needed. It must only be called with mInFlightLock held. 1175 void removeInFlightRequestIfReadyLocked(int idx); 1176 // Remove all in-flight requests and return all buffers. 1177 // This is used after HAL interface is closed to cleanup any request/buffers 1178 // not returned by HAL. 1179 void flushInflightRequests(); 1180 1181 /**** End scope for mInFlightLock ****/ 1182 1183 /** 1184 * Distortion correction support 1185 */ 1186 1187 camera3::DistortionMapper mDistortionMapper; 1188 1189 // Debug tracker for metadata tag value changes 1190 // - Enabled with the -m <taglist> option to dumpsys, such as 1191 // dumpsys -m android.control.aeState,android.control.aeMode 1192 // - Disabled with -m off 1193 // - dumpsys -m 3a is a shortcut for ae/af/awbMode, State, and Triggers 1194 TagMonitor mTagMonitor; 1195 1196 void monitorMetadata(TagMonitor::eventSource source, int64_t frameNumber, 1197 nsecs_t timestamp, const CameraMetadata& metadata); 1198 1199 metadata_vendor_id_t mVendorTagId; 1200 1201 // Cached last requested template id 1202 int mLastTemplateId; 1203 1204 /** 1205 * Static callback forwarding methods from HAL to instance 1206 */ 1207 static callbacks_process_capture_result_t sProcessCaptureResult; 1208 1209 static callbacks_notify_t sNotify; 1210 1211 }; // class Camera3Device 1212 1213 }; // namespace android 1214 1215 #endif 1216
