1 /* 2 * Copyright (C) 2018 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 #ifndef _ACAMERA_DEVICE_H 17 #define _ACAMERA_DEVICE_H 18 19 #include <memory> 20 #include <map> 21 #include <set> 22 #include <atomic> 23 #include <utility> 24 #include <vector> 25 #include <utils/StrongPointer.h> 26 #include <utils/Mutex.h> 27 #include <utils/List.h> 28 #include <utils/Vector.h> 29 #include <android/frameworks/cameraservice/device/2.0/ICameraDeviceUser.h> 30 #include <android/frameworks/cameraservice/device/2.0/ICameraDeviceCallback.h> 31 #include <android/frameworks/cameraservice/device/2.0/types.h> 32 #include <fmq/MessageQueue.h> 33 #include <media/stagefright/foundation/ALooper.h> 34 #include <media/stagefright/foundation/AHandler.h> 35 #include <media/stagefright/foundation/AMessage.h> 36 37 #include <camera/NdkCameraManager.h> 38 #include <camera/NdkCameraCaptureSession.h> 39 #include "ACameraMetadata.h" 40 #include "utils.h" 41 42 namespace android { 43 namespace acam { 44 45 using ICameraDeviceCallback = frameworks::cameraservice::device::V2_0::ICameraDeviceCallback; 46 using ICameraDeviceUser = frameworks::cameraservice::device::V2_0::ICameraDeviceUser; 47 using CaptureResultExtras = frameworks::cameraservice::device::V2_0::CaptureResultExtras; 48 using PhysicalCaptureResultInfo = frameworks::cameraservice::device::V2_0::PhysicalCaptureResultInfo; 49 using PhysicalCameraSettings = frameworks::cameraservice::device::V2_0::PhysicalCameraSettings; 50 using SubmitInfo = frameworks::cameraservice::device::V2_0::SubmitInfo; 51 using CaptureResultExtras = frameworks::cameraservice::device::V2_0::CaptureResultExtras; 52 using ErrorCode = frameworks::cameraservice::device::V2_0::ErrorCode; 53 using FmqSizeOrMetadata = frameworks::cameraservice::device::V2_0::FmqSizeOrMetadata; 54 using StreamConfigurationMode = frameworks::cameraservice::device::V2_0::StreamConfigurationMode; 55 using Status = frameworks::cameraservice::common::V2_0::Status; 56 using ResultMetadataQueue = hardware::MessageQueue<uint8_t, hardware::kSynchronizedReadWrite>; 57 using RequestMetadataQueue = hardware::MessageQueue<uint8_t, hardware::kSynchronizedReadWrite>; 58 using CameraStatusAndId = frameworks::cameraservice::service::V2_0::CameraStatusAndId; 59 60 using hardware::hidl_vec; 61 using hardware::hidl_string; 62 using utils::native_handle_ptr_wrapper; 63 using utils::CaptureRequest; 64 using utils::OutputConfigurationWrapper; 65 66 // Wrap ACameraCaptureFailure so it can be ref-counted 67 struct CameraCaptureFailure : public RefBase, public ACameraCaptureFailure { }; 68 69 // Wrap PhysicalCaptureResultInfo so that it can be ref-counted 70 struct PhysicalCaptureResultInfoLocal { 71 std::string physicalCameraId; 72 CameraMetadata physicalMetadata; 73 }; 74 75 struct ACameraPhysicalCaptureResultInfo: public RefBase { 76 ACameraPhysicalCaptureResultInfo(const std::vector<PhysicalCaptureResultInfoLocal>& info, 77 int64_t frameNumber) : 78 mPhysicalResultInfo(info), mFrameNumber(frameNumber) {} 79 80 std::vector<PhysicalCaptureResultInfoLocal> mPhysicalResultInfo; 81 int64_t mFrameNumber; 82 }; 83 84 class CameraDevice final : public RefBase { 85 public: 86 CameraDevice(const char* id, ACameraDevice_StateCallbacks* cb, 87 sp<ACameraMetadata> chars, 88 ACameraDevice* wrapper); 89 ~CameraDevice(); 90 91 inline const char* getId() const { return mCameraId.c_str(); } 92 93 camera_status_t createCaptureRequest( 94 ACameraDevice_request_template templateId, 95 const ACameraIdList* physicalCameraIdList, 96 ACaptureRequest** request) const; 97 98 camera_status_t createCaptureSession( 99 const ACaptureSessionOutputContainer* outputs, 100 const ACaptureRequest* sessionParameters, 101 const ACameraCaptureSession_stateCallbacks* callbacks, 102 /*out*/ACameraCaptureSession** session); 103 104 camera_status_t isSessionConfigurationSupported( 105 const ACaptureSessionOutputContainer* sessionOutputContainer) const; 106 107 // Callbacks from camera service 108 class ServiceCallback : public ICameraDeviceCallback { 109 public: 110 explicit ServiceCallback(CameraDevice* device) : mDevice(device) {} 111 android::hardware::Return<void> onDeviceError(ErrorCode errorCode, 112 const CaptureResultExtras& resultExtras) override; 113 android::hardware::Return<void> onDeviceIdle() override; 114 android::hardware::Return<void> onCaptureStarted(const CaptureResultExtras& resultExtras, 115 uint64_t timestamp) override; 116 android::hardware::Return<void> onResultReceived(const FmqSizeOrMetadata& result, 117 const CaptureResultExtras& resultExtras, 118 const hidl_vec<PhysicalCaptureResultInfo>& physicalResultInfos) override; 119 android::hardware::Return<void> onRepeatingRequestError(uint64_t lastFrameNumber, 120 int32_t stoppedSequenceId) override; 121 private: 122 camera_status_t readOneResultMetadata(const FmqSizeOrMetadata& fmqSizeOrMetadata, 123 ResultMetadataQueue* metadataQueue, CameraMetadata* metadata); 124 const wp<CameraDevice> mDevice; 125 }; 126 inline sp<ICameraDeviceCallback> getServiceCallback() { 127 return mServiceCallback; 128 }; 129 130 // Camera device is only functional after remote being set 131 void setRemoteDevice(sp<ICameraDeviceUser> remote); 132 133 bool setDeviceMetadataQueues(); 134 inline ACameraDevice* getWrapper() const { return mWrapper; }; 135 136 // Stop the looper thread and unregister the handler 137 void stopLooper(); 138 139 private: 140 friend ACameraCaptureSession; 141 142 camera_status_t checkCameraClosedOrErrorLocked() const; 143 144 // device goes into fatal error state after this 145 void setCameraDeviceErrorLocked(camera_status_t error); 146 147 void disconnectLocked(sp<ACameraCaptureSession>& session); // disconnect from camera service 148 149 camera_status_t stopRepeatingLocked(); 150 151 camera_status_t flushLocked(ACameraCaptureSession*); 152 153 camera_status_t waitUntilIdleLocked(); 154 155 template<class T> 156 camera_status_t captureLocked(sp<ACameraCaptureSession> session, 157 /*optional*/T* cbs, 158 int numRequests, ACaptureRequest** requests, 159 /*optional*/int* captureSequenceId); 160 161 template<class T> 162 camera_status_t setRepeatingRequestsLocked(sp<ACameraCaptureSession> session, 163 /*optional*/T* cbs, 164 int numRequests, ACaptureRequest** requests, 165 /*optional*/int* captureSequenceId); 166 167 template<class T> 168 camera_status_t submitRequestsLocked( 169 sp<ACameraCaptureSession> session, 170 /*optional*/T* cbs, 171 int numRequests, ACaptureRequest** requests, 172 /*out*/int* captureSequenceId, 173 bool isRepeating); 174 175 void addRequestSettingsMetadata(ACaptureRequest *aCaptureRequest, sp<CaptureRequest> &req); 176 177 camera_status_t updateOutputConfigurationLocked(ACaptureSessionOutput *output); 178 179 // Since this writes to ICameraDeviceUser's fmq, clients must take care that: 180 // a) This function is called serially. 181 // b) This function is called in accordance with ICameraDeviceUser.submitRequestList, 182 // otherwise, the wrong capture request might have the wrong settings 183 // metadata associated with it. 184 camera_status_t allocateCaptureRequestLocked( 185 const ACaptureRequest* request, sp<CaptureRequest>& outReq); 186 void allocateOneCaptureRequestMetadata( 187 PhysicalCameraSettings& cameraSettings, 188 const std::string& id, const sp<ACameraMetadata>& metadata); 189 190 static ACaptureRequest* allocateACaptureRequest(sp<CaptureRequest>& req, const char* deviceId); 191 static void freeACaptureRequest(ACaptureRequest*); 192 193 // only For session to hold device lock 194 // Always grab device lock before grabbing session lock 195 void lockDeviceForSessionOps() const { mDeviceLock.lock(); }; 196 void unlockDevice() const { mDeviceLock.unlock(); }; 197 198 // For capture session to notify its end of life 199 void notifySessionEndOfLifeLocked(ACameraCaptureSession* session); 200 201 camera_status_t configureStreamsLocked(const ACaptureSessionOutputContainer* outputs, 202 const ACaptureRequest* sessionParameters); 203 204 // Input message will be posted and cleared after this returns 205 void postSessionMsgAndCleanup(sp<AMessage>& msg); 206 207 mutable Mutex mDeviceLock; 208 const hidl_string mCameraId; // Camera ID 209 const ACameraDevice_StateCallbacks mAppCallbacks; // Callback to app 210 const sp<ACameraMetadata> mChars; // Camera characteristics 211 const sp<ServiceCallback> mServiceCallback; 212 ACameraDevice* mWrapper; 213 214 // stream id -> pair of (ACameraWindowType* from application, OutputConfiguration used for 215 // camera service) 216 std::map<int, std::pair<native_handle_ptr_wrapper, OutputConfigurationWrapper>> mConfiguredOutputs; 217 218 // TODO: maybe a bool will suffice for synchronous implementation? 219 std::atomic_bool mClosing; 220 inline bool isClosed() { return mClosing; } 221 222 bool mInError = false; 223 camera_status_t mError = ACAMERA_OK; 224 void onCaptureErrorLocked( 225 ErrorCode errorCode, 226 const CaptureResultExtras& resultExtras); 227 228 bool mIdle = true; 229 // This will avoid a busy session being deleted before it's back to idle state 230 sp<ACameraCaptureSession> mBusySession; 231 232 sp<ICameraDeviceUser> mRemote; 233 234 // Looper thread to handle callback to app 235 sp<ALooper> mCbLooper; 236 // definition of handler and message 237 enum { 238 // Device state callbacks 239 kWhatOnDisconnected, // onDisconnected 240 kWhatOnError, // onError 241 // Session state callbacks 242 kWhatSessionStateCb, // onReady, onActive 243 // Capture callbacks 244 kWhatCaptureStart, // onCaptureStarted 245 kWhatCaptureResult, // onCaptureProgressed, onCaptureCompleted 246 kWhatLogicalCaptureResult, // onLogicalCameraCaptureCompleted 247 kWhatCaptureFail, // onCaptureFailed 248 kWhatLogicalCaptureFail, // onLogicalCameraCaptureFailed 249 kWhatCaptureSeqEnd, // onCaptureSequenceCompleted 250 kWhatCaptureSeqAbort, // onCaptureSequenceAborted 251 kWhatCaptureBufferLost,// onCaptureBufferLost 252 // Internal cleanup 253 kWhatCleanUpSessions // Cleanup cached sp<ACameraCaptureSession> 254 }; 255 static const char* kContextKey; 256 static const char* kDeviceKey; 257 static const char* kErrorCodeKey; 258 static const char* kCallbackFpKey; 259 static const char* kSessionSpKey; 260 static const char* kCaptureRequestKey; 261 static const char* kTimeStampKey; 262 static const char* kCaptureResultKey; 263 static const char* kPhysicalCaptureResultKey; 264 static const char* kCaptureFailureKey; 265 static const char* kSequenceIdKey; 266 static const char* kFrameNumberKey; 267 static const char* kAnwKey; 268 static const char* kFailingPhysicalCameraId; 269 270 class CallbackHandler : public AHandler { 271 public: 272 explicit CallbackHandler(const char *id); 273 void onMessageReceived(const sp<AMessage> &msg) override; 274 275 private: 276 std::string mId; 277 // This handler will cache all capture session sp until kWhatCleanUpSessions 278 // is processed. This is used to guarantee the last session reference is always 279 // being removed in callback thread without holding camera device lock 280 Vector<sp<ACameraCaptureSession>> mCachedSessions; 281 }; 282 sp<CallbackHandler> mHandler; 283 284 /*********************************** 285 * Capture session related members * 286 ***********************************/ 287 // The current active session 288 wp<ACameraCaptureSession> mCurrentSession; 289 bool mFlushing = false; 290 291 int mNextSessionId = 0; 292 // TODO: might need another looper/handler to handle callbacks from service 293 294 static const int REQUEST_ID_NONE = -1; 295 int mRepeatingSequenceId = REQUEST_ID_NONE; 296 297 // sequence id -> last frame number map 298 std::map<int32_t, int64_t> mSequenceLastFrameNumberMap; 299 300 struct CallbackHolder { 301 CallbackHolder(sp<ACameraCaptureSession> session, 302 const Vector<sp<CaptureRequest>>& requests, 303 bool isRepeating, 304 ACameraCaptureSession_captureCallbacks* cbs); 305 CallbackHolder(sp<ACameraCaptureSession> session, 306 const Vector<sp<CaptureRequest>>& requests, 307 bool isRepeating, 308 ACameraCaptureSession_logicalCamera_captureCallbacks* lcbs); 309 310 template <class T> 311 void initCaptureCallbacks(T* cbs) { 312 mContext = nullptr; 313 mOnCaptureStarted = nullptr; 314 mOnCaptureProgressed = nullptr; 315 mOnCaptureCompleted = nullptr; 316 mOnLogicalCameraCaptureCompleted = nullptr; 317 mOnLogicalCameraCaptureFailed = nullptr; 318 mOnCaptureFailed = nullptr; 319 mOnCaptureSequenceCompleted = nullptr; 320 mOnCaptureSequenceAborted = nullptr; 321 mOnCaptureBufferLost = nullptr; 322 if (cbs != nullptr) { 323 mContext = cbs->context; 324 mOnCaptureStarted = cbs->onCaptureStarted; 325 mOnCaptureProgressed = cbs->onCaptureProgressed; 326 mOnCaptureSequenceCompleted = cbs->onCaptureSequenceCompleted; 327 mOnCaptureSequenceAborted = cbs->onCaptureSequenceAborted; 328 mOnCaptureBufferLost = cbs->onCaptureBufferLost; 329 } 330 } 331 332 sp<ACameraCaptureSession> mSession; 333 Vector<sp<CaptureRequest>> mRequests; 334 const bool mIsRepeating; 335 const bool mIsLogicalCameraCallback; 336 337 void* mContext; 338 ACameraCaptureSession_captureCallback_start mOnCaptureStarted; 339 ACameraCaptureSession_captureCallback_result mOnCaptureProgressed; 340 ACameraCaptureSession_captureCallback_result mOnCaptureCompleted; 341 ACameraCaptureSession_logicalCamera_captureCallback_result mOnLogicalCameraCaptureCompleted; 342 ACameraCaptureSession_logicalCamera_captureCallback_failed mOnLogicalCameraCaptureFailed; 343 ACameraCaptureSession_captureCallback_failed mOnCaptureFailed; 344 ACameraCaptureSession_captureCallback_sequenceEnd mOnCaptureSequenceCompleted; 345 ACameraCaptureSession_captureCallback_sequenceAbort mOnCaptureSequenceAborted; 346 ACameraCaptureSession_captureCallback_bufferLost mOnCaptureBufferLost; 347 }; 348 // sequence id -> callbacks map 349 std::map<int, CallbackHolder> mSequenceCallbackMap; 350 351 static const int64_t NO_FRAMES_CAPTURED = -1; 352 class FrameNumberTracker { 353 public: 354 // TODO: Called in onResultReceived and onCaptureErrorLocked 355 void updateTracker(int64_t frameNumber, bool isError); 356 inline int64_t getCompletedFrameNumber() { return mCompletedFrameNumber; } 357 private: 358 void update(); 359 void updateCompletedFrameNumber(int64_t frameNumber); 360 361 int64_t mCompletedFrameNumber = NO_FRAMES_CAPTURED; 362 List<int64_t> mSkippedFrameNumbers; 363 std::set<int64_t> mFutureErrorSet; 364 }; 365 FrameNumberTracker mFrameNumberTracker; 366 367 void checkRepeatingSequenceCompleteLocked(const int sequenceId, const int64_t lastFrameNumber); 368 void checkAndFireSequenceCompleteLocked(); 369 370 // Misc variables 371 int32_t mShadingMapSize[2]; // const after constructor 372 int32_t mPartialResultCount; // const after constructor 373 std::shared_ptr<ResultMetadataQueue> mCaptureRequestMetadataQueue = nullptr; 374 std::shared_ptr<ResultMetadataQueue> mCaptureResultMetadataQueue = nullptr; 375 }; 376 377 } // namespace acam; 378 } // namespace android; 379 380 /** 381 * ACameraDevice opaque struct definition 382 * Leave outside of android namespace because it's NDK struct 383 */ 384 struct ACameraDevice { 385 ACameraDevice(const char* id, ACameraDevice_StateCallbacks* cb, 386 sp<ACameraMetadata> chars) : 387 mDevice(new android::acam::CameraDevice(id, cb, std::move(chars), this)) {} 388 389 ~ACameraDevice(); 390 391 /******************* 392 * NDK public APIs * 393 *******************/ 394 inline const char* getId() const { return mDevice->getId(); } 395 396 camera_status_t createCaptureRequest( 397 ACameraDevice_request_template templateId, 398 const ACameraIdList* physicalCameraIdList, 399 ACaptureRequest** request) const { 400 return mDevice->createCaptureRequest(templateId, physicalCameraIdList, request); 401 } 402 403 camera_status_t createCaptureSession( 404 const ACaptureSessionOutputContainer* outputs, 405 const ACaptureRequest* sessionParameters, 406 const ACameraCaptureSession_stateCallbacks* callbacks, 407 /*out*/ACameraCaptureSession** session) { 408 return mDevice->createCaptureSession(outputs, sessionParameters, callbacks, session); 409 } 410 411 camera_status_t isSessionConfigurationSupported( 412 const ACaptureSessionOutputContainer* sessionOutputContainer) const { 413 return mDevice->isSessionConfigurationSupported(sessionOutputContainer); 414 } 415 416 /*********************** 417 * Device interal APIs * 418 ***********************/ 419 inline android::sp<android::acam::ICameraDeviceCallback> getServiceCallback() { 420 return mDevice->getServiceCallback(); 421 }; 422 423 // Camera device is only functional after remote being set 424 inline void setRemoteDevice(android::sp<android::acam::ICameraDeviceUser> remote) { 425 mDevice->setRemoteDevice(remote); 426 } 427 inline bool setDeviceMetadataQueues() { 428 return mDevice->setDeviceMetadataQueues(); 429 } 430 private: 431 android::sp<android::acam::CameraDevice> mDevice; 432 }; 433 434 #endif // _ACAMERA_DEVICE_H 435
