1 /* 2 * Copyright (C) 2018 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17 #ifndef ANDROID_HARDWARE_CAMERA_DEVICE_V3_4_EXTCAMERADEVICE3SESSION_H 18 #define ANDROID_HARDWARE_CAMERA_DEVICE_V3_4_EXTCAMERADEVICE3SESSION_H 19 20 #include <android/hardware/camera/device/3.2/ICameraDevice.h> 21 #include <android/hardware/camera/device/3.4/ICameraDeviceSession.h> 22 #include <fmq/MessageQueue.h> 23 #include <hidl/MQDescriptor.h> 24 #include <hidl/Status.h> 25 #include <include/convert.h> 26 #include <chrono> 27 #include <condition_variable> 28 #include <list> 29 #include <unordered_map> 30 #include <unordered_set> 31 #include "CameraMetadata.h" 32 #include "HandleImporter.h" 33 #include "Exif.h" 34 #include "utils/KeyedVector.h" 35 #include "utils/Mutex.h" 36 #include "utils/Thread.h" 37 #include "android-base/unique_fd.h" 38 #include "ExternalCameraUtils.h" 39 40 namespace android { 41 namespace hardware { 42 namespace camera { 43 namespace device { 44 namespace V3_4 { 45 namespace implementation { 46 47 using ::android::hardware::camera::device::V3_2::BufferCache; 48 using ::android::hardware::camera::device::V3_2::BufferStatus; 49 using ::android::hardware::camera::device::V3_2::CameraMetadata; 50 using ::android::hardware::camera::device::V3_2::CaptureRequest; 51 using ::android::hardware::camera::device::V3_2::CaptureResult; 52 using ::android::hardware::camera::device::V3_2::ErrorCode; 53 using ::android::hardware::camera::device::V3_2::ICameraDeviceCallback; 54 using ::android::hardware::camera::device::V3_2::MsgType; 55 using ::android::hardware::camera::device::V3_2::NotifyMsg; 56 using ::android::hardware::camera::device::V3_2::RequestTemplate; 57 using ::android::hardware::camera::device::V3_2::Stream; 58 using ::android::hardware::camera::device::V3_4::StreamConfiguration; 59 using ::android::hardware::camera::device::V3_2::StreamConfigurationMode; 60 using ::android::hardware::camera::device::V3_2::StreamRotation; 61 using ::android::hardware::camera::device::V3_2::StreamType; 62 using ::android::hardware::camera::device::V3_2::DataspaceFlags; 63 using ::android::hardware::camera::device::V3_2::CameraBlob; 64 using ::android::hardware::camera::device::V3_2::CameraBlobId; 65 using ::android::hardware::camera::device::V3_4::HalStreamConfiguration; 66 using ::android::hardware::camera::device::V3_4::ICameraDeviceSession; 67 using ::android::hardware::camera::common::V1_0::Status; 68 using ::android::hardware::camera::common::V1_0::helper::HandleImporter; 69 using ::android::hardware::camera::common::V1_0::helper::ExifUtils; 70 using ::android::hardware::camera::external::common::ExternalCameraConfig; 71 using ::android::hardware::camera::external::common::Size; 72 using ::android::hardware::camera::external::common::SizeHasher; 73 using ::android::hardware::graphics::common::V1_0::BufferUsage; 74 using ::android::hardware::graphics::common::V1_0::Dataspace; 75 using ::android::hardware::graphics::common::V1_0::PixelFormat; 76 using ::android::hardware::kSynchronizedReadWrite; 77 using ::android::hardware::MessageQueue; 78 using ::android::hardware::MQDescriptorSync; 79 using ::android::hardware::Return; 80 using ::android::hardware::Void; 81 using ::android::hardware::hidl_vec; 82 using ::android::hardware::hidl_string; 83 using ::android::sp; 84 using ::android::Mutex; 85 using ::android::base::unique_fd; 86 87 struct ExternalCameraDeviceSession : public virtual RefBase { 88 89 ExternalCameraDeviceSession(const sp<ICameraDeviceCallback>&, 90 const ExternalCameraConfig& cfg, 91 const std::vector<SupportedV4L2Format>& sortedFormats, 92 const CroppingType& croppingType, 93 const common::V1_0::helper::CameraMetadata& chars, 94 const std::string& cameraId, 95 unique_fd v4l2Fd); 96 virtual ~ExternalCameraDeviceSession(); 97 // Call by CameraDevice to dump active device states 98 void dumpState(const native_handle_t*); 99 // Caller must use this method to check if CameraDeviceSession ctor failed 100 bool isInitFailed() { return mInitFail; } 101 bool isClosed(); 102 103 // Retrieve the HIDL interface, split into its own class to avoid inheritance issues when 104 // dealing with minor version revs and simultaneous implementation and interface inheritance 105 virtual sp<ICameraDeviceSession> getInterface() { 106 return new TrampolineSessionInterface_3_4(this); 107 } 108 109 static const int kMaxProcessedStream = 2; 110 static const int kMaxStallStream = 1; 111 static const uint32_t kMaxBytesPerPixel = 2; 112 113 protected: 114 115 // Methods from ::android::hardware::camera::device::V3_2::ICameraDeviceSession follow 116 117 Return<void> constructDefaultRequestSettings( 118 RequestTemplate, 119 ICameraDeviceSession::constructDefaultRequestSettings_cb _hidl_cb); 120 121 Return<void> configureStreams( 122 const V3_2::StreamConfiguration&, 123 ICameraDeviceSession::configureStreams_cb); 124 125 Return<void> getCaptureRequestMetadataQueue( 126 ICameraDeviceSession::getCaptureRequestMetadataQueue_cb); 127 128 Return<void> getCaptureResultMetadataQueue( 129 ICameraDeviceSession::getCaptureResultMetadataQueue_cb); 130 131 Return<void> processCaptureRequest( 132 const hidl_vec<CaptureRequest>&, 133 const hidl_vec<BufferCache>&, 134 ICameraDeviceSession::processCaptureRequest_cb); 135 136 Return<Status> flush(); 137 Return<void> close(); 138 139 Return<void> configureStreams_3_3( 140 const V3_2::StreamConfiguration&, 141 ICameraDeviceSession::configureStreams_3_3_cb); 142 143 Return<void> configureStreams_3_4( 144 const V3_4::StreamConfiguration& requestedConfiguration, 145 ICameraDeviceSession::configureStreams_3_4_cb _hidl_cb); 146 147 Return<void> processCaptureRequest_3_4( 148 const hidl_vec<V3_4::CaptureRequest>& requests, 149 const hidl_vec<V3_2::BufferCache>& cachesToRemove, 150 ICameraDeviceSession::processCaptureRequest_3_4_cb _hidl_cb); 151 152 protected: 153 struct HalStreamBuffer { 154 int32_t streamId; 155 uint64_t bufferId; 156 uint32_t width; 157 uint32_t height; 158 PixelFormat format; 159 V3_2::BufferUsageFlags usage; 160 buffer_handle_t* bufPtr; 161 int acquireFence; 162 bool fenceTimeout; 163 }; 164 165 struct HalRequest { 166 uint32_t frameNumber; 167 common::V1_0::helper::CameraMetadata setting; 168 sp<V4L2Frame> frameIn; 169 nsecs_t shutterTs; 170 std::vector<HalStreamBuffer> buffers; 171 }; 172 173 Status constructDefaultRequestSettingsRaw(RequestTemplate type, 174 V3_2::CameraMetadata *outMetadata); 175 176 bool initialize(); 177 Status initStatus() const; 178 status_t initDefaultRequests(); 179 status_t fillCaptureResult(common::V1_0::helper::CameraMetadata& md, nsecs_t timestamp); 180 Status configureStreams(const V3_2::StreamConfiguration&, V3_3::HalStreamConfiguration* out); 181 // fps = 0.0 means default, which is 182 // slowest fps that is at least 30, or fastest fps if 30 is not supported 183 int configureV4l2StreamLocked(const SupportedV4L2Format& fmt, double fps = 0.0); 184 int v4l2StreamOffLocked(); 185 int setV4l2FpsLocked(double fps); 186 187 // TODO: change to unique_ptr for better tracking 188 sp<V4L2Frame> dequeueV4l2FrameLocked(/*out*/nsecs_t* shutterTs); // Called with mLock hold 189 void enqueueV4l2Frame(const sp<V4L2Frame>&); 190 191 // Check if input Stream is one of supported stream setting on this device 192 bool isSupported(const Stream&); 193 194 // Validate and import request's output buffers and acquire fence 195 Status importRequest( 196 const CaptureRequest& request, 197 hidl_vec<buffer_handle_t*>& allBufPtrs, 198 hidl_vec<int>& allFences); 199 static void cleanupInflightFences( 200 hidl_vec<int>& allFences, size_t numFences); 201 void cleanupBuffersLocked(int id); 202 void updateBufferCaches(const hidl_vec<BufferCache>& cachesToRemove); 203 204 Status processOneCaptureRequest(const CaptureRequest& request); 205 206 Status processCaptureResult(std::shared_ptr<HalRequest>&); 207 Status processCaptureRequestError(const std::shared_ptr<HalRequest>&); 208 void notifyShutter(uint32_t frameNumber, nsecs_t shutterTs); 209 void notifyError(uint32_t frameNumber, int32_t streamId, ErrorCode ec); 210 void invokeProcessCaptureResultCallback( 211 hidl_vec<CaptureResult> &results, bool tryWriteFmq); 212 static void freeReleaseFences(hidl_vec<CaptureResult>&); 213 214 Size getMaxJpegResolution() const; 215 Size getMaxThumbResolution() const; 216 217 ssize_t getJpegBufferSize(uint32_t width, uint32_t height) const; 218 219 int waitForV4L2BufferReturnLocked(std::unique_lock<std::mutex>& lk); 220 221 class OutputThread : public android::Thread { 222 public: 223 OutputThread(wp<ExternalCameraDeviceSession> parent, CroppingType); 224 ~OutputThread(); 225 226 Status allocateIntermediateBuffers( 227 const Size& v4lSize, const Size& thumbSize, 228 const hidl_vec<Stream>& streams); 229 Status submitRequest(const std::shared_ptr<HalRequest>&); 230 void flush(); 231 void dump(int fd); 232 virtual bool threadLoop() override; 233 234 void setExifMakeModel(const std::string& make, const std::string& model); 235 private: 236 static const uint32_t FLEX_YUV_GENERIC = static_cast<uint32_t>('F') | 237 static_cast<uint32_t>('L') << 8 | static_cast<uint32_t>('E') << 16 | 238 static_cast<uint32_t>('X') << 24; 239 // returns FLEX_YUV_GENERIC for formats other than YV12/YU12/NV12/NV21 240 static uint32_t getFourCcFromLayout(const YCbCrLayout&); 241 static int getCropRect( 242 CroppingType ct, const Size& inSize, const Size& outSize, IMapper::Rect* out); 243 244 static const int kFlushWaitTimeoutSec = 3; // 3 sec 245 static const int kReqWaitTimeoutMs = 33; // 33ms 246 static const int kReqWaitTimesMax = 90; // 33ms * 90 ~= 3 sec 247 248 void waitForNextRequest(std::shared_ptr<HalRequest>* out); 249 void signalRequestDone(); 250 251 int cropAndScaleLocked( 252 sp<AllocatedFrame>& in, const Size& outSize, 253 YCbCrLayout* out); 254 255 int cropAndScaleThumbLocked( 256 sp<AllocatedFrame>& in, const Size& outSize, 257 YCbCrLayout* out); 258 259 int formatConvertLocked(const YCbCrLayout& in, const YCbCrLayout& out, 260 Size sz, uint32_t format); 261 262 static int encodeJpegYU12(const Size &inSz, 263 const YCbCrLayout& inLayout, int jpegQuality, 264 const void *app1Buffer, size_t app1Size, 265 void *out, size_t maxOutSize, 266 size_t &actualCodeSize); 267 268 int createJpegLocked(HalStreamBuffer &halBuf, const std::shared_ptr<HalRequest>& req); 269 270 const wp<ExternalCameraDeviceSession> mParent; 271 const CroppingType mCroppingType; 272 273 mutable std::mutex mRequestListLock; // Protect acccess to mRequestList, 274 // mProcessingRequest and mProcessingFrameNumer 275 std::condition_variable mRequestCond; // signaled when a new request is submitted 276 std::condition_variable mRequestDoneCond; // signaled when a request is done processing 277 std::list<std::shared_ptr<HalRequest>> mRequestList; 278 bool mProcessingRequest = false; 279 uint32_t mProcessingFrameNumer = 0; 280 281 // V4L2 frameIn 282 // (MJPG decode)-> mYu12Frame 283 // (Scale)-> mScaledYu12Frames 284 // (Format convert) -> output gralloc frames 285 mutable std::mutex mBufferLock; // Protect access to intermediate buffers 286 sp<AllocatedFrame> mYu12Frame; 287 sp<AllocatedFrame> mYu12ThumbFrame; 288 std::unordered_map<Size, sp<AllocatedFrame>, SizeHasher> mIntermediateBuffers; 289 std::unordered_map<Size, sp<AllocatedFrame>, SizeHasher> mScaledYu12Frames; 290 YCbCrLayout mYu12FrameLayout; 291 YCbCrLayout mYu12ThumbFrameLayout; 292 293 std::string mExifMake; 294 std::string mExifModel; 295 }; 296 297 // Protect (most of) HIDL interface methods from synchronized-entering 298 mutable Mutex mInterfaceLock; 299 300 mutable Mutex mLock; // Protect all private members except otherwise noted 301 const sp<ICameraDeviceCallback> mCallback; 302 const ExternalCameraConfig& mCfg; 303 const common::V1_0::helper::CameraMetadata mCameraCharacteristics; 304 const std::vector<SupportedV4L2Format> mSupportedFormats; 305 const CroppingType mCroppingType; 306 const std::string& mCameraId; 307 308 // Not protected by mLock, this is almost a const. 309 // Setup in constructor, reset in close() after OutputThread is joined 310 unique_fd mV4l2Fd; 311 312 // device is closed either 313 // - closed by user 314 // - init failed 315 // - camera disconnected 316 bool mClosed = false; 317 bool mInitFail = false; 318 bool mFirstRequest = false; 319 common::V1_0::helper::CameraMetadata mLatestReqSetting; 320 321 bool mV4l2Streaming = false; 322 SupportedV4L2Format mV4l2StreamingFmt; 323 double mV4l2StreamingFps = 0.0; 324 size_t mV4L2BufferCount = 0; 325 326 static const int kBufferWaitTimeoutSec = 3; // TODO: handle long exposure (or not allowing) 327 std::mutex mV4l2BufferLock; // protect the buffer count and condition below 328 std::condition_variable mV4L2BufferReturned; 329 size_t mNumDequeuedV4l2Buffers = 0; 330 uint32_t mMaxV4L2BufferSize = 0; 331 332 // Not protected by mLock (but might be used when mLock is locked) 333 sp<OutputThread> mOutputThread; 334 335 // Stream ID -> Camera3Stream cache 336 std::unordered_map<int, Stream> mStreamMap; 337 338 std::mutex mInflightFramesLock; // protect mInflightFrames 339 std::unordered_set<uint32_t> mInflightFrames; 340 341 // buffers currently circulating between HAL and camera service 342 // key: bufferId sent via HIDL interface 343 // value: imported buffer_handle_t 344 // Buffer will be imported during processCaptureRequest and will be freed 345 // when the its stream is deleted or camera device session is closed 346 typedef std::unordered_map<uint64_t, buffer_handle_t> CirculatingBuffers; 347 // Stream ID -> circulating buffers map 348 std::map<int, CirculatingBuffers> mCirculatingBuffers; 349 350 std::mutex mAfTriggerLock; // protect mAfTrigger 351 bool mAfTrigger = false; 352 353 static HandleImporter sHandleImporter; 354 355 /* Beginning of members not changed after initialize() */ 356 using RequestMetadataQueue = MessageQueue<uint8_t, kSynchronizedReadWrite>; 357 std::unique_ptr<RequestMetadataQueue> mRequestMetadataQueue; 358 using ResultMetadataQueue = MessageQueue<uint8_t, kSynchronizedReadWrite>; 359 std::shared_ptr<ResultMetadataQueue> mResultMetadataQueue; 360 361 // Protect against invokeProcessCaptureResultCallback() 362 Mutex mProcessCaptureResultLock; 363 364 std::unordered_map<RequestTemplate, CameraMetadata> mDefaultRequests; 365 366 const Size mMaxThumbResolution; 367 const Size mMaxJpegResolution; 368 /* End of members not changed after initialize() */ 369 370 private: 371 372 struct TrampolineSessionInterface_3_4 : public ICameraDeviceSession { 373 TrampolineSessionInterface_3_4(sp<ExternalCameraDeviceSession> parent) : 374 mParent(parent) {} 375 376 virtual Return<void> constructDefaultRequestSettings( 377 RequestTemplate type, 378 V3_3::ICameraDeviceSession::constructDefaultRequestSettings_cb _hidl_cb) override { 379 return mParent->constructDefaultRequestSettings(type, _hidl_cb); 380 } 381 382 virtual Return<void> configureStreams( 383 const V3_2::StreamConfiguration& requestedConfiguration, 384 V3_3::ICameraDeviceSession::configureStreams_cb _hidl_cb) override { 385 return mParent->configureStreams(requestedConfiguration, _hidl_cb); 386 } 387 388 virtual Return<void> processCaptureRequest(const hidl_vec<V3_2::CaptureRequest>& requests, 389 const hidl_vec<V3_2::BufferCache>& cachesToRemove, 390 V3_3::ICameraDeviceSession::processCaptureRequest_cb _hidl_cb) override { 391 return mParent->processCaptureRequest(requests, cachesToRemove, _hidl_cb); 392 } 393 394 virtual Return<void> getCaptureRequestMetadataQueue( 395 V3_3::ICameraDeviceSession::getCaptureRequestMetadataQueue_cb _hidl_cb) override { 396 return mParent->getCaptureRequestMetadataQueue(_hidl_cb); 397 } 398 399 virtual Return<void> getCaptureResultMetadataQueue( 400 V3_3::ICameraDeviceSession::getCaptureResultMetadataQueue_cb _hidl_cb) override { 401 return mParent->getCaptureResultMetadataQueue(_hidl_cb); 402 } 403 404 virtual Return<Status> flush() override { 405 return mParent->flush(); 406 } 407 408 virtual Return<void> close() override { 409 return mParent->close(); 410 } 411 412 virtual Return<void> configureStreams_3_3( 413 const V3_2::StreamConfiguration& requestedConfiguration, 414 configureStreams_3_3_cb _hidl_cb) override { 415 return mParent->configureStreams_3_3(requestedConfiguration, _hidl_cb); 416 } 417 418 virtual Return<void> configureStreams_3_4( 419 const V3_4::StreamConfiguration& requestedConfiguration, 420 configureStreams_3_4_cb _hidl_cb) override { 421 return mParent->configureStreams_3_4(requestedConfiguration, _hidl_cb); 422 } 423 424 virtual Return<void> processCaptureRequest_3_4(const hidl_vec<V3_4::CaptureRequest>& requests, 425 const hidl_vec<V3_2::BufferCache>& cachesToRemove, 426 ICameraDeviceSession::processCaptureRequest_3_4_cb _hidl_cb) override { 427 return mParent->processCaptureRequest_3_4(requests, cachesToRemove, _hidl_cb); 428 } 429 430 private: 431 sp<ExternalCameraDeviceSession> mParent; 432 }; 433 }; 434 435 } // namespace implementation 436 } // namespace V3_4 437 } // namespace device 438 } // namespace camera 439 } // namespace hardware 440 } // namespace android 441 442 #endif // ANDROID_HARDWARE_CAMERA_DEVICE_V3_4_EXTCAMERADEVICE3SESSION_H 443
