1 /* 2 * Copyright (C) 2012-2018 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17 #define LOG_TAG "Camera2-CallbackProcessor" 18 #define ATRACE_TAG ATRACE_TAG_CAMERA 19 //#define LOG_NDEBUG 0 20 21 #include <utils/Log.h> 22 #include <utils/Trace.h> 23 #include <gui/Surface.h> 24 25 #include "common/CameraDeviceBase.h" 26 #include "api1/Camera2Client.h" 27 #include "api1/client2/CallbackProcessor.h" 28 29 #define ALIGN(x, mask) ( ((x) + (mask) - 1) & ~((mask) - 1) ) 30 31 namespace android { 32 namespace camera2 { 33 34 CallbackProcessor::CallbackProcessor(sp<Camera2Client> client): 35 Thread(false), 36 mClient(client), 37 mDevice(client->getCameraDevice()), 38 mId(client->getCameraId()), 39 mCallbackAvailable(false), 40 mCallbackToApp(false), 41 mCallbackStreamId(NO_STREAM) { 42 } 43 44 CallbackProcessor::~CallbackProcessor() { 45 ALOGV("%s: Exit", __FUNCTION__); 46 deleteStream(); 47 } 48 49 void CallbackProcessor::onFrameAvailable(const BufferItem& /*item*/) { 50 Mutex::Autolock l(mInputMutex); 51 if (!mCallbackAvailable) { 52 mCallbackAvailable = true; 53 mCallbackAvailableSignal.signal(); 54 } 55 } 56 57 status_t CallbackProcessor::setCallbackWindow( 58 const sp<Surface>& callbackWindow) { 59 ATRACE_CALL(); 60 status_t res; 61 62 Mutex::Autolock l(mInputMutex); 63 64 sp<Camera2Client> client = mClient.promote(); 65 if (client == 0) return OK; 66 sp<CameraDeviceBase> device = client->getCameraDevice(); 67 68 // If the window is changing, clear out stream if it already exists 69 if (mCallbackWindow != callbackWindow && mCallbackStreamId != NO_STREAM) { 70 res = device->deleteStream(mCallbackStreamId); 71 if (res != OK) { 72 ALOGE("%s: Camera %d: Unable to delete old stream " 73 "for callbacks: %s (%d)", __FUNCTION__, 74 client->getCameraId(), strerror(-res), res); 75 return res; 76 } 77 mCallbackStreamId = NO_STREAM; 78 mCallbackConsumer.clear(); 79 } 80 mCallbackWindow = callbackWindow; 81 mCallbackToApp = (mCallbackWindow != NULL); 82 83 return OK; 84 } 85 86 status_t CallbackProcessor::updateStream(const Parameters ¶ms) { 87 ATRACE_CALL(); 88 status_t res; 89 90 Mutex::Autolock l(mInputMutex); 91 92 sp<CameraDeviceBase> device = mDevice.promote(); 93 if (device == 0) { 94 ALOGE("%s: Camera %d: Device does not exist", __FUNCTION__, mId); 95 return INVALID_OPERATION; 96 } 97 98 // If possible, use the flexible YUV format 99 int32_t callbackFormat = params.previewFormat; 100 if (mCallbackToApp) { 101 // TODO: etalvala: This should use the flexible YUV format as well, but 102 // need to reconcile HAL2/HAL3 requirements. 103 callbackFormat = HAL_PIXEL_FORMAT_YV12; 104 } else if(params.fastInfo.useFlexibleYuv && 105 (params.previewFormat == HAL_PIXEL_FORMAT_YCrCb_420_SP || 106 params.previewFormat == HAL_PIXEL_FORMAT_YV12) ) { 107 callbackFormat = HAL_PIXEL_FORMAT_YCbCr_420_888; 108 } 109 110 if (!mCallbackToApp && mCallbackConsumer == 0) { 111 // Create CPU buffer queue endpoint, since app hasn't given us one 112 // Make it async to avoid disconnect deadlocks 113 sp<IGraphicBufferProducer> producer; 114 sp<IGraphicBufferConsumer> consumer; 115 BufferQueue::createBufferQueue(&producer, &consumer); 116 mCallbackConsumer = new CpuConsumer(consumer, kCallbackHeapCount); 117 mCallbackConsumer->setFrameAvailableListener(this); 118 mCallbackConsumer->setName(String8("Camera2-CallbackConsumer")); 119 mCallbackWindow = new Surface(producer); 120 } 121 122 if (mCallbackStreamId != NO_STREAM) { 123 // Check if stream parameters have to change 124 CameraDeviceBase::StreamInfo streamInfo; 125 res = device->getStreamInfo(mCallbackStreamId, &streamInfo); 126 if (res != OK) { 127 ALOGE("%s: Camera %d: Error querying callback output stream info: " 128 "%s (%d)", __FUNCTION__, mId, 129 strerror(-res), res); 130 return res; 131 } 132 if (streamInfo.width != (uint32_t)params.previewWidth || 133 streamInfo.height != (uint32_t)params.previewHeight || 134 !streamInfo.matchFormat((uint32_t)callbackFormat)) { 135 // Since size should only change while preview is not running, 136 // assuming that all existing use of old callback stream is 137 // completed. 138 ALOGV("%s: Camera %d: Deleting stream %d since the buffer " 139 "parameters changed", __FUNCTION__, mId, mCallbackStreamId); 140 res = device->deleteStream(mCallbackStreamId); 141 if (res != OK) { 142 ALOGE("%s: Camera %d: Unable to delete old output stream " 143 "for callbacks: %s (%d)", __FUNCTION__, 144 mId, strerror(-res), res); 145 return res; 146 } 147 mCallbackStreamId = NO_STREAM; 148 } 149 } 150 151 if (mCallbackStreamId == NO_STREAM) { 152 ALOGV("Creating callback stream: %d x %d, format 0x%x, API format 0x%x", 153 params.previewWidth, params.previewHeight, 154 callbackFormat, params.previewFormat); 155 res = device->createStream(mCallbackWindow, 156 params.previewWidth, params.previewHeight, callbackFormat, 157 HAL_DATASPACE_V0_JFIF, CAMERA3_STREAM_ROTATION_0, &mCallbackStreamId, 158 String8()); 159 if (res != OK) { 160 ALOGE("%s: Camera %d: Can't create output stream for callbacks: " 161 "%s (%d)", __FUNCTION__, mId, 162 strerror(-res), res); 163 return res; 164 } 165 } 166 167 return OK; 168 } 169 170 status_t CallbackProcessor::deleteStream() { 171 ATRACE_CALL(); 172 sp<CameraDeviceBase> device; 173 status_t res; 174 { 175 Mutex::Autolock l(mInputMutex); 176 177 if (mCallbackStreamId == NO_STREAM) { 178 return OK; 179 } 180 device = mDevice.promote(); 181 if (device == 0) { 182 ALOGE("%s: Camera %d: Device does not exist", __FUNCTION__, mId); 183 return INVALID_OPERATION; 184 } 185 } 186 res = device->waitUntilDrained(); 187 if (res != OK) { 188 ALOGE("%s: Error waiting for HAL to drain: %s (%d)", 189 __FUNCTION__, strerror(-res), res); 190 return res; 191 } 192 193 res = device->deleteStream(mCallbackStreamId); 194 if (res != OK) { 195 ALOGE("%s: Unable to delete callback stream: %s (%d)", 196 __FUNCTION__, strerror(-res), res); 197 return res; 198 } 199 200 { 201 Mutex::Autolock l(mInputMutex); 202 203 mCallbackHeap.clear(); 204 mCallbackWindow.clear(); 205 mCallbackConsumer.clear(); 206 207 mCallbackStreamId = NO_STREAM; 208 } 209 return OK; 210 } 211 212 int CallbackProcessor::getStreamId() const { 213 Mutex::Autolock l(mInputMutex); 214 return mCallbackStreamId; 215 } 216 217 void CallbackProcessor::dump(int /*fd*/, const Vector<String16>& /*args*/) const { 218 } 219 220 bool CallbackProcessor::threadLoop() { 221 status_t res; 222 223 { 224 Mutex::Autolock l(mInputMutex); 225 while (!mCallbackAvailable) { 226 res = mCallbackAvailableSignal.waitRelative(mInputMutex, 227 kWaitDuration); 228 if (res == TIMED_OUT) return true; 229 } 230 mCallbackAvailable = false; 231 } 232 233 do { 234 sp<Camera2Client> client = mClient.promote(); 235 if (client == 0) { 236 res = discardNewCallback(); 237 } else { 238 res = processNewCallback(client); 239 } 240 } while (res == OK); 241 242 return true; 243 } 244 245 status_t CallbackProcessor::discardNewCallback() { 246 ATRACE_CALL(); 247 status_t res; 248 CpuConsumer::LockedBuffer imgBuffer; 249 res = mCallbackConsumer->lockNextBuffer(&imgBuffer); 250 if (res != OK) { 251 if (res != BAD_VALUE) { 252 ALOGE("%s: Camera %d: Error receiving next callback buffer: " 253 "%s (%d)", __FUNCTION__, mId, strerror(-res), res); 254 } 255 return res; 256 } 257 mCallbackConsumer->unlockBuffer(imgBuffer); 258 return OK; 259 } 260 261 status_t CallbackProcessor::processNewCallback(sp<Camera2Client> &client) { 262 ATRACE_CALL(); 263 status_t res; 264 265 sp<Camera2Heap> callbackHeap; 266 bool useFlexibleYuv = false; 267 int32_t previewFormat = 0; 268 size_t heapIdx; 269 270 { 271 /* acquire SharedParameters before mMutex so we don't dead lock 272 with Camera2Client code calling into StreamingProcessor */ 273 SharedParameters::Lock l(client->getParameters()); 274 Mutex::Autolock m(mInputMutex); 275 CpuConsumer::LockedBuffer imgBuffer; 276 if (mCallbackStreamId == NO_STREAM) { 277 ALOGV("%s: Camera %d:No stream is available" 278 , __FUNCTION__, mId); 279 return INVALID_OPERATION; 280 } 281 282 ALOGV("%s: Getting buffer", __FUNCTION__); 283 res = mCallbackConsumer->lockNextBuffer(&imgBuffer); 284 if (res != OK) { 285 if (res != BAD_VALUE) { 286 ALOGE("%s: Camera %d: Error receiving next callback buffer: " 287 "%s (%d)", __FUNCTION__, mId, strerror(-res), res); 288 } 289 return res; 290 } 291 ALOGV("%s: Camera %d: Preview callback available", __FUNCTION__, 292 mId); 293 294 if ( l.mParameters.state != Parameters::PREVIEW 295 && l.mParameters.state != Parameters::RECORD 296 && l.mParameters.state != Parameters::VIDEO_SNAPSHOT) { 297 ALOGV("%s: Camera %d: No longer streaming", 298 __FUNCTION__, mId); 299 mCallbackConsumer->unlockBuffer(imgBuffer); 300 return OK; 301 } 302 303 if (! (l.mParameters.previewCallbackFlags & 304 CAMERA_FRAME_CALLBACK_FLAG_ENABLE_MASK) ) { 305 ALOGV("%s: No longer enabled, dropping", __FUNCTION__); 306 mCallbackConsumer->unlockBuffer(imgBuffer); 307 return OK; 308 } 309 if ((l.mParameters.previewCallbackFlags & 310 CAMERA_FRAME_CALLBACK_FLAG_ONE_SHOT_MASK) && 311 !l.mParameters.previewCallbackOneShot) { 312 ALOGV("%s: One shot mode, already sent, dropping", __FUNCTION__); 313 mCallbackConsumer->unlockBuffer(imgBuffer); 314 return OK; 315 } 316 317 if (imgBuffer.width != static_cast<uint32_t>(l.mParameters.previewWidth) || 318 imgBuffer.height != static_cast<uint32_t>(l.mParameters.previewHeight)) { 319 ALOGW("%s: The preview size has changed to %d x %d from %d x %d, this buffer is" 320 " no longer valid, dropping",__FUNCTION__, 321 l.mParameters.previewWidth, l.mParameters.previewHeight, 322 imgBuffer.width, imgBuffer.height); 323 mCallbackConsumer->unlockBuffer(imgBuffer); 324 return OK; 325 } 326 327 previewFormat = l.mParameters.previewFormat; 328 useFlexibleYuv = l.mParameters.fastInfo.useFlexibleYuv && 329 (previewFormat == HAL_PIXEL_FORMAT_YCrCb_420_SP || 330 previewFormat == HAL_PIXEL_FORMAT_YV12); 331 332 int32_t expectedFormat = useFlexibleYuv ? 333 HAL_PIXEL_FORMAT_YCbCr_420_888 : previewFormat; 334 335 if (imgBuffer.format != expectedFormat) { 336 ALOGE("%s: Camera %d: Unexpected format for callback: " 337 "0x%x, expected 0x%x", __FUNCTION__, mId, 338 imgBuffer.format, expectedFormat); 339 mCallbackConsumer->unlockBuffer(imgBuffer); 340 return INVALID_OPERATION; 341 } 342 343 // In one-shot mode, stop sending callbacks after the first one 344 if (l.mParameters.previewCallbackFlags & 345 CAMERA_FRAME_CALLBACK_FLAG_ONE_SHOT_MASK) { 346 ALOGV("%s: clearing oneshot", __FUNCTION__); 347 l.mParameters.previewCallbackOneShot = false; 348 } 349 350 uint32_t destYStride = 0; 351 uint32_t destCStride = 0; 352 if (useFlexibleYuv) { 353 if (previewFormat == HAL_PIXEL_FORMAT_YV12) { 354 // Strides must align to 16 for YV12 355 destYStride = ALIGN(imgBuffer.width, 16); 356 destCStride = ALIGN(destYStride / 2, 16); 357 } else { 358 // No padding for NV21 359 ALOG_ASSERT(previewFormat == HAL_PIXEL_FORMAT_YCrCb_420_SP, 360 "Unexpected preview format 0x%x", previewFormat); 361 destYStride = imgBuffer.width; 362 destCStride = destYStride / 2; 363 } 364 } else { 365 destYStride = imgBuffer.stride; 366 // don't care about cStride 367 } 368 369 size_t bufferSize = Camera2Client::calculateBufferSize( 370 imgBuffer.width, imgBuffer.height, 371 previewFormat, destYStride); 372 size_t currentBufferSize = (mCallbackHeap == 0) ? 373 0 : (mCallbackHeap->mHeap->getSize() / kCallbackHeapCount); 374 if (bufferSize != currentBufferSize) { 375 mCallbackHeap.clear(); 376 mCallbackHeap = new Camera2Heap(bufferSize, kCallbackHeapCount, 377 "Camera2Client::CallbackHeap"); 378 if (mCallbackHeap->mHeap->getSize() == 0) { 379 ALOGE("%s: Camera %d: Unable to allocate memory for callbacks", 380 __FUNCTION__, mId); 381 mCallbackConsumer->unlockBuffer(imgBuffer); 382 return INVALID_OPERATION; 383 } 384 385 mCallbackHeapHead = 0; 386 mCallbackHeapFree = kCallbackHeapCount; 387 } 388 389 if (mCallbackHeapFree == 0) { 390 ALOGE("%s: Camera %d: No free callback buffers, dropping frame", 391 __FUNCTION__, mId); 392 mCallbackConsumer->unlockBuffer(imgBuffer); 393 return OK; 394 } 395 396 heapIdx = mCallbackHeapHead; 397 398 mCallbackHeapHead = (mCallbackHeapHead + 1) % kCallbackHeapCount; 399 mCallbackHeapFree--; 400 401 // TODO: Get rid of this copy by passing the gralloc queue all the way 402 // to app 403 404 ssize_t offset; 405 size_t size; 406 sp<IMemoryHeap> heap = 407 mCallbackHeap->mBuffers[heapIdx]->getMemory(&offset, 408 &size); 409 uint8_t *data = (uint8_t*)heap->getBase() + offset; 410 411 if (!useFlexibleYuv) { 412 // Can just memcpy when HAL format matches API format 413 memcpy(data, imgBuffer.data, bufferSize); 414 } else { 415 res = convertFromFlexibleYuv(previewFormat, data, imgBuffer, 416 destYStride, destCStride); 417 if (res != OK) { 418 ALOGE("%s: Camera %d: Can't convert between 0x%x and 0x%x formats!", 419 __FUNCTION__, mId, imgBuffer.format, previewFormat); 420 mCallbackConsumer->unlockBuffer(imgBuffer); 421 return BAD_VALUE; 422 } 423 } 424 425 ALOGV("%s: Freeing buffer", __FUNCTION__); 426 mCallbackConsumer->unlockBuffer(imgBuffer); 427 428 // mCallbackHeap may get freed up once input mutex is released 429 callbackHeap = mCallbackHeap; 430 } 431 432 // Call outside parameter lock to allow re-entrancy from notification 433 { 434 Camera2Client::SharedCameraCallbacks::Lock 435 l(client->mSharedCameraCallbacks); 436 if (l.mRemoteCallback != 0) { 437 ALOGV("%s: Camera %d: Invoking client data callback", 438 __FUNCTION__, mId); 439 l.mRemoteCallback->dataCallback(CAMERA_MSG_PREVIEW_FRAME, 440 callbackHeap->mBuffers[heapIdx], NULL); 441 } 442 } 443 444 // Only increment free if we're still using the same heap 445 mCallbackHeapFree++; 446 447 ALOGV("%s: exit", __FUNCTION__); 448 449 return OK; 450 } 451 452 status_t CallbackProcessor::convertFromFlexibleYuv(int32_t previewFormat, 453 uint8_t *dst, 454 const CpuConsumer::LockedBuffer &src, 455 uint32_t dstYStride, 456 uint32_t dstCStride) const { 457 458 if (previewFormat != HAL_PIXEL_FORMAT_YCrCb_420_SP && 459 previewFormat != HAL_PIXEL_FORMAT_YV12) { 460 ALOGE("%s: Camera %d: Unexpected preview format when using " 461 "flexible YUV: 0x%x", __FUNCTION__, mId, previewFormat); 462 return INVALID_OPERATION; 463 } 464 465 // Copy Y plane, adjusting for stride 466 const uint8_t *ySrc = src.data; 467 uint8_t *yDst = dst; 468 for (size_t row = 0; row < src.height; row++) { 469 memcpy(yDst, ySrc, src.width); 470 ySrc += src.stride; 471 yDst += dstYStride; 472 } 473 474 // Copy/swizzle chroma planes, 4:2:0 subsampling 475 const uint8_t *cbSrc = src.dataCb; 476 const uint8_t *crSrc = src.dataCr; 477 size_t chromaHeight = src.height / 2; 478 size_t chromaWidth = src.width / 2; 479 ssize_t chromaGap = src.chromaStride - 480 (chromaWidth * src.chromaStep); 481 size_t dstChromaGap = dstCStride - chromaWidth; 482 483 if (previewFormat == HAL_PIXEL_FORMAT_YCrCb_420_SP) { 484 // Flexible YUV chroma to NV21 chroma 485 uint8_t *crcbDst = yDst; 486 // Check for shortcuts 487 if (cbSrc == crSrc + 1 && src.chromaStep == 2) { 488 ALOGV("%s: Fast NV21->NV21", __FUNCTION__); 489 // Source has semiplanar CrCb chroma layout, can copy by rows 490 for (size_t row = 0; row < chromaHeight; row++) { 491 memcpy(crcbDst, crSrc, src.width); 492 crcbDst += src.width; 493 crSrc += src.chromaStride; 494 } 495 } else { 496 ALOGV("%s: Generic->NV21", __FUNCTION__); 497 // Generic copy, always works but not very efficient 498 for (size_t row = 0; row < chromaHeight; row++) { 499 for (size_t col = 0; col < chromaWidth; col++) { 500 *(crcbDst++) = *crSrc; 501 *(crcbDst++) = *cbSrc; 502 crSrc += src.chromaStep; 503 cbSrc += src.chromaStep; 504 } 505 crSrc += chromaGap; 506 cbSrc += chromaGap; 507 } 508 } 509 } else { 510 // flexible YUV chroma to YV12 chroma 511 ALOG_ASSERT(previewFormat == HAL_PIXEL_FORMAT_YV12, 512 "Unexpected preview format 0x%x", previewFormat); 513 uint8_t *crDst = yDst; 514 uint8_t *cbDst = yDst + chromaHeight * dstCStride; 515 if (src.chromaStep == 1) { 516 ALOGV("%s: Fast YV12->YV12", __FUNCTION__); 517 // Source has planar chroma layout, can copy by row 518 for (size_t row = 0; row < chromaHeight; row++) { 519 memcpy(crDst, crSrc, chromaWidth); 520 crDst += dstCStride; 521 crSrc += src.chromaStride; 522 } 523 for (size_t row = 0; row < chromaHeight; row++) { 524 memcpy(cbDst, cbSrc, chromaWidth); 525 cbDst += dstCStride; 526 cbSrc += src.chromaStride; 527 } 528 } else { 529 ALOGV("%s: Generic->YV12", __FUNCTION__); 530 // Generic copy, always works but not very efficient 531 for (size_t row = 0; row < chromaHeight; row++) { 532 for (size_t col = 0; col < chromaWidth; col++) { 533 *(crDst++) = *crSrc; 534 *(cbDst++) = *cbSrc; 535 crSrc += src.chromaStep; 536 cbSrc += src.chromaStep; 537 } 538 crSrc += chromaGap; 539 cbSrc += chromaGap; 540 crDst += dstChromaGap; 541 cbDst += dstChromaGap; 542 } 543 } 544 } 545 546 return OK; 547 } 548 549 }; // namespace camera2 550 }; // namespace android 551