1 /* 2 * Copyright (C) 2012 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17 #define LOG_TAG "Camera2-ZslProcessor" 18 #define ATRACE_TAG ATRACE_TAG_CAMERA 19 //#define LOG_NDEBUG 0 20 //#define LOG_NNDEBUG 0 21 22 #ifdef LOG_NNDEBUG 23 #define ALOGVV(...) ALOGV(__VA_ARGS__) 24 #else 25 #define ALOGVV(...) ((void)0) 26 #endif 27 28 #include <inttypes.h> 29 30 #include <utils/Log.h> 31 #include <utils/Trace.h> 32 #include <gui/Surface.h> 33 34 #include "common/CameraDeviceBase.h" 35 #include "api1/Camera2Client.h" 36 #include "api1/client2/CaptureSequencer.h" 37 #include "api1/client2/ZslProcessor.h" 38 39 namespace android { 40 namespace camera2 { 41 42 ZslProcessor::ZslProcessor( 43 sp<Camera2Client> client, 44 wp<CaptureSequencer> sequencer): 45 Thread(false), 46 mState(RUNNING), 47 mClient(client), 48 mDevice(client->getCameraDevice()), 49 mSequencer(sequencer), 50 mId(client->getCameraId()), 51 mDeleted(false), 52 mZslBufferAvailable(false), 53 mZslStreamId(NO_STREAM), 54 mZslReprocessStreamId(NO_STREAM), 55 mFrameListHead(0), 56 mZslQueueHead(0), 57 mZslQueueTail(0) { 58 mZslQueue.insertAt(0, kZslBufferDepth); 59 mFrameList.insertAt(0, kFrameListDepth); 60 sp<CaptureSequencer> captureSequencer = mSequencer.promote(); 61 if (captureSequencer != 0) captureSequencer->setZslProcessor(this); 62 } 63 64 ZslProcessor::~ZslProcessor() { 65 ALOGV("%s: Exit", __FUNCTION__); 66 disconnect(); 67 } 68 69 void ZslProcessor::onFrameAvailable() { 70 Mutex::Autolock l(mInputMutex); 71 if (!mZslBufferAvailable) { 72 mZslBufferAvailable = true; 73 mZslBufferAvailableSignal.signal(); 74 } 75 } 76 77 void ZslProcessor::onResultAvailable(const CaptureResult &result) { 78 ATRACE_CALL(); 79 ALOGV("%s:", __FUNCTION__); 80 Mutex::Autolock l(mInputMutex); 81 camera_metadata_ro_entry_t entry; 82 entry = result.mMetadata.find(ANDROID_SENSOR_TIMESTAMP); 83 nsecs_t timestamp = entry.data.i64[0]; 84 (void)timestamp; 85 ALOGVV("Got preview frame for timestamp %" PRId64, timestamp); 86 87 if (mState != RUNNING) return; 88 89 mFrameList.editItemAt(mFrameListHead) = result.mMetadata; 90 mFrameListHead = (mFrameListHead + 1) % kFrameListDepth; 91 92 findMatchesLocked(); 93 } 94 95 void ZslProcessor::onBufferReleased(buffer_handle_t *handle) { 96 Mutex::Autolock l(mInputMutex); 97 98 // Verify that the buffer is in our queue 99 size_t i = 0; 100 for (; i < mZslQueue.size(); i++) { 101 if (&(mZslQueue[i].buffer.mGraphicBuffer->handle) == handle) break; 102 } 103 if (i == mZslQueue.size()) { 104 ALOGW("%s: Released buffer %p not found in queue", 105 __FUNCTION__, handle); 106 } 107 108 // Erase entire ZSL queue since we've now completed the capture and preview 109 // is stopped. 110 clearZslQueueLocked(); 111 112 mState = RUNNING; 113 } 114 115 status_t ZslProcessor::updateStream(const Parameters ¶ms) { 116 ATRACE_CALL(); 117 ALOGV("%s: Configuring ZSL streams", __FUNCTION__); 118 status_t res; 119 120 Mutex::Autolock l(mInputMutex); 121 122 sp<Camera2Client> client = mClient.promote(); 123 if (client == 0) { 124 ALOGE("%s: Camera %d: Client does not exist", __FUNCTION__, mId); 125 return INVALID_OPERATION; 126 } 127 sp<CameraDeviceBase> device = mDevice.promote(); 128 if (device == 0) { 129 ALOGE("%s: Camera %d: Device does not exist", __FUNCTION__, mId); 130 return INVALID_OPERATION; 131 } 132 133 if (mZslConsumer == 0) { 134 // Create CPU buffer queue endpoint 135 sp<IGraphicBufferProducer> producer; 136 sp<IGraphicBufferConsumer> consumer; 137 BufferQueue::createBufferQueue(&producer, &consumer); 138 mZslConsumer = new BufferItemConsumer(consumer, 139 GRALLOC_USAGE_HW_CAMERA_ZSL, 140 kZslBufferDepth); 141 mZslConsumer->setFrameAvailableListener(this); 142 mZslConsumer->setName(String8("Camera2Client::ZslConsumer")); 143 mZslWindow = new Surface(producer); 144 } 145 146 if (mZslStreamId != NO_STREAM) { 147 // Check if stream parameters have to change 148 uint32_t currentWidth, currentHeight; 149 res = device->getStreamInfo(mZslStreamId, 150 ¤tWidth, ¤tHeight, 0); 151 if (res != OK) { 152 ALOGE("%s: Camera %d: Error querying capture output stream info: " 153 "%s (%d)", __FUNCTION__, 154 mId, strerror(-res), res); 155 return res; 156 } 157 if (currentWidth != (uint32_t)params.fastInfo.arrayWidth || 158 currentHeight != (uint32_t)params.fastInfo.arrayHeight) { 159 res = device->deleteReprocessStream(mZslReprocessStreamId); 160 if (res != OK) { 161 ALOGE("%s: Camera %d: Unable to delete old reprocess stream " 162 "for ZSL: %s (%d)", __FUNCTION__, 163 mId, strerror(-res), res); 164 return res; 165 } 166 ALOGV("%s: Camera %d: Deleting stream %d since the buffer dimensions changed", 167 __FUNCTION__, mId, mZslStreamId); 168 res = device->deleteStream(mZslStreamId); 169 if (res != OK) { 170 ALOGE("%s: Camera %d: Unable to delete old output stream " 171 "for ZSL: %s (%d)", __FUNCTION__, 172 mId, strerror(-res), res); 173 return res; 174 } 175 mZslStreamId = NO_STREAM; 176 } 177 } 178 179 mDeleted = false; 180 181 if (mZslStreamId == NO_STREAM) { 182 // Create stream for HAL production 183 // TODO: Sort out better way to select resolution for ZSL 184 int streamType = params.quirks.useZslFormat ? 185 (int)CAMERA2_HAL_PIXEL_FORMAT_ZSL : 186 (int)HAL_PIXEL_FORMAT_IMPLEMENTATION_DEFINED; 187 res = device->createStream(mZslWindow, 188 params.fastInfo.arrayWidth, params.fastInfo.arrayHeight, 189 streamType, &mZslStreamId); 190 if (res != OK) { 191 ALOGE("%s: Camera %d: Can't create output stream for ZSL: " 192 "%s (%d)", __FUNCTION__, mId, 193 strerror(-res), res); 194 return res; 195 } 196 res = device->createReprocessStreamFromStream(mZslStreamId, 197 &mZslReprocessStreamId); 198 if (res != OK) { 199 ALOGE("%s: Camera %d: Can't create reprocess stream for ZSL: " 200 "%s (%d)", __FUNCTION__, mId, 201 strerror(-res), res); 202 return res; 203 } 204 } 205 client->registerFrameListener(Camera2Client::kPreviewRequestIdStart, 206 Camera2Client::kPreviewRequestIdEnd, 207 this, 208 /*sendPartials*/false); 209 210 return OK; 211 } 212 213 status_t ZslProcessor::deleteStream() { 214 ATRACE_CALL(); 215 Mutex::Autolock l(mInputMutex); 216 // WAR(b/15408128): do not delete stream unless client is being disconnected. 217 mDeleted = true; 218 return OK; 219 } 220 221 status_t ZslProcessor::disconnect() { 222 ATRACE_CALL(); 223 status_t res; 224 225 Mutex::Autolock l(mInputMutex); 226 227 if (mZslStreamId != NO_STREAM) { 228 sp<CameraDeviceBase> device = mDevice.promote(); 229 if (device == 0) { 230 ALOGE("%s: Camera %d: Device does not exist", __FUNCTION__, mId); 231 return INVALID_OPERATION; 232 } 233 234 clearZslQueueLocked(); 235 236 res = device->deleteReprocessStream(mZslReprocessStreamId); 237 if (res != OK) { 238 ALOGE("%s: Camera %d: Cannot delete ZSL reprocessing stream %d: " 239 "%s (%d)", __FUNCTION__, mId, 240 mZslReprocessStreamId, strerror(-res), res); 241 return res; 242 } 243 244 mZslReprocessStreamId = NO_STREAM; 245 res = device->deleteStream(mZslStreamId); 246 if (res != OK) { 247 ALOGE("%s: Camera %d: Cannot delete ZSL output stream %d: " 248 "%s (%d)", __FUNCTION__, mId, 249 mZslStreamId, strerror(-res), res); 250 return res; 251 } 252 253 mZslWindow.clear(); 254 mZslConsumer.clear(); 255 256 mZslStreamId = NO_STREAM; 257 } 258 return OK; 259 } 260 261 int ZslProcessor::getStreamId() const { 262 Mutex::Autolock l(mInputMutex); 263 return mZslStreamId; 264 } 265 266 status_t ZslProcessor::pushToReprocess(int32_t requestId) { 267 ALOGV("%s: Send in reprocess request with id %d", 268 __FUNCTION__, requestId); 269 Mutex::Autolock l(mInputMutex); 270 status_t res; 271 sp<Camera2Client> client = mClient.promote(); 272 273 if (client == 0) { 274 ALOGE("%s: Camera %d: Client does not exist", __FUNCTION__, mId); 275 return INVALID_OPERATION; 276 } 277 278 IF_ALOGV() { 279 dumpZslQueue(-1); 280 } 281 282 if (mZslQueueTail != mZslQueueHead) { 283 CameraMetadata request; 284 size_t index = mZslQueueTail; 285 while (index != mZslQueueHead) { 286 if (!mZslQueue[index].frame.isEmpty()) { 287 request = mZslQueue[index].frame; 288 break; 289 } 290 index = (index + 1) % kZslBufferDepth; 291 } 292 if (index == mZslQueueHead) { 293 ALOGV("%s: ZSL queue has no valid frames to send yet.", 294 __FUNCTION__); 295 return NOT_ENOUGH_DATA; 296 } 297 // Verify that the frame is reasonable for reprocessing 298 299 camera_metadata_entry_t entry; 300 entry = request.find(ANDROID_CONTROL_AE_STATE); 301 if (entry.count == 0) { 302 ALOGE("%s: ZSL queue frame has no AE state field!", 303 __FUNCTION__); 304 return BAD_VALUE; 305 } 306 if (entry.data.u8[0] != ANDROID_CONTROL_AE_STATE_CONVERGED && 307 entry.data.u8[0] != ANDROID_CONTROL_AE_STATE_LOCKED) { 308 ALOGV("%s: ZSL queue frame AE state is %d, need full capture", 309 __FUNCTION__, entry.data.u8[0]); 310 return NOT_ENOUGH_DATA; 311 } 312 313 buffer_handle_t *handle = 314 &(mZslQueue[index].buffer.mGraphicBuffer->handle); 315 316 uint8_t requestType = ANDROID_REQUEST_TYPE_REPROCESS; 317 res = request.update(ANDROID_REQUEST_TYPE, 318 &requestType, 1); 319 int32_t inputStreams[1] = 320 { mZslReprocessStreamId }; 321 if (res == OK) request.update(ANDROID_REQUEST_INPUT_STREAMS, 322 inputStreams, 1); 323 int32_t outputStreams[1] = 324 { client->getCaptureStreamId() }; 325 if (res == OK) request.update(ANDROID_REQUEST_OUTPUT_STREAMS, 326 outputStreams, 1); 327 res = request.update(ANDROID_REQUEST_ID, 328 &requestId, 1); 329 330 if (res != OK ) { 331 ALOGE("%s: Unable to update frame to a reprocess request", __FUNCTION__); 332 return INVALID_OPERATION; 333 } 334 335 res = client->stopStream(); 336 if (res != OK) { 337 ALOGE("%s: Camera %d: Unable to stop preview for ZSL capture: " 338 "%s (%d)", 339 __FUNCTION__, mId, strerror(-res), res); 340 return INVALID_OPERATION; 341 } 342 // TODO: have push-and-clear be atomic 343 res = client->getCameraDevice()->pushReprocessBuffer(mZslReprocessStreamId, 344 handle, this); 345 if (res != OK) { 346 ALOGE("%s: Unable to push buffer for reprocessing: %s (%d)", 347 __FUNCTION__, strerror(-res), res); 348 return res; 349 } 350 351 // Update JPEG settings 352 { 353 SharedParameters::Lock l(client->getParameters()); 354 res = l.mParameters.updateRequestJpeg(&request); 355 if (res != OK) { 356 ALOGE("%s: Camera %d: Unable to update JPEG entries of ZSL " 357 "capture request: %s (%d)", __FUNCTION__, 358 mId, 359 strerror(-res), res); 360 return res; 361 } 362 } 363 364 mLatestCapturedRequest = request; 365 res = client->getCameraDevice()->capture(request); 366 if (res != OK ) { 367 ALOGE("%s: Unable to send ZSL reprocess request to capture: %s (%d)", 368 __FUNCTION__, strerror(-res), res); 369 return res; 370 } 371 372 mState = LOCKED; 373 } else { 374 ALOGV("%s: No ZSL buffers yet", __FUNCTION__); 375 return NOT_ENOUGH_DATA; 376 } 377 return OK; 378 } 379 380 status_t ZslProcessor::clearZslQueue() { 381 Mutex::Autolock l(mInputMutex); 382 // If in middle of capture, can't clear out queue 383 if (mState == LOCKED) return OK; 384 385 return clearZslQueueLocked(); 386 } 387 388 status_t ZslProcessor::clearZslQueueLocked() { 389 for (size_t i = 0; i < mZslQueue.size(); i++) { 390 if (mZslQueue[i].buffer.mTimestamp != 0) { 391 mZslConsumer->releaseBuffer(mZslQueue[i].buffer); 392 } 393 mZslQueue.replaceAt(i); 394 } 395 mZslQueueHead = 0; 396 mZslQueueTail = 0; 397 return OK; 398 } 399 400 void ZslProcessor::dump(int fd, const Vector<String16>& /*args*/) const { 401 Mutex::Autolock l(mInputMutex); 402 if (!mLatestCapturedRequest.isEmpty()) { 403 String8 result(" Latest ZSL capture request:\n"); 404 write(fd, result.string(), result.size()); 405 mLatestCapturedRequest.dump(fd, 2, 6); 406 } else { 407 String8 result(" Latest ZSL capture request: none yet\n"); 408 write(fd, result.string(), result.size()); 409 } 410 dumpZslQueue(fd); 411 } 412 413 bool ZslProcessor::threadLoop() { 414 status_t res; 415 416 { 417 Mutex::Autolock l(mInputMutex); 418 while (!mZslBufferAvailable) { 419 res = mZslBufferAvailableSignal.waitRelative(mInputMutex, 420 kWaitDuration); 421 if (res == TIMED_OUT) return true; 422 } 423 mZslBufferAvailable = false; 424 } 425 426 do { 427 res = processNewZslBuffer(); 428 } while (res == OK); 429 430 return true; 431 } 432 433 status_t ZslProcessor::processNewZslBuffer() { 434 ATRACE_CALL(); 435 status_t res; 436 sp<BufferItemConsumer> zslConsumer; 437 { 438 Mutex::Autolock l(mInputMutex); 439 if (mZslConsumer == 0) return OK; 440 zslConsumer = mZslConsumer; 441 } 442 ALOGVV("Trying to get next buffer"); 443 BufferItemConsumer::BufferItem item; 444 res = zslConsumer->acquireBuffer(&item, 0); 445 if (res != OK) { 446 if (res != BufferItemConsumer::NO_BUFFER_AVAILABLE) { 447 ALOGE("%s: Camera %d: Error receiving ZSL image buffer: " 448 "%s (%d)", __FUNCTION__, 449 mId, strerror(-res), res); 450 } else { 451 ALOGVV(" No buffer"); 452 } 453 return res; 454 } 455 456 Mutex::Autolock l(mInputMutex); 457 458 if (mState == LOCKED) { 459 ALOGVV("In capture, discarding new ZSL buffers"); 460 zslConsumer->releaseBuffer(item); 461 return OK; 462 } 463 464 ALOGVV("Got ZSL buffer: head: %d, tail: %d", mZslQueueHead, mZslQueueTail); 465 466 if ( (mZslQueueHead + 1) % kZslBufferDepth == mZslQueueTail) { 467 ALOGVV("Releasing oldest buffer"); 468 zslConsumer->releaseBuffer(mZslQueue[mZslQueueTail].buffer); 469 mZslQueue.replaceAt(mZslQueueTail); 470 mZslQueueTail = (mZslQueueTail + 1) % kZslBufferDepth; 471 } 472 473 ZslPair &queueHead = mZslQueue.editItemAt(mZslQueueHead); 474 475 queueHead.buffer = item; 476 queueHead.frame.release(); 477 478 mZslQueueHead = (mZslQueueHead + 1) % kZslBufferDepth; 479 480 ALOGVV(" Acquired buffer, timestamp %" PRId64, queueHead.buffer.mTimestamp); 481 482 findMatchesLocked(); 483 484 return OK; 485 } 486 487 void ZslProcessor::findMatchesLocked() { 488 ALOGVV("Scanning"); 489 for (size_t i = 0; i < mZslQueue.size(); i++) { 490 ZslPair &queueEntry = mZslQueue.editItemAt(i); 491 nsecs_t bufferTimestamp = queueEntry.buffer.mTimestamp; 492 IF_ALOGV() { 493 camera_metadata_entry_t entry; 494 nsecs_t frameTimestamp = 0; 495 if (!queueEntry.frame.isEmpty()) { 496 entry = queueEntry.frame.find(ANDROID_SENSOR_TIMESTAMP); 497 frameTimestamp = entry.data.i64[0]; 498 } 499 ALOGVV(" %d: b: %" PRId64 "\tf: %" PRId64, i, 500 bufferTimestamp, frameTimestamp ); 501 } 502 if (queueEntry.frame.isEmpty() && bufferTimestamp != 0) { 503 // Have buffer, no matching frame. Look for one 504 for (size_t j = 0; j < mFrameList.size(); j++) { 505 bool match = false; 506 CameraMetadata &frame = mFrameList.editItemAt(j); 507 if (!frame.isEmpty()) { 508 camera_metadata_entry_t entry; 509 entry = frame.find(ANDROID_SENSOR_TIMESTAMP); 510 if (entry.count == 0) { 511 ALOGE("%s: Can't find timestamp in frame!", 512 __FUNCTION__); 513 continue; 514 } 515 nsecs_t frameTimestamp = entry.data.i64[0]; 516 if (bufferTimestamp == frameTimestamp) { 517 ALOGVV("%s: Found match %" PRId64, __FUNCTION__, 518 frameTimestamp); 519 match = true; 520 } else { 521 int64_t delta = abs(bufferTimestamp - frameTimestamp); 522 if ( delta < 1000000) { 523 ALOGVV("%s: Found close match %" PRId64 " (delta %" PRId64 ")", 524 __FUNCTION__, bufferTimestamp, delta); 525 match = true; 526 } 527 } 528 } 529 if (match) { 530 queueEntry.frame.acquire(frame); 531 break; 532 } 533 } 534 } 535 } 536 } 537 538 void ZslProcessor::dumpZslQueue(int fd) const { 539 String8 header("ZSL queue contents:"); 540 String8 indent(" "); 541 ALOGV("%s", header.string()); 542 if (fd != -1) { 543 header = indent + header + "\n"; 544 write(fd, header.string(), header.size()); 545 } 546 for (size_t i = 0; i < mZslQueue.size(); i++) { 547 const ZslPair &queueEntry = mZslQueue[i]; 548 nsecs_t bufferTimestamp = queueEntry.buffer.mTimestamp; 549 camera_metadata_ro_entry_t entry; 550 nsecs_t frameTimestamp = 0; 551 int frameAeState = -1; 552 if (!queueEntry.frame.isEmpty()) { 553 entry = queueEntry.frame.find(ANDROID_SENSOR_TIMESTAMP); 554 if (entry.count > 0) frameTimestamp = entry.data.i64[0]; 555 entry = queueEntry.frame.find(ANDROID_CONTROL_AE_STATE); 556 if (entry.count > 0) frameAeState = entry.data.u8[0]; 557 } 558 String8 result = 559 String8::format(" %zu: b: %" PRId64 "\tf: %" PRId64 ", AE state: %d", i, 560 bufferTimestamp, frameTimestamp, frameAeState); 561 ALOGV("%s", result.string()); 562 if (fd != -1) { 563 result = indent + result + "\n"; 564 write(fd, result.string(), result.size()); 565 } 566 567 } 568 } 569 570 }; // namespace camera2 571 }; // namespace android 572