1 /* 2 * Copyright (C) 2012 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17 #define LOG_TAG "Camera2-ZslProcessor" 18 #define ATRACE_TAG ATRACE_TAG_CAMERA 19 //#define LOG_NDEBUG 0 20 //#define LOG_NNDEBUG 0 21 22 #ifdef LOG_NNDEBUG 23 #define ALOGVV(...) ALOGV(__VA_ARGS__) 24 #else 25 #define ALOGVV(...) ((void)0) 26 #endif 27 28 #include <utils/Log.h> 29 #include <utils/Trace.h> 30 31 #include "ZslProcessor.h" 32 #include <gui/Surface.h> 33 #include "../CameraDeviceBase.h" 34 #include "../Camera2Client.h" 35 36 37 namespace android { 38 namespace camera2 { 39 40 ZslProcessor::ZslProcessor( 41 sp<Camera2Client> client, 42 wp<CaptureSequencer> sequencer): 43 Thread(false), 44 mState(RUNNING), 45 mClient(client), 46 mDevice(client->getCameraDevice()), 47 mSequencer(sequencer), 48 mId(client->getCameraId()), 49 mZslBufferAvailable(false), 50 mZslStreamId(NO_STREAM), 51 mZslReprocessStreamId(NO_STREAM), 52 mFrameListHead(0), 53 mZslQueueHead(0), 54 mZslQueueTail(0) { 55 mZslQueue.insertAt(0, kZslBufferDepth); 56 mFrameList.insertAt(0, kFrameListDepth); 57 sp<CaptureSequencer> captureSequencer = mSequencer.promote(); 58 if (captureSequencer != 0) captureSequencer->setZslProcessor(this); 59 } 60 61 ZslProcessor::~ZslProcessor() { 62 ALOGV("%s: Exit", __FUNCTION__); 63 deleteStream(); 64 } 65 66 void ZslProcessor::onFrameAvailable() { 67 Mutex::Autolock l(mInputMutex); 68 if (!mZslBufferAvailable) { 69 mZslBufferAvailable = true; 70 mZslBufferAvailableSignal.signal(); 71 } 72 } 73 74 void ZslProcessor::onFrameAvailable(int32_t /*frameId*/, 75 const CameraMetadata &frame) { 76 Mutex::Autolock l(mInputMutex); 77 camera_metadata_ro_entry_t entry; 78 entry = frame.find(ANDROID_SENSOR_TIMESTAMP); 79 nsecs_t timestamp = entry.data.i64[0]; 80 (void)timestamp; 81 ALOGVV("Got preview frame for timestamp %lld", timestamp); 82 83 if (mState != RUNNING) return; 84 85 mFrameList.editItemAt(mFrameListHead) = frame; 86 mFrameListHead = (mFrameListHead + 1) % kFrameListDepth; 87 88 findMatchesLocked(); 89 } 90 91 void ZslProcessor::onBufferReleased(buffer_handle_t *handle) { 92 Mutex::Autolock l(mInputMutex); 93 94 // Verify that the buffer is in our queue 95 size_t i = 0; 96 for (; i < mZslQueue.size(); i++) { 97 if (&(mZslQueue[i].buffer.mGraphicBuffer->handle) == handle) break; 98 } 99 if (i == mZslQueue.size()) { 100 ALOGW("%s: Released buffer %p not found in queue", 101 __FUNCTION__, handle); 102 } 103 104 // Erase entire ZSL queue since we've now completed the capture and preview 105 // is stopped. 106 clearZslQueueLocked(); 107 108 mState = RUNNING; 109 } 110 111 status_t ZslProcessor::updateStream(const Parameters ¶ms) { 112 ATRACE_CALL(); 113 ALOGV("%s: Configuring ZSL streams", __FUNCTION__); 114 status_t res; 115 116 Mutex::Autolock l(mInputMutex); 117 118 sp<Camera2Client> client = mClient.promote(); 119 if (client == 0) { 120 ALOGE("%s: Camera %d: Client does not exist", __FUNCTION__, mId); 121 return INVALID_OPERATION; 122 } 123 sp<CameraDeviceBase> device = mDevice.promote(); 124 if (device == 0) { 125 ALOGE("%s: Camera %d: Device does not exist", __FUNCTION__, mId); 126 return INVALID_OPERATION; 127 } 128 129 if (mZslConsumer == 0) { 130 // Create CPU buffer queue endpoint 131 mZslConsumer = new BufferItemConsumer( 132 GRALLOC_USAGE_HW_CAMERA_ZSL, 133 kZslBufferDepth, 134 true); 135 mZslConsumer->setFrameAvailableListener(this); 136 mZslConsumer->setName(String8("Camera2Client::ZslConsumer")); 137 mZslWindow = new Surface( 138 mZslConsumer->getProducerInterface()); 139 } 140 141 if (mZslStreamId != NO_STREAM) { 142 // Check if stream parameters have to change 143 uint32_t currentWidth, currentHeight; 144 res = device->getStreamInfo(mZslStreamId, 145 ¤tWidth, ¤tHeight, 0); 146 if (res != OK) { 147 ALOGE("%s: Camera %d: Error querying capture output stream info: " 148 "%s (%d)", __FUNCTION__, 149 mId, strerror(-res), res); 150 return res; 151 } 152 if (currentWidth != (uint32_t)params.fastInfo.arrayWidth || 153 currentHeight != (uint32_t)params.fastInfo.arrayHeight) { 154 res = device->deleteReprocessStream(mZslReprocessStreamId); 155 if (res != OK) { 156 ALOGE("%s: Camera %d: Unable to delete old reprocess stream " 157 "for ZSL: %s (%d)", __FUNCTION__, 158 mId, strerror(-res), res); 159 return res; 160 } 161 ALOGV("%s: Camera %d: Deleting stream %d since the buffer dimensions changed", 162 __FUNCTION__, mId, mZslStreamId); 163 res = device->deleteStream(mZslStreamId); 164 if (res != OK) { 165 ALOGE("%s: Camera %d: Unable to delete old output stream " 166 "for ZSL: %s (%d)", __FUNCTION__, 167 mId, strerror(-res), res); 168 return res; 169 } 170 mZslStreamId = NO_STREAM; 171 } 172 } 173 174 if (mZslStreamId == NO_STREAM) { 175 // Create stream for HAL production 176 // TODO: Sort out better way to select resolution for ZSL 177 int streamType = params.quirks.useZslFormat ? 178 (int)CAMERA2_HAL_PIXEL_FORMAT_ZSL : 179 (int)HAL_PIXEL_FORMAT_IMPLEMENTATION_DEFINED; 180 res = device->createStream(mZslWindow, 181 params.fastInfo.arrayWidth, params.fastInfo.arrayHeight, 182 streamType, 0, 183 &mZslStreamId); 184 if (res != OK) { 185 ALOGE("%s: Camera %d: Can't create output stream for ZSL: " 186 "%s (%d)", __FUNCTION__, mId, 187 strerror(-res), res); 188 return res; 189 } 190 res = device->createReprocessStreamFromStream(mZslStreamId, 191 &mZslReprocessStreamId); 192 if (res != OK) { 193 ALOGE("%s: Camera %d: Can't create reprocess stream for ZSL: " 194 "%s (%d)", __FUNCTION__, mId, 195 strerror(-res), res); 196 return res; 197 } 198 } 199 client->registerFrameListener(Camera2Client::kPreviewRequestIdStart, 200 Camera2Client::kPreviewRequestIdEnd, 201 this); 202 203 return OK; 204 } 205 206 status_t ZslProcessor::deleteStream() { 207 ATRACE_CALL(); 208 status_t res; 209 210 Mutex::Autolock l(mInputMutex); 211 212 if (mZslStreamId != NO_STREAM) { 213 sp<CameraDeviceBase> device = mDevice.promote(); 214 if (device == 0) { 215 ALOGE("%s: Camera %d: Device does not exist", __FUNCTION__, mId); 216 return INVALID_OPERATION; 217 } 218 219 clearZslQueueLocked(); 220 221 res = device->deleteReprocessStream(mZslReprocessStreamId); 222 if (res != OK) { 223 ALOGE("%s: Camera %d: Cannot delete ZSL reprocessing stream %d: " 224 "%s (%d)", __FUNCTION__, mId, 225 mZslReprocessStreamId, strerror(-res), res); 226 return res; 227 } 228 229 mZslReprocessStreamId = NO_STREAM; 230 res = device->deleteStream(mZslStreamId); 231 if (res != OK) { 232 ALOGE("%s: Camera %d: Cannot delete ZSL output stream %d: " 233 "%s (%d)", __FUNCTION__, mId, 234 mZslStreamId, strerror(-res), res); 235 return res; 236 } 237 238 mZslWindow.clear(); 239 mZslConsumer.clear(); 240 241 mZslStreamId = NO_STREAM; 242 } 243 return OK; 244 } 245 246 int ZslProcessor::getStreamId() const { 247 Mutex::Autolock l(mInputMutex); 248 return mZslStreamId; 249 } 250 251 status_t ZslProcessor::pushToReprocess(int32_t requestId) { 252 ALOGV("%s: Send in reprocess request with id %d", 253 __FUNCTION__, requestId); 254 Mutex::Autolock l(mInputMutex); 255 status_t res; 256 sp<Camera2Client> client = mClient.promote(); 257 258 if (client == 0) { 259 ALOGE("%s: Camera %d: Client does not exist", __FUNCTION__, mId); 260 return INVALID_OPERATION; 261 } 262 263 IF_ALOGV() { 264 dumpZslQueue(-1); 265 } 266 267 if (mZslQueueTail != mZslQueueHead) { 268 CameraMetadata request; 269 size_t index = mZslQueueTail; 270 while (index != mZslQueueHead) { 271 if (!mZslQueue[index].frame.isEmpty()) { 272 request = mZslQueue[index].frame; 273 break; 274 } 275 index = (index + 1) % kZslBufferDepth; 276 } 277 if (index == mZslQueueHead) { 278 ALOGV("%s: ZSL queue has no valid frames to send yet.", 279 __FUNCTION__); 280 return NOT_ENOUGH_DATA; 281 } 282 // Verify that the frame is reasonable for reprocessing 283 284 camera_metadata_entry_t entry; 285 entry = request.find(ANDROID_CONTROL_AE_STATE); 286 if (entry.count == 0) { 287 ALOGE("%s: ZSL queue frame has no AE state field!", 288 __FUNCTION__); 289 return BAD_VALUE; 290 } 291 if (entry.data.u8[0] != ANDROID_CONTROL_AE_STATE_CONVERGED && 292 entry.data.u8[0] != ANDROID_CONTROL_AE_STATE_LOCKED) { 293 ALOGV("%s: ZSL queue frame AE state is %d, need full capture", 294 __FUNCTION__, entry.data.u8[0]); 295 return NOT_ENOUGH_DATA; 296 } 297 298 buffer_handle_t *handle = 299 &(mZslQueue[index].buffer.mGraphicBuffer->handle); 300 301 uint8_t requestType = ANDROID_REQUEST_TYPE_REPROCESS; 302 res = request.update(ANDROID_REQUEST_TYPE, 303 &requestType, 1); 304 uint8_t inputStreams[1] = 305 { static_cast<uint8_t>(mZslReprocessStreamId) }; 306 if (res == OK) request.update(ANDROID_REQUEST_INPUT_STREAMS, 307 inputStreams, 1); 308 uint8_t outputStreams[1] = 309 { static_cast<uint8_t>(client->getCaptureStreamId()) }; 310 if (res == OK) request.update(ANDROID_REQUEST_OUTPUT_STREAMS, 311 outputStreams, 1); 312 res = request.update(ANDROID_REQUEST_ID, 313 &requestId, 1); 314 315 if (res != OK ) { 316 ALOGE("%s: Unable to update frame to a reprocess request", __FUNCTION__); 317 return INVALID_OPERATION; 318 } 319 320 res = client->stopStream(); 321 if (res != OK) { 322 ALOGE("%s: Camera %d: Unable to stop preview for ZSL capture: " 323 "%s (%d)", 324 __FUNCTION__, mId, strerror(-res), res); 325 return INVALID_OPERATION; 326 } 327 // TODO: have push-and-clear be atomic 328 res = client->getCameraDevice()->pushReprocessBuffer(mZslReprocessStreamId, 329 handle, this); 330 if (res != OK) { 331 ALOGE("%s: Unable to push buffer for reprocessing: %s (%d)", 332 __FUNCTION__, strerror(-res), res); 333 return res; 334 } 335 336 // Update JPEG settings 337 { 338 SharedParameters::Lock l(client->getParameters()); 339 res = l.mParameters.updateRequestJpeg(&request); 340 if (res != OK) { 341 ALOGE("%s: Camera %d: Unable to update JPEG entries of ZSL " 342 "capture request: %s (%d)", __FUNCTION__, 343 mId, 344 strerror(-res), res); 345 return res; 346 } 347 } 348 349 mLatestCapturedRequest = request; 350 res = client->getCameraDevice()->capture(request); 351 if (res != OK ) { 352 ALOGE("%s: Unable to send ZSL reprocess request to capture: %s (%d)", 353 __FUNCTION__, strerror(-res), res); 354 return res; 355 } 356 357 mState = LOCKED; 358 } else { 359 ALOGV("%s: No ZSL buffers yet", __FUNCTION__); 360 return NOT_ENOUGH_DATA; 361 } 362 return OK; 363 } 364 365 status_t ZslProcessor::clearZslQueue() { 366 Mutex::Autolock l(mInputMutex); 367 // If in middle of capture, can't clear out queue 368 if (mState == LOCKED) return OK; 369 370 return clearZslQueueLocked(); 371 } 372 373 status_t ZslProcessor::clearZslQueueLocked() { 374 for (size_t i = 0; i < mZslQueue.size(); i++) { 375 if (mZslQueue[i].buffer.mTimestamp != 0) { 376 mZslConsumer->releaseBuffer(mZslQueue[i].buffer); 377 } 378 mZslQueue.replaceAt(i); 379 } 380 mZslQueueHead = 0; 381 mZslQueueTail = 0; 382 return OK; 383 } 384 385 void ZslProcessor::dump(int fd, const Vector<String16>& /*args*/) const { 386 Mutex::Autolock l(mInputMutex); 387 if (!mLatestCapturedRequest.isEmpty()) { 388 String8 result(" Latest ZSL capture request:\n"); 389 write(fd, result.string(), result.size()); 390 mLatestCapturedRequest.dump(fd, 2, 6); 391 } else { 392 String8 result(" Latest ZSL capture request: none yet\n"); 393 write(fd, result.string(), result.size()); 394 } 395 dumpZslQueue(fd); 396 } 397 398 bool ZslProcessor::threadLoop() { 399 status_t res; 400 401 { 402 Mutex::Autolock l(mInputMutex); 403 while (!mZslBufferAvailable) { 404 res = mZslBufferAvailableSignal.waitRelative(mInputMutex, 405 kWaitDuration); 406 if (res == TIMED_OUT) return true; 407 } 408 mZslBufferAvailable = false; 409 } 410 411 do { 412 res = processNewZslBuffer(); 413 } while (res == OK); 414 415 return true; 416 } 417 418 status_t ZslProcessor::processNewZslBuffer() { 419 ATRACE_CALL(); 420 status_t res; 421 sp<BufferItemConsumer> zslConsumer; 422 { 423 Mutex::Autolock l(mInputMutex); 424 if (mZslConsumer == 0) return OK; 425 zslConsumer = mZslConsumer; 426 } 427 ALOGVV("Trying to get next buffer"); 428 BufferItemConsumer::BufferItem item; 429 res = zslConsumer->acquireBuffer(&item); 430 if (res != OK) { 431 if (res != BufferItemConsumer::NO_BUFFER_AVAILABLE) { 432 ALOGE("%s: Camera %d: Error receiving ZSL image buffer: " 433 "%s (%d)", __FUNCTION__, 434 mId, strerror(-res), res); 435 } else { 436 ALOGVV(" No buffer"); 437 } 438 return res; 439 } 440 441 Mutex::Autolock l(mInputMutex); 442 443 if (mState == LOCKED) { 444 ALOGVV("In capture, discarding new ZSL buffers"); 445 zslConsumer->releaseBuffer(item); 446 return OK; 447 } 448 449 ALOGVV("Got ZSL buffer: head: %d, tail: %d", mZslQueueHead, mZslQueueTail); 450 451 if ( (mZslQueueHead + 1) % kZslBufferDepth == mZslQueueTail) { 452 ALOGVV("Releasing oldest buffer"); 453 zslConsumer->releaseBuffer(mZslQueue[mZslQueueTail].buffer); 454 mZslQueue.replaceAt(mZslQueueTail); 455 mZslQueueTail = (mZslQueueTail + 1) % kZslBufferDepth; 456 } 457 458 ZslPair &queueHead = mZslQueue.editItemAt(mZslQueueHead); 459 460 queueHead.buffer = item; 461 queueHead.frame.release(); 462 463 mZslQueueHead = (mZslQueueHead + 1) % kZslBufferDepth; 464 465 ALOGVV(" Acquired buffer, timestamp %lld", queueHead.buffer.mTimestamp); 466 467 findMatchesLocked(); 468 469 return OK; 470 } 471 472 void ZslProcessor::findMatchesLocked() { 473 ALOGVV("Scanning"); 474 for (size_t i = 0; i < mZslQueue.size(); i++) { 475 ZslPair &queueEntry = mZslQueue.editItemAt(i); 476 nsecs_t bufferTimestamp = queueEntry.buffer.mTimestamp; 477 IF_ALOGV() { 478 camera_metadata_entry_t entry; 479 nsecs_t frameTimestamp = 0; 480 if (!queueEntry.frame.isEmpty()) { 481 entry = queueEntry.frame.find(ANDROID_SENSOR_TIMESTAMP); 482 frameTimestamp = entry.data.i64[0]; 483 } 484 ALOGVV(" %d: b: %lld\tf: %lld", i, 485 bufferTimestamp, frameTimestamp ); 486 } 487 if (queueEntry.frame.isEmpty() && bufferTimestamp != 0) { 488 // Have buffer, no matching frame. Look for one 489 for (size_t j = 0; j < mFrameList.size(); j++) { 490 bool match = false; 491 CameraMetadata &frame = mFrameList.editItemAt(j); 492 if (!frame.isEmpty()) { 493 camera_metadata_entry_t entry; 494 entry = frame.find(ANDROID_SENSOR_TIMESTAMP); 495 if (entry.count == 0) { 496 ALOGE("%s: Can't find timestamp in frame!", 497 __FUNCTION__); 498 continue; 499 } 500 nsecs_t frameTimestamp = entry.data.i64[0]; 501 if (bufferTimestamp == frameTimestamp) { 502 ALOGVV("%s: Found match %lld", __FUNCTION__, 503 frameTimestamp); 504 match = true; 505 } else { 506 int64_t delta = abs(bufferTimestamp - frameTimestamp); 507 if ( delta < 1000000) { 508 ALOGVV("%s: Found close match %lld (delta %lld)", 509 __FUNCTION__, bufferTimestamp, delta); 510 match = true; 511 } 512 } 513 } 514 if (match) { 515 queueEntry.frame.acquire(frame); 516 break; 517 } 518 } 519 } 520 } 521 } 522 523 void ZslProcessor::dumpZslQueue(int fd) const { 524 String8 header("ZSL queue contents:"); 525 String8 indent(" "); 526 ALOGV("%s", header.string()); 527 if (fd != -1) { 528 header = indent + header + "\n"; 529 write(fd, header.string(), header.size()); 530 } 531 for (size_t i = 0; i < mZslQueue.size(); i++) { 532 const ZslPair &queueEntry = mZslQueue[i]; 533 nsecs_t bufferTimestamp = queueEntry.buffer.mTimestamp; 534 camera_metadata_ro_entry_t entry; 535 nsecs_t frameTimestamp = 0; 536 int frameAeState = -1; 537 if (!queueEntry.frame.isEmpty()) { 538 entry = queueEntry.frame.find(ANDROID_SENSOR_TIMESTAMP); 539 if (entry.count > 0) frameTimestamp = entry.data.i64[0]; 540 entry = queueEntry.frame.find(ANDROID_CONTROL_AE_STATE); 541 if (entry.count > 0) frameAeState = entry.data.u8[0]; 542 } 543 String8 result = 544 String8::format(" %d: b: %lld\tf: %lld, AE state: %d", i, 545 bufferTimestamp, frameTimestamp, frameAeState); 546 ALOGV("%s", result.string()); 547 if (fd != -1) { 548 result = indent + result + "\n"; 549 write(fd, result.string(), result.size()); 550 } 551 552 } 553 } 554 555 }; // namespace camera2 556 }; // namespace android 557