1 /* 2 * Copyright 2014,2016 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17 #include <inttypes.h> 18 19 #define LOG_TAG "Camera3StreamSplitter" 20 #define ATRACE_TAG ATRACE_TAG_CAMERA 21 //#define LOG_NDEBUG 0 22 23 #include <gui/BufferItem.h> 24 #include <gui/IGraphicBufferConsumer.h> 25 #include <gui/IGraphicBufferProducer.h> 26 #include <gui/BufferQueue.h> 27 #include <gui/Surface.h> 28 29 #include <ui/GraphicBuffer.h> 30 31 #include <binder/ProcessState.h> 32 33 #include <utils/Trace.h> 34 35 #include <cutils/atomic.h> 36 37 #include "Camera3StreamSplitter.h" 38 39 namespace android { 40 41 status_t Camera3StreamSplitter::connect(const std::vector<sp<Surface> >& surfaces, 42 uint64_t consumerUsage, size_t halMaxBuffers, sp<Surface>* consumer) { 43 ATRACE_CALL(); 44 if (consumer == nullptr) { 45 SP_LOGE("%s: consumer pointer is NULL", __FUNCTION__); 46 return BAD_VALUE; 47 } 48 49 Mutex::Autolock lock(mMutex); 50 status_t res = OK; 51 52 if (mOutputs.size() > 0 || mConsumer != nullptr) { 53 SP_LOGE("%s: already connected", __FUNCTION__); 54 return BAD_VALUE; 55 } 56 if (mBuffers.size() > 0) { 57 SP_LOGE("%s: still has %zu pending buffers", __FUNCTION__, mBuffers.size()); 58 return BAD_VALUE; 59 } 60 61 mMaxHalBuffers = halMaxBuffers; 62 mConsumerName = getUniqueConsumerName(); 63 // Add output surfaces. This has to be before creating internal buffer queue 64 // in order to get max consumer side buffers. 65 for (size_t i = 0; i < surfaces.size(); i++) { 66 if (surfaces[i] == nullptr) { 67 SP_LOGE("%s: Fatal: surface is NULL", __FUNCTION__); 68 return BAD_VALUE; 69 } 70 res = addOutputLocked(surfaces[i]); 71 if (res != OK) { 72 SP_LOGE("%s: Failed to add output surface: %s(%d)", 73 __FUNCTION__, strerror(-res), res); 74 return res; 75 } 76 } 77 78 // Create BufferQueue for input 79 BufferQueue::createBufferQueue(&mProducer, &mConsumer); 80 81 // Allocate 1 extra buffer to handle the case where all buffers are detached 82 // from input, and attached to the outputs. In this case, the input queue's 83 // dequeueBuffer can still allocate 1 extra buffer before being blocked by 84 // the output's attachBuffer(). 85 mBufferItemConsumer = new BufferItemConsumer(mConsumer, consumerUsage, 86 mMaxConsumerBuffers+1); 87 if (mBufferItemConsumer == nullptr) { 88 return NO_MEMORY; 89 } 90 mConsumer->setConsumerName(mConsumerName); 91 92 *consumer = new Surface(mProducer); 93 if (*consumer == nullptr) { 94 return NO_MEMORY; 95 } 96 97 res = mConsumer->consumerConnect(this, /* controlledByApp */ false); 98 99 SP_LOGV("%s: connected", __FUNCTION__); 100 return res; 101 } 102 103 status_t Camera3StreamSplitter::getOnFrameAvailableResult() { 104 ATRACE_CALL(); 105 return mOnFrameAvailableRes.load(); 106 } 107 108 void Camera3StreamSplitter::disconnect() { 109 ATRACE_CALL(); 110 Mutex::Autolock lock(mMutex); 111 112 for (auto& notifier : mNotifiers) { 113 sp<IGraphicBufferProducer> producer = notifier.first; 114 sp<OutputListener> listener = notifier.second; 115 IInterface::asBinder(producer)->unlinkToDeath(listener); 116 } 117 mNotifiers.clear(); 118 119 for (auto& output : mOutputs) { 120 output->disconnect(NATIVE_WINDOW_API_CAMERA); 121 } 122 mOutputs.clear(); 123 mOutputSlots.clear(); 124 125 mConsumer->consumerDisconnect(); 126 127 if (mBuffers.size() > 0) { 128 SP_LOGW("%zu buffers still being tracked", mBuffers.size()); 129 mBuffers.clear(); 130 } 131 132 mMaxHalBuffers = 0; 133 mMaxConsumerBuffers = 0; 134 SP_LOGV("%s: Disconnected", __FUNCTION__); 135 } 136 137 138 Camera3StreamSplitter::~Camera3StreamSplitter() { 139 disconnect(); 140 } 141 142 status_t Camera3StreamSplitter::addOutput(const sp<Surface>& outputQueue) { 143 ATRACE_CALL(); 144 Mutex::Autolock lock(mMutex); 145 status_t res = addOutputLocked(outputQueue); 146 147 if (res != OK) { 148 SP_LOGE("%s: addOutputLocked failed %d", __FUNCTION__, res); 149 return res; 150 } 151 152 res = mConsumer->setMaxAcquiredBufferCount(mMaxConsumerBuffers+1); 153 154 return res; 155 } 156 157 status_t Camera3StreamSplitter::addOutputLocked(const sp<Surface>& outputQueue) { 158 ATRACE_CALL(); 159 if (outputQueue == nullptr) { 160 SP_LOGE("addOutput: outputQueue must not be NULL"); 161 return BAD_VALUE; 162 } 163 164 sp<IGraphicBufferProducer> gbp = outputQueue->getIGraphicBufferProducer(); 165 // Connect to the buffer producer 166 sp<OutputListener> listener(new OutputListener(this, gbp)); 167 IInterface::asBinder(gbp)->linkToDeath(listener); 168 status_t res = outputQueue->connect(NATIVE_WINDOW_API_CAMERA, listener); 169 if (res != NO_ERROR) { 170 SP_LOGE("addOutput: failed to connect (%d)", res); 171 return res; 172 } 173 174 // Query consumer side buffer count, and update overall buffer count 175 int maxConsumerBuffers = 0; 176 res = static_cast<ANativeWindow*>(outputQueue.get())->query( 177 outputQueue.get(), 178 NATIVE_WINDOW_MIN_UNDEQUEUED_BUFFERS, &maxConsumerBuffers); 179 if (res != OK) { 180 SP_LOGE("%s: Unable to query consumer undequeued buffer count" 181 " for surface", __FUNCTION__); 182 return res; 183 } 184 185 SP_LOGV("%s: Consumer wants %d buffers, Producer wants %zu", __FUNCTION__, 186 maxConsumerBuffers, mMaxHalBuffers); 187 size_t totalBufferCount = maxConsumerBuffers + mMaxHalBuffers; 188 res = native_window_set_buffer_count(outputQueue.get(), 189 totalBufferCount); 190 if (res != OK) { 191 SP_LOGE("%s: Unable to set buffer count for surface %p", 192 __FUNCTION__, outputQueue.get()); 193 return res; 194 } 195 196 // Set dequeueBuffer/attachBuffer timeout if the consumer is not hw composer or hw texture. 197 // We need skip these cases as timeout will disable the non-blocking (async) mode. 198 uint64_t usage = 0; 199 res = native_window_get_consumer_usage(static_cast<ANativeWindow*>(outputQueue.get()), &usage); 200 if (!(usage & (GRALLOC_USAGE_HW_COMPOSER | GRALLOC_USAGE_HW_TEXTURE))) { 201 outputQueue->setDequeueTimeout(kDequeueBufferTimeout); 202 } 203 204 res = gbp->allowAllocation(false); 205 if (res != OK) { 206 SP_LOGE("%s: Failed to turn off allocation for outputQueue", __FUNCTION__); 207 return res; 208 } 209 210 // Add new entry into mOutputs 211 mOutputs.push_back(gbp); 212 mNotifiers[gbp] = listener; 213 mOutputSlots[gbp] = std::make_unique<OutputSlots>(totalBufferCount); 214 215 mMaxConsumerBuffers += maxConsumerBuffers; 216 return NO_ERROR; 217 } 218 219 status_t Camera3StreamSplitter::outputBufferLocked(const sp<IGraphicBufferProducer>& output, 220 const BufferItem& bufferItem) { 221 ATRACE_CALL(); 222 status_t res; 223 IGraphicBufferProducer::QueueBufferInput queueInput( 224 bufferItem.mTimestamp, bufferItem.mIsAutoTimestamp, 225 bufferItem.mDataSpace, bufferItem.mCrop, 226 static_cast<int32_t>(bufferItem.mScalingMode), 227 bufferItem.mTransform, bufferItem.mFence); 228 229 IGraphicBufferProducer::QueueBufferOutput queueOutput; 230 231 uint64_t bufferId = bufferItem.mGraphicBuffer->getId(); 232 const BufferTracker& tracker = *(mBuffers[bufferId]); 233 int slot = getSlotForOutputLocked(output, tracker.getBuffer()); 234 235 // In case the output BufferQueue has its own lock, if we hold splitter lock while calling 236 // queueBuffer (which will try to acquire the output lock), the output could be holding its 237 // own lock calling releaseBuffer (which will try to acquire the splitter lock), running into 238 // circular lock situation. 239 mMutex.unlock(); 240 res = output->queueBuffer(slot, queueInput, &queueOutput); 241 mMutex.lock(); 242 243 SP_LOGV("%s: Queuing buffer to buffer queue %p slot %d returns %d", 244 __FUNCTION__, output.get(), slot, res); 245 if (res != OK) { 246 if (res != NO_INIT && res != DEAD_OBJECT) { 247 SP_LOGE("Queuing buffer to output failed (%d)", res); 248 } 249 // If we just discovered that this output has been abandoned, note 250 // that, increment the release count so that we still release this 251 // buffer eventually, and move on to the next output 252 onAbandonedLocked(); 253 decrementBufRefCountLocked(bufferItem.mGraphicBuffer->getId(), output); 254 return res; 255 } 256 257 // If the queued buffer replaces a pending buffer in the async 258 // queue, no onBufferReleased is called by the buffer queue. 259 // Proactively trigger the callback to avoid buffer loss. 260 if (queueOutput.bufferReplaced) { 261 onBufferReleasedByOutputLocked(output); 262 } 263 264 return res; 265 } 266 267 String8 Camera3StreamSplitter::getUniqueConsumerName() { 268 static volatile int32_t counter = 0; 269 return String8::format("Camera3StreamSplitter-%d", android_atomic_inc(&counter)); 270 } 271 272 status_t Camera3StreamSplitter::notifyBufferReleased(const sp<GraphicBuffer>& buffer) { 273 ATRACE_CALL(); 274 status_t res = OK; 275 276 Mutex::Autolock lock(mMutex); 277 278 uint64_t bufferId = buffer->getId(); 279 std::unique_ptr<BufferTracker> tracker_ptr = std::move(mBuffers[bufferId]); 280 mBuffers.erase(bufferId); 281 282 for (const auto surface : tracker_ptr->requestedSurfaces()) { 283 sp<IGraphicBufferProducer>& gbp = mOutputs[surface]; 284 OutputSlots& outputSlots = *(mOutputSlots[gbp]); 285 int slot = getSlotForOutputLocked(gbp, buffer); 286 if (slot != BufferItem::INVALID_BUFFER_SLOT) { 287 gbp->detachBuffer(slot); 288 outputSlots[slot].clear(); 289 } 290 } 291 292 return res; 293 } 294 295 status_t Camera3StreamSplitter::attachBufferToOutputs(ANativeWindowBuffer* anb, 296 const std::vector<size_t>& surface_ids) { 297 ATRACE_CALL(); 298 status_t res = OK; 299 300 Mutex::Autolock lock(mMutex); 301 302 sp<GraphicBuffer> gb(static_cast<GraphicBuffer*>(anb)); 303 uint64_t bufferId = gb->getId(); 304 305 // Initialize buffer tracker for this input buffer 306 auto tracker = std::make_unique<BufferTracker>(gb, surface_ids); 307 308 for (auto& surface_id : surface_ids) { 309 sp<IGraphicBufferProducer>& gbp = mOutputs[surface_id]; 310 int slot = BufferItem::INVALID_BUFFER_SLOT; 311 //Temporarly Unlock the mutex when trying to attachBuffer to the output 312 //queue, because attachBuffer could block in case of a slow consumer. If 313 //we block while holding the lock, onFrameAvailable and onBufferReleased 314 //will block as well because they need to acquire the same lock. 315 mMutex.unlock(); 316 res = gbp->attachBuffer(&slot, gb); 317 mMutex.lock(); 318 if (res != OK) { 319 SP_LOGE("%s: Cannot acquireBuffer from GraphicBufferProducer %p: %s (%d)", 320 __FUNCTION__, gbp.get(), strerror(-res), res); 321 return res; 322 } 323 auto& outputSlots = *mOutputSlots[gbp]; 324 if (outputSlots[slot] != nullptr) { 325 // If the buffer is attached to a slot which already contains a buffer, 326 // the previous buffer will be removed from the output queue. Decrement 327 // the reference count accordingly. 328 decrementBufRefCountLocked(outputSlots[slot]->getId(), gbp); 329 } 330 SP_LOGV("%s: Attached buffer %p to slot %d on output %p.",__FUNCTION__, gb.get(), 331 slot, gbp.get()); 332 outputSlots[slot] = gb; 333 } 334 335 mBuffers[bufferId] = std::move(tracker); 336 337 return res; 338 } 339 340 void Camera3StreamSplitter::onFrameAvailable(const BufferItem& /*item*/) { 341 ATRACE_CALL(); 342 Mutex::Autolock lock(mMutex); 343 344 // Acquire and detach the buffer from the input 345 BufferItem bufferItem; 346 status_t res = mConsumer->acquireBuffer(&bufferItem, /* presentWhen */ 0); 347 if (res != NO_ERROR) { 348 SP_LOGE("%s: Acquiring buffer from input failed (%d)", __FUNCTION__, res); 349 mOnFrameAvailableRes.store(res); 350 return; 351 } 352 if (mBuffers.find(bufferItem.mGraphicBuffer->getId()) == mBuffers.end()) { 353 SP_LOGE("%s: Acquired buffer doesn't exist in attached buffer map", 354 __FUNCTION__); 355 mOnFrameAvailableRes.store(INVALID_OPERATION); 356 return; 357 } 358 359 SP_LOGV("acquired buffer %" PRId64 " from input at slot %d", 360 bufferItem.mGraphicBuffer->getId(), bufferItem.mSlot); 361 362 res = mConsumer->detachBuffer(bufferItem.mSlot); 363 if (res != NO_ERROR) { 364 SP_LOGE("%s: detaching buffer from input failed (%d)", __FUNCTION__, res); 365 mOnFrameAvailableRes.store(res); 366 return; 367 } 368 369 // Attach and queue the buffer to each of the outputs 370 BufferTracker& tracker = *(mBuffers[bufferItem.mGraphicBuffer->getId()]); 371 372 SP_LOGV("%s: BufferTracker for buffer %" PRId64 ", number of requests %zu", 373 __FUNCTION__, bufferItem.mGraphicBuffer->getId(), tracker.requestedSurfaces().size()); 374 for (const auto id : tracker.requestedSurfaces()) { 375 376 LOG_ALWAYS_FATAL_IF(id >= mOutputs.size(), 377 "requested surface id exceeding max registered ids"); 378 379 res = outputBufferLocked(mOutputs[id], bufferItem); 380 if (res != OK) { 381 SP_LOGE("%s: outputBufferLocked failed %d", __FUNCTION__, res); 382 mOnFrameAvailableRes.store(res); 383 // If we fail to send buffer to certain output, keep sending to 384 // other outputs. 385 continue; 386 } 387 } 388 389 mOnFrameAvailableRes.store(res); 390 } 391 392 void Camera3StreamSplitter::decrementBufRefCountLocked(uint64_t id, 393 const sp<IGraphicBufferProducer>& from) { 394 ATRACE_CALL(); 395 size_t referenceCount = mBuffers[id]->decrementReferenceCountLocked(); 396 397 removeSlotForOutputLocked(from, mBuffers[id]->getBuffer()); 398 if (referenceCount > 0) { 399 return; 400 } 401 402 // We no longer need to track the buffer now that it is being returned to the 403 // input. Note that this should happen before we unlock the mutex and call 404 // releaseBuffer, to avoid the case where the same bufferId is acquired in 405 // attachBufferToOutputs resulting in a new BufferTracker with same bufferId 406 // overwrites the current one. 407 std::unique_ptr<BufferTracker> tracker_ptr = std::move(mBuffers[id]); 408 mBuffers.erase(id); 409 410 // Attach and release the buffer back to the input 411 int consumerSlot = BufferItem::INVALID_BUFFER_SLOT; 412 status_t res = mConsumer->attachBuffer(&consumerSlot, tracker_ptr->getBuffer()); 413 if (res != NO_ERROR) { 414 SP_LOGE("%s: attaching buffer to input failed (%d)", __FUNCTION__, res); 415 return; 416 } 417 418 // Temporarily unlock mutex to avoid circular lock: 419 // 1. This function holds splitter lock, calls releaseBuffer which triggers 420 // onBufferReleased in Camera3OutputStream. onBufferReleased waits on the 421 // OutputStream lock 422 // 2. Camera3SharedOutputStream::getBufferLocked calls 423 // attachBufferToOutputs, which holds the stream lock, and waits for the 424 // splitter lock. 425 sp<IGraphicBufferConsumer> consumer(mConsumer); 426 mMutex.unlock(); 427 if (consumer != nullptr) { 428 res = consumer->releaseBuffer(consumerSlot, /* frameNumber */ 0, 429 EGL_NO_DISPLAY, EGL_NO_SYNC_KHR, tracker_ptr->getMergedFence()); 430 } else { 431 SP_LOGE("%s: consumer has become null!", __FUNCTION__); 432 } 433 mMutex.lock(); 434 // If the producer of this queue is disconnected, -22 error will occur 435 if (res != NO_ERROR) { 436 SP_LOGE("%s: releaseBuffer returns %d", __FUNCTION__, res); 437 } 438 } 439 440 void Camera3StreamSplitter::onBufferReleasedByOutput( 441 const sp<IGraphicBufferProducer>& from) { 442 ATRACE_CALL(); 443 Mutex::Autolock lock(mMutex); 444 445 onBufferReleasedByOutputLocked(from); 446 } 447 448 void Camera3StreamSplitter::onBufferReleasedByOutputLocked( 449 const sp<IGraphicBufferProducer>& from) { 450 ATRACE_CALL(); 451 sp<GraphicBuffer> buffer; 452 sp<Fence> fence; 453 status_t res = from->detachNextBuffer(&buffer, &fence); 454 if (res == NO_INIT) { 455 // If we just discovered that this output has been abandoned, note that, 456 // but we can't do anything else, since buffer is invalid 457 onAbandonedLocked(); 458 return; 459 } else if (res == NO_MEMORY) { 460 SP_LOGV("%s: No free buffers", __FUNCTION__); 461 return; 462 } else if (res != OK) { 463 SP_LOGE("%s: detaching buffer from output failed (%d)", __FUNCTION__, res); 464 return; 465 } 466 467 BufferTracker& tracker = *(mBuffers[buffer->getId()]); 468 // Merge the release fence of the incoming buffer so that the fence we send 469 // back to the input includes all of the outputs' fences 470 if (fence != nullptr && fence->isValid()) { 471 tracker.mergeFence(fence); 472 } 473 SP_LOGV("detached buffer %" PRId64 " %p from output %p", 474 buffer->getId(), buffer.get(), from.get()); 475 476 // Check to see if this is the last outstanding reference to this buffer 477 decrementBufRefCountLocked(buffer->getId(), from); 478 } 479 480 void Camera3StreamSplitter::onAbandonedLocked() { 481 // If this is called from binderDied callback, it means the app process 482 // holding the binder has died. CameraService will be notified of the binder 483 // death, and camera device will be closed, which in turn calls 484 // disconnect(). 485 // 486 // If this is called from onBufferReleasedByOutput or onFrameAvailable, one 487 // consumer being abanoned shouldn't impact the other consumer. So we won't 488 // stop the buffer flow. 489 // 490 // In both cases, we don't need to do anything here. 491 SP_LOGV("One of my outputs has abandoned me"); 492 } 493 494 int Camera3StreamSplitter::getSlotForOutputLocked(const sp<IGraphicBufferProducer>& gbp, 495 const sp<GraphicBuffer>& gb) { 496 auto& outputSlots = *mOutputSlots[gbp]; 497 498 for (size_t i = 0; i < outputSlots.size(); i++) { 499 if (outputSlots[i] == gb) { 500 return (int)i; 501 } 502 } 503 504 SP_LOGE("%s: Cannot find slot for gb %p on output %p", __FUNCTION__, gb.get(), 505 gbp.get()); 506 return BufferItem::INVALID_BUFFER_SLOT; 507 } 508 509 status_t Camera3StreamSplitter::removeSlotForOutputLocked(const sp<IGraphicBufferProducer>& gbp, 510 const sp<GraphicBuffer>& gb) { 511 auto& outputSlots = *mOutputSlots[gbp]; 512 513 for (size_t i = 0; i < outputSlots.size(); i++) { 514 if (outputSlots[i] == gb) { 515 outputSlots[i].clear(); 516 return NO_ERROR; 517 } 518 } 519 520 SP_LOGE("%s: Cannot find slot for gb %p on output %p", __FUNCTION__, gb.get(), 521 gbp.get()); 522 return BAD_VALUE; 523 } 524 525 Camera3StreamSplitter::OutputListener::OutputListener( 526 wp<Camera3StreamSplitter> splitter, 527 wp<IGraphicBufferProducer> output) 528 : mSplitter(splitter), mOutput(output) {} 529 530 void Camera3StreamSplitter::OutputListener::onBufferReleased() { 531 ATRACE_CALL(); 532 sp<Camera3StreamSplitter> splitter = mSplitter.promote(); 533 sp<IGraphicBufferProducer> output = mOutput.promote(); 534 if (splitter != nullptr && output != nullptr) { 535 splitter->onBufferReleasedByOutput(output); 536 } 537 } 538 539 void Camera3StreamSplitter::OutputListener::binderDied(const wp<IBinder>& /* who */) { 540 sp<Camera3StreamSplitter> splitter = mSplitter.promote(); 541 if (splitter != nullptr) { 542 Mutex::Autolock lock(splitter->mMutex); 543 splitter->onAbandonedLocked(); 544 } 545 } 546 547 Camera3StreamSplitter::BufferTracker::BufferTracker( 548 const sp<GraphicBuffer>& buffer, const std::vector<size_t>& requestedSurfaces) 549 : mBuffer(buffer), mMergedFence(Fence::NO_FENCE), mRequestedSurfaces(requestedSurfaces), 550 mReferenceCount(requestedSurfaces.size()) {} 551 552 void Camera3StreamSplitter::BufferTracker::mergeFence(const sp<Fence>& with) { 553 mMergedFence = Fence::merge(String8("Camera3StreamSplitter"), mMergedFence, with); 554 } 555 556 size_t Camera3StreamSplitter::BufferTracker::decrementReferenceCountLocked() { 557 if (mReferenceCount > 0) 558 --mReferenceCount; 559 return mReferenceCount; 560 } 561 562 } // namespace android 563