1 /* 2 * Copyright 2017, The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17 //#define LOG_NDEBUG 0 18 #define LOG_TAG "CCodecBufferChannel" 19 #include <utils/Log.h> 20 21 #include <numeric> 22 23 #include <C2AllocatorGralloc.h> 24 #include <C2PlatformSupport.h> 25 #include <C2BlockInternal.h> 26 #include <C2Config.h> 27 #include <C2Debug.h> 28 29 #include <android/hardware/cas/native/1.0/IDescrambler.h> 30 #include <android-base/stringprintf.h> 31 #include <binder/MemoryDealer.h> 32 #include <gui/Surface.h> 33 #include <media/openmax/OMX_Core.h> 34 #include <media/stagefright/foundation/ABuffer.h> 35 #include <media/stagefright/foundation/ALookup.h> 36 #include <media/stagefright/foundation/AMessage.h> 37 #include <media/stagefright/foundation/AUtils.h> 38 #include <media/stagefright/foundation/hexdump.h> 39 #include <media/stagefright/MediaCodec.h> 40 #include <media/stagefright/MediaCodecConstants.h> 41 #include <media/MediaCodecBuffer.h> 42 #include <system/window.h> 43 44 #include "CCodecBufferChannel.h" 45 #include "Codec2Buffer.h" 46 #include "SkipCutBuffer.h" 47 48 namespace android { 49 50 using android::base::StringPrintf; 51 using hardware::hidl_handle; 52 using hardware::hidl_string; 53 using hardware::hidl_vec; 54 using namespace hardware::cas::V1_0; 55 using namespace hardware::cas::native::V1_0; 56 57 using CasStatus = hardware::cas::V1_0::Status; 58 59 namespace { 60 61 constexpr size_t kSmoothnessFactor = 4; 62 constexpr size_t kRenderingDepth = 3; 63 64 // This is for keeping IGBP's buffer dropping logic in legacy mode other 65 // than making it non-blocking. Do not change this value. 66 const static size_t kDequeueTimeoutNs = 0; 67 68 } // namespace 69 70 CCodecBufferChannel::QueueGuard::QueueGuard( 71 CCodecBufferChannel::QueueSync &sync) : mSync(sync) { 72 Mutex::Autolock l(mSync.mGuardLock); 73 // At this point it's guaranteed that mSync is not under state transition, 74 // as we are holding its mutex. 75 76 Mutexed<CCodecBufferChannel::QueueSync::Counter>::Locked count(mSync.mCount); 77 if (count->value == -1) { 78 mRunning = false; 79 } else { 80 ++count->value; 81 mRunning = true; 82 } 83 } 84 85 CCodecBufferChannel::QueueGuard::~QueueGuard() { 86 if (mRunning) { 87 // We are not holding mGuardLock at this point so that QueueSync::stop() can 88 // keep holding the lock until mCount reaches zero. 89 Mutexed<CCodecBufferChannel::QueueSync::Counter>::Locked count(mSync.mCount); 90 --count->value; 91 count->cond.broadcast(); 92 } 93 } 94 95 void CCodecBufferChannel::QueueSync::start() { 96 Mutex::Autolock l(mGuardLock); 97 // If stopped, it goes to running state; otherwise no-op. 98 Mutexed<Counter>::Locked count(mCount); 99 if (count->value == -1) { 100 count->value = 0; 101 } 102 } 103 104 void CCodecBufferChannel::QueueSync::stop() { 105 Mutex::Autolock l(mGuardLock); 106 Mutexed<Counter>::Locked count(mCount); 107 if (count->value == -1) { 108 // no-op 109 return; 110 } 111 // Holding mGuardLock here blocks creation of additional QueueGuard objects, so 112 // mCount can only decrement. In other words, threads that acquired the lock 113 // are allowed to finish execution but additional threads trying to acquire 114 // the lock at this point will block, and then get QueueGuard at STOPPED 115 // state. 116 while (count->value != 0) { 117 count.waitForCondition(count->cond); 118 } 119 count->value = -1; 120 } 121 122 // CCodecBufferChannel::ReorderStash 123 124 CCodecBufferChannel::ReorderStash::ReorderStash() { 125 clear(); 126 } 127 128 void CCodecBufferChannel::ReorderStash::clear() { 129 mPending.clear(); 130 mStash.clear(); 131 mDepth = 0; 132 mKey = C2Config::ORDINAL; 133 } 134 135 void CCodecBufferChannel::ReorderStash::flush() { 136 mPending.clear(); 137 mStash.clear(); 138 } 139 140 void CCodecBufferChannel::ReorderStash::setDepth(uint32_t depth) { 141 mPending.splice(mPending.end(), mStash); 142 mDepth = depth; 143 } 144 145 void CCodecBufferChannel::ReorderStash::setKey(C2Config::ordinal_key_t key) { 146 mPending.splice(mPending.end(), mStash); 147 mKey = key; 148 } 149 150 bool CCodecBufferChannel::ReorderStash::pop(Entry *entry) { 151 if (mPending.empty()) { 152 return false; 153 } 154 entry->buffer = mPending.front().buffer; 155 entry->timestamp = mPending.front().timestamp; 156 entry->flags = mPending.front().flags; 157 entry->ordinal = mPending.front().ordinal; 158 mPending.pop_front(); 159 return true; 160 } 161 162 void CCodecBufferChannel::ReorderStash::emplace( 163 const std::shared_ptr<C2Buffer> &buffer, 164 int64_t timestamp, 165 int32_t flags, 166 const C2WorkOrdinalStruct &ordinal) { 167 bool eos = flags & MediaCodec::BUFFER_FLAG_EOS; 168 if (!buffer && eos) { 169 // TRICKY: we may be violating ordering of the stash here. Because we 170 // don't expect any more emplace() calls after this, the ordering should 171 // not matter. 172 mStash.emplace_back(buffer, timestamp, flags, ordinal); 173 } else { 174 flags = flags & ~MediaCodec::BUFFER_FLAG_EOS; 175 auto it = mStash.begin(); 176 for (; it != mStash.end(); ++it) { 177 if (less(ordinal, it->ordinal)) { 178 break; 179 } 180 } 181 mStash.emplace(it, buffer, timestamp, flags, ordinal); 182 if (eos) { 183 mStash.back().flags = mStash.back().flags | MediaCodec::BUFFER_FLAG_EOS; 184 } 185 } 186 while (!mStash.empty() && mStash.size() > mDepth) { 187 mPending.push_back(mStash.front()); 188 mStash.pop_front(); 189 } 190 } 191 192 void CCodecBufferChannel::ReorderStash::defer( 193 const CCodecBufferChannel::ReorderStash::Entry &entry) { 194 mPending.push_front(entry); 195 } 196 197 bool CCodecBufferChannel::ReorderStash::hasPending() const { 198 return !mPending.empty(); 199 } 200 201 bool CCodecBufferChannel::ReorderStash::less( 202 const C2WorkOrdinalStruct &o1, const C2WorkOrdinalStruct &o2) { 203 switch (mKey) { 204 case C2Config::ORDINAL: return o1.frameIndex < o2.frameIndex; 205 case C2Config::TIMESTAMP: return o1.timestamp < o2.timestamp; 206 case C2Config::CUSTOM: return o1.customOrdinal < o2.customOrdinal; 207 default: 208 ALOGD("Unrecognized key; default to timestamp"); 209 return o1.frameIndex < o2.frameIndex; 210 } 211 } 212 213 // Input 214 215 CCodecBufferChannel::Input::Input() : extraBuffers("extra") {} 216 217 // CCodecBufferChannel 218 219 CCodecBufferChannel::CCodecBufferChannel( 220 const std::shared_ptr<CCodecCallback> &callback) 221 : mHeapSeqNum(-1), 222 mCCodecCallback(callback), 223 mFrameIndex(0u), 224 mFirstValidFrameIndex(0u), 225 mMetaMode(MODE_NONE), 226 mInputMetEos(false) { 227 mOutputSurface.lock()->maxDequeueBuffers = kSmoothnessFactor + kRenderingDepth; 228 { 229 Mutexed<Input>::Locked input(mInput); 230 input->buffers.reset(new DummyInputBuffers("")); 231 input->extraBuffers.flush(); 232 input->inputDelay = 0u; 233 input->pipelineDelay = 0u; 234 input->numSlots = kSmoothnessFactor; 235 input->numExtraSlots = 0u; 236 } 237 { 238 Mutexed<Output>::Locked output(mOutput); 239 output->outputDelay = 0u; 240 output->numSlots = kSmoothnessFactor; 241 } 242 } 243 244 CCodecBufferChannel::~CCodecBufferChannel() { 245 if (mCrypto != nullptr && mDealer != nullptr && mHeapSeqNum >= 0) { 246 mCrypto->unsetHeap(mHeapSeqNum); 247 } 248 } 249 250 void CCodecBufferChannel::setComponent( 251 const std::shared_ptr<Codec2Client::Component> &component) { 252 mComponent = component; 253 mComponentName = component->getName() + StringPrintf("#%d", int(uintptr_t(component.get()) % 997)); 254 mName = mComponentName.c_str(); 255 } 256 257 status_t CCodecBufferChannel::setInputSurface( 258 const std::shared_ptr<InputSurfaceWrapper> &surface) { 259 ALOGV("[%s] setInputSurface", mName); 260 mInputSurface = surface; 261 return mInputSurface->connect(mComponent); 262 } 263 264 status_t CCodecBufferChannel::signalEndOfInputStream() { 265 if (mInputSurface == nullptr) { 266 return INVALID_OPERATION; 267 } 268 return mInputSurface->signalEndOfInputStream(); 269 } 270 271 status_t CCodecBufferChannel::queueInputBufferInternal(sp<MediaCodecBuffer> buffer) { 272 int64_t timeUs; 273 CHECK(buffer->meta()->findInt64("timeUs", &timeUs)); 274 275 if (mInputMetEos) { 276 ALOGD("[%s] buffers after EOS ignored (%lld us)", mName, (long long)timeUs); 277 return OK; 278 } 279 280 int32_t flags = 0; 281 int32_t tmp = 0; 282 bool eos = false; 283 if (buffer->meta()->findInt32("eos", &tmp) && tmp) { 284 eos = true; 285 mInputMetEos = true; 286 ALOGV("[%s] input EOS", mName); 287 } 288 if (buffer->meta()->findInt32("csd", &tmp) && tmp) { 289 flags |= C2FrameData::FLAG_CODEC_CONFIG; 290 } 291 ALOGV("[%s] queueInputBuffer: buffer->size() = %zu", mName, buffer->size()); 292 std::unique_ptr<C2Work> work(new C2Work); 293 work->input.ordinal.timestamp = timeUs; 294 work->input.ordinal.frameIndex = mFrameIndex++; 295 // WORKAROUND: until codecs support handling work after EOS and max output sizing, use timestamp 296 // manipulation to achieve image encoding via video codec, and to constrain encoded output. 297 // Keep client timestamp in customOrdinal 298 work->input.ordinal.customOrdinal = timeUs; 299 work->input.buffers.clear(); 300 301 uint64_t queuedFrameIndex = work->input.ordinal.frameIndex.peeku(); 302 std::vector<std::shared_ptr<C2Buffer>> queuedBuffers; 303 sp<Codec2Buffer> copy; 304 305 if (buffer->size() > 0u) { 306 Mutexed<Input>::Locked input(mInput); 307 std::shared_ptr<C2Buffer> c2buffer; 308 if (!input->buffers->releaseBuffer(buffer, &c2buffer, false)) { 309 return -ENOENT; 310 } 311 // TODO: we want to delay copying buffers. 312 if (input->extraBuffers.numComponentBuffers() < input->numExtraSlots) { 313 copy = input->buffers->cloneAndReleaseBuffer(buffer); 314 if (copy != nullptr) { 315 (void)input->extraBuffers.assignSlot(copy); 316 if (!input->extraBuffers.releaseSlot(copy, &c2buffer, false)) { 317 return UNKNOWN_ERROR; 318 } 319 bool released = input->buffers->releaseBuffer(buffer, nullptr, true); 320 ALOGV("[%s] queueInputBuffer: buffer copied; %sreleased", 321 mName, released ? "" : "not "); 322 buffer.clear(); 323 } else { 324 ALOGW("[%s] queueInputBuffer: failed to copy a buffer; this may cause input " 325 "buffer starvation on component.", mName); 326 } 327 } 328 work->input.buffers.push_back(c2buffer); 329 queuedBuffers.push_back(c2buffer); 330 } else if (eos) { 331 flags |= C2FrameData::FLAG_END_OF_STREAM; 332 } 333 work->input.flags = (C2FrameData::flags_t)flags; 334 // TODO: fill info's 335 336 work->input.configUpdate = std::move(mParamsToBeSet); 337 work->worklets.clear(); 338 work->worklets.emplace_back(new C2Worklet); 339 340 std::list<std::unique_ptr<C2Work>> items; 341 items.push_back(std::move(work)); 342 mPipelineWatcher.lock()->onWorkQueued( 343 queuedFrameIndex, 344 std::move(queuedBuffers), 345 PipelineWatcher::Clock::now()); 346 c2_status_t err = mComponent->queue(&items); 347 if (err != C2_OK) { 348 mPipelineWatcher.lock()->onWorkDone(queuedFrameIndex); 349 } 350 351 if (err == C2_OK && eos && buffer->size() > 0u) { 352 work.reset(new C2Work); 353 work->input.ordinal.timestamp = timeUs; 354 work->input.ordinal.frameIndex = mFrameIndex++; 355 // WORKAROUND: keep client timestamp in customOrdinal 356 work->input.ordinal.customOrdinal = timeUs; 357 work->input.buffers.clear(); 358 work->input.flags = C2FrameData::FLAG_END_OF_STREAM; 359 work->worklets.emplace_back(new C2Worklet); 360 361 queuedFrameIndex = work->input.ordinal.frameIndex.peeku(); 362 queuedBuffers.clear(); 363 364 items.clear(); 365 items.push_back(std::move(work)); 366 367 mPipelineWatcher.lock()->onWorkQueued( 368 queuedFrameIndex, 369 std::move(queuedBuffers), 370 PipelineWatcher::Clock::now()); 371 err = mComponent->queue(&items); 372 if (err != C2_OK) { 373 mPipelineWatcher.lock()->onWorkDone(queuedFrameIndex); 374 } 375 } 376 if (err == C2_OK) { 377 Mutexed<Input>::Locked input(mInput); 378 bool released = false; 379 if (buffer) { 380 released = input->buffers->releaseBuffer(buffer, nullptr, true); 381 } else if (copy) { 382 released = input->extraBuffers.releaseSlot(copy, nullptr, true); 383 } 384 ALOGV("[%s] queueInputBuffer: buffer%s %sreleased", 385 mName, (buffer == nullptr) ? "(copy)" : "", released ? "" : "not "); 386 } 387 388 feedInputBufferIfAvailableInternal(); 389 return err; 390 } 391 392 status_t CCodecBufferChannel::setParameters(std::vector<std::unique_ptr<C2Param>> ¶ms) { 393 QueueGuard guard(mSync); 394 if (!guard.isRunning()) { 395 ALOGD("[%s] setParameters is only supported in the running state.", mName); 396 return -ENOSYS; 397 } 398 mParamsToBeSet.insert(mParamsToBeSet.end(), 399 std::make_move_iterator(params.begin()), 400 std::make_move_iterator(params.end())); 401 params.clear(); 402 return OK; 403 } 404 405 status_t CCodecBufferChannel::queueInputBuffer(const sp<MediaCodecBuffer> &buffer) { 406 QueueGuard guard(mSync); 407 if (!guard.isRunning()) { 408 ALOGD("[%s] No more buffers should be queued at current state.", mName); 409 return -ENOSYS; 410 } 411 return queueInputBufferInternal(buffer); 412 } 413 414 status_t CCodecBufferChannel::queueSecureInputBuffer( 415 const sp<MediaCodecBuffer> &buffer, bool secure, const uint8_t *key, 416 const uint8_t *iv, CryptoPlugin::Mode mode, CryptoPlugin::Pattern pattern, 417 const CryptoPlugin::SubSample *subSamples, size_t numSubSamples, 418 AString *errorDetailMsg) { 419 QueueGuard guard(mSync); 420 if (!guard.isRunning()) { 421 ALOGD("[%s] No more buffers should be queued at current state.", mName); 422 return -ENOSYS; 423 } 424 425 if (!hasCryptoOrDescrambler()) { 426 return -ENOSYS; 427 } 428 sp<EncryptedLinearBlockBuffer> encryptedBuffer((EncryptedLinearBlockBuffer *)buffer.get()); 429 430 ssize_t result = -1; 431 ssize_t codecDataOffset = 0; 432 if (mCrypto != nullptr) { 433 ICrypto::DestinationBuffer destination; 434 if (secure) { 435 destination.mType = ICrypto::kDestinationTypeNativeHandle; 436 destination.mHandle = encryptedBuffer->handle(); 437 } else { 438 destination.mType = ICrypto::kDestinationTypeSharedMemory; 439 destination.mSharedMemory = mDecryptDestination; 440 } 441 ICrypto::SourceBuffer source; 442 encryptedBuffer->fillSourceBuffer(&source); 443 result = mCrypto->decrypt( 444 key, iv, mode, pattern, source, buffer->offset(), 445 subSamples, numSubSamples, destination, errorDetailMsg); 446 if (result < 0) { 447 return result; 448 } 449 if (destination.mType == ICrypto::kDestinationTypeSharedMemory) { 450 encryptedBuffer->copyDecryptedContent(mDecryptDestination, result); 451 } 452 } else { 453 // Here we cast CryptoPlugin::SubSample to hardware::cas::native::V1_0::SubSample 454 // directly, the structure definitions should match as checked in DescramblerImpl.cpp. 455 hidl_vec<SubSample> hidlSubSamples; 456 hidlSubSamples.setToExternal((SubSample *)subSamples, numSubSamples, false /*own*/); 457 458 hardware::cas::native::V1_0::SharedBuffer srcBuffer; 459 encryptedBuffer->fillSourceBuffer(&srcBuffer); 460 461 DestinationBuffer dstBuffer; 462 if (secure) { 463 dstBuffer.type = BufferType::NATIVE_HANDLE; 464 dstBuffer.secureMemory = hidl_handle(encryptedBuffer->handle()); 465 } else { 466 dstBuffer.type = BufferType::SHARED_MEMORY; 467 dstBuffer.nonsecureMemory = srcBuffer; 468 } 469 470 CasStatus status = CasStatus::OK; 471 hidl_string detailedError; 472 ScramblingControl sctrl = ScramblingControl::UNSCRAMBLED; 473 474 if (key != nullptr) { 475 sctrl = (ScramblingControl)key[0]; 476 // Adjust for the PES offset 477 codecDataOffset = key[2] | (key[3] << 8); 478 } 479 480 auto returnVoid = mDescrambler->descramble( 481 sctrl, 482 hidlSubSamples, 483 srcBuffer, 484 0, 485 dstBuffer, 486 0, 487 [&status, &result, &detailedError] ( 488 CasStatus _status, uint32_t _bytesWritten, 489 const hidl_string& _detailedError) { 490 status = _status; 491 result = (ssize_t)_bytesWritten; 492 detailedError = _detailedError; 493 }); 494 495 if (!returnVoid.isOk() || status != CasStatus::OK || result < 0) { 496 ALOGI("[%s] descramble failed, trans=%s, status=%d, result=%zd", 497 mName, returnVoid.description().c_str(), status, result); 498 return UNKNOWN_ERROR; 499 } 500 501 if (result < codecDataOffset) { 502 ALOGD("invalid codec data offset: %zd, result %zd", codecDataOffset, result); 503 return BAD_VALUE; 504 } 505 506 ALOGV("[%s] descramble succeeded, %zd bytes", mName, result); 507 508 if (dstBuffer.type == BufferType::SHARED_MEMORY) { 509 encryptedBuffer->copyDecryptedContentFromMemory(result); 510 } 511 } 512 513 buffer->setRange(codecDataOffset, result - codecDataOffset); 514 return queueInputBufferInternal(buffer); 515 } 516 517 void CCodecBufferChannel::feedInputBufferIfAvailable() { 518 QueueGuard guard(mSync); 519 if (!guard.isRunning()) { 520 ALOGV("[%s] We're not running --- no input buffer reported", mName); 521 return; 522 } 523 feedInputBufferIfAvailableInternal(); 524 } 525 526 void CCodecBufferChannel::feedInputBufferIfAvailableInternal() { 527 if (mInputMetEos || 528 mReorderStash.lock()->hasPending() || 529 mPipelineWatcher.lock()->pipelineFull()) { 530 return; 531 } else { 532 Mutexed<Output>::Locked output(mOutput); 533 if (output->buffers->numClientBuffers() >= output->numSlots) { 534 return; 535 } 536 } 537 size_t numInputSlots = mInput.lock()->numSlots; 538 for (size_t i = 0; i < numInputSlots; ++i) { 539 sp<MediaCodecBuffer> inBuffer; 540 size_t index; 541 { 542 Mutexed<Input>::Locked input(mInput); 543 if (input->buffers->numClientBuffers() >= input->numSlots) { 544 return; 545 } 546 if (!input->buffers->requestNewBuffer(&index, &inBuffer)) { 547 ALOGV("[%s] no new buffer available", mName); 548 break; 549 } 550 } 551 ALOGV("[%s] new input index = %zu [%p]", mName, index, inBuffer.get()); 552 mCallback->onInputBufferAvailable(index, inBuffer); 553 } 554 } 555 556 status_t CCodecBufferChannel::renderOutputBuffer( 557 const sp<MediaCodecBuffer> &buffer, int64_t timestampNs) { 558 ALOGV("[%s] renderOutputBuffer: %p", mName, buffer.get()); 559 std::shared_ptr<C2Buffer> c2Buffer; 560 bool released = false; 561 { 562 Mutexed<Output>::Locked output(mOutput); 563 if (output->buffers) { 564 released = output->buffers->releaseBuffer(buffer, &c2Buffer); 565 } 566 } 567 // NOTE: some apps try to releaseOutputBuffer() with timestamp and/or render 568 // set to true. 569 sendOutputBuffers(); 570 // input buffer feeding may have been gated by pending output buffers 571 feedInputBufferIfAvailable(); 572 if (!c2Buffer) { 573 if (released) { 574 std::call_once(mRenderWarningFlag, [this] { 575 ALOGW("[%s] The app is calling releaseOutputBuffer() with " 576 "timestamp or render=true with non-video buffers. Apps should " 577 "call releaseOutputBuffer() with render=false for those.", 578 mName); 579 }); 580 } 581 return INVALID_OPERATION; 582 } 583 584 #if 0 585 const std::vector<std::shared_ptr<const C2Info>> infoParams = c2Buffer->info(); 586 ALOGV("[%s] queuing gfx buffer with %zu infos", mName, infoParams.size()); 587 for (const std::shared_ptr<const C2Info> &info : infoParams) { 588 AString res; 589 for (size_t ix = 0; ix + 3 < info->size(); ix += 4) { 590 if (ix) res.append(", "); 591 res.append(*((int32_t*)info.get() + (ix / 4))); 592 } 593 ALOGV(" [%s]", res.c_str()); 594 } 595 #endif 596 std::shared_ptr<const C2StreamRotationInfo::output> rotation = 597 std::static_pointer_cast<const C2StreamRotationInfo::output>( 598 c2Buffer->getInfo(C2StreamRotationInfo::output::PARAM_TYPE)); 599 bool flip = rotation && (rotation->flip & 1); 600 uint32_t quarters = ((rotation ? rotation->value : 0) / 90) & 3; 601 uint32_t transform = 0; 602 switch (quarters) { 603 case 0: // no rotation 604 transform = flip ? HAL_TRANSFORM_FLIP_H : 0; 605 break; 606 case 1: // 90 degrees counter-clockwise 607 transform = flip ? (HAL_TRANSFORM_FLIP_V | HAL_TRANSFORM_ROT_90) 608 : HAL_TRANSFORM_ROT_270; 609 break; 610 case 2: // 180 degrees 611 transform = flip ? HAL_TRANSFORM_FLIP_V : HAL_TRANSFORM_ROT_180; 612 break; 613 case 3: // 90 degrees clockwise 614 transform = flip ? (HAL_TRANSFORM_FLIP_H | HAL_TRANSFORM_ROT_90) 615 : HAL_TRANSFORM_ROT_90; 616 break; 617 } 618 619 std::shared_ptr<const C2StreamSurfaceScalingInfo::output> surfaceScaling = 620 std::static_pointer_cast<const C2StreamSurfaceScalingInfo::output>( 621 c2Buffer->getInfo(C2StreamSurfaceScalingInfo::output::PARAM_TYPE)); 622 uint32_t videoScalingMode = NATIVE_WINDOW_SCALING_MODE_SCALE_TO_WINDOW; 623 if (surfaceScaling) { 624 videoScalingMode = surfaceScaling->value; 625 } 626 627 // Use dataspace from format as it has the default aspects already applied 628 android_dataspace_t dataSpace = HAL_DATASPACE_UNKNOWN; // this is 0 629 (void)buffer->format()->findInt32("android._dataspace", (int32_t *)&dataSpace); 630 631 // HDR static info 632 std::shared_ptr<const C2StreamHdrStaticInfo::output> hdrStaticInfo = 633 std::static_pointer_cast<const C2StreamHdrStaticInfo::output>( 634 c2Buffer->getInfo(C2StreamHdrStaticInfo::output::PARAM_TYPE)); 635 636 // HDR10 plus info 637 std::shared_ptr<const C2StreamHdr10PlusInfo::output> hdr10PlusInfo = 638 std::static_pointer_cast<const C2StreamHdr10PlusInfo::output>( 639 c2Buffer->getInfo(C2StreamHdr10PlusInfo::output::PARAM_TYPE)); 640 641 { 642 Mutexed<OutputSurface>::Locked output(mOutputSurface); 643 if (output->surface == nullptr) { 644 ALOGI("[%s] cannot render buffer without surface", mName); 645 return OK; 646 } 647 } 648 649 std::vector<C2ConstGraphicBlock> blocks = c2Buffer->data().graphicBlocks(); 650 if (blocks.size() != 1u) { 651 ALOGD("[%s] expected 1 graphic block, but got %zu", mName, blocks.size()); 652 return UNKNOWN_ERROR; 653 } 654 const C2ConstGraphicBlock &block = blocks.front(); 655 656 // TODO: revisit this after C2Fence implementation. 657 android::IGraphicBufferProducer::QueueBufferInput qbi( 658 timestampNs, 659 false, // droppable 660 dataSpace, 661 Rect(blocks.front().crop().left, 662 blocks.front().crop().top, 663 blocks.front().crop().right(), 664 blocks.front().crop().bottom()), 665 videoScalingMode, 666 transform, 667 Fence::NO_FENCE, 0); 668 if (hdrStaticInfo || hdr10PlusInfo) { 669 HdrMetadata hdr; 670 if (hdrStaticInfo) { 671 struct android_smpte2086_metadata smpte2086_meta = { 672 .displayPrimaryRed = { 673 hdrStaticInfo->mastering.red.x, hdrStaticInfo->mastering.red.y 674 }, 675 .displayPrimaryGreen = { 676 hdrStaticInfo->mastering.green.x, hdrStaticInfo->mastering.green.y 677 }, 678 .displayPrimaryBlue = { 679 hdrStaticInfo->mastering.blue.x, hdrStaticInfo->mastering.blue.y 680 }, 681 .whitePoint = { 682 hdrStaticInfo->mastering.white.x, hdrStaticInfo->mastering.white.y 683 }, 684 .maxLuminance = hdrStaticInfo->mastering.maxLuminance, 685 .minLuminance = hdrStaticInfo->mastering.minLuminance, 686 }; 687 688 struct android_cta861_3_metadata cta861_meta = { 689 .maxContentLightLevel = hdrStaticInfo->maxCll, 690 .maxFrameAverageLightLevel = hdrStaticInfo->maxFall, 691 }; 692 693 hdr.validTypes = HdrMetadata::SMPTE2086 | HdrMetadata::CTA861_3; 694 hdr.smpte2086 = smpte2086_meta; 695 hdr.cta8613 = cta861_meta; 696 } 697 if (hdr10PlusInfo) { 698 hdr.validTypes |= HdrMetadata::HDR10PLUS; 699 hdr.hdr10plus.assign( 700 hdr10PlusInfo->m.value, 701 hdr10PlusInfo->m.value + hdr10PlusInfo->flexCount()); 702 } 703 qbi.setHdrMetadata(hdr); 704 } 705 // we don't have dirty regions 706 qbi.setSurfaceDamage(Region::INVALID_REGION); 707 android::IGraphicBufferProducer::QueueBufferOutput qbo; 708 status_t result = mComponent->queueToOutputSurface(block, qbi, &qbo); 709 if (result != OK) { 710 ALOGI("[%s] queueBuffer failed: %d", mName, result); 711 return result; 712 } 713 ALOGV("[%s] queue buffer successful", mName); 714 715 int64_t mediaTimeUs = 0; 716 (void)buffer->meta()->findInt64("timeUs", &mediaTimeUs); 717 mCCodecCallback->onOutputFramesRendered(mediaTimeUs, timestampNs); 718 719 return OK; 720 } 721 722 status_t CCodecBufferChannel::discardBuffer(const sp<MediaCodecBuffer> &buffer) { 723 ALOGV("[%s] discardBuffer: %p", mName, buffer.get()); 724 bool released = false; 725 { 726 Mutexed<Input>::Locked input(mInput); 727 if (input->buffers && input->buffers->releaseBuffer(buffer, nullptr, true)) { 728 released = true; 729 } 730 } 731 { 732 Mutexed<Output>::Locked output(mOutput); 733 if (output->buffers && output->buffers->releaseBuffer(buffer, nullptr)) { 734 released = true; 735 } 736 } 737 if (released) { 738 sendOutputBuffers(); 739 feedInputBufferIfAvailable(); 740 } else { 741 ALOGD("[%s] MediaCodec discarded an unknown buffer", mName); 742 } 743 return OK; 744 } 745 746 void CCodecBufferChannel::getInputBufferArray(Vector<sp<MediaCodecBuffer>> *array) { 747 array->clear(); 748 Mutexed<Input>::Locked input(mInput); 749 750 if (!input->buffers->isArrayMode()) { 751 input->buffers = input->buffers->toArrayMode(input->numSlots); 752 } 753 754 input->buffers->getArray(array); 755 } 756 757 void CCodecBufferChannel::getOutputBufferArray(Vector<sp<MediaCodecBuffer>> *array) { 758 array->clear(); 759 Mutexed<Output>::Locked output(mOutput); 760 761 if (!output->buffers->isArrayMode()) { 762 output->buffers = output->buffers->toArrayMode(output->numSlots); 763 } 764 765 output->buffers->getArray(array); 766 } 767 768 status_t CCodecBufferChannel::start( 769 const sp<AMessage> &inputFormat, const sp<AMessage> &outputFormat) { 770 C2StreamBufferTypeSetting::input iStreamFormat(0u); 771 C2StreamBufferTypeSetting::output oStreamFormat(0u); 772 C2PortReorderBufferDepthTuning::output reorderDepth; 773 C2PortReorderKeySetting::output reorderKey; 774 C2PortActualDelayTuning::input inputDelay(0); 775 C2PortActualDelayTuning::output outputDelay(0); 776 C2ActualPipelineDelayTuning pipelineDelay(0); 777 778 c2_status_t err = mComponent->query( 779 { 780 &iStreamFormat, 781 &oStreamFormat, 782 &reorderDepth, 783 &reorderKey, 784 &inputDelay, 785 &pipelineDelay, 786 &outputDelay, 787 }, 788 {}, 789 C2_DONT_BLOCK, 790 nullptr); 791 if (err == C2_BAD_INDEX) { 792 if (!iStreamFormat || !oStreamFormat) { 793 return UNKNOWN_ERROR; 794 } 795 } else if (err != C2_OK) { 796 return UNKNOWN_ERROR; 797 } 798 799 { 800 Mutexed<ReorderStash>::Locked reorder(mReorderStash); 801 reorder->clear(); 802 if (reorderDepth) { 803 reorder->setDepth(reorderDepth.value); 804 } 805 if (reorderKey) { 806 reorder->setKey(reorderKey.value); 807 } 808 } 809 810 uint32_t inputDelayValue = inputDelay ? inputDelay.value : 0; 811 uint32_t pipelineDelayValue = pipelineDelay ? pipelineDelay.value : 0; 812 uint32_t outputDelayValue = outputDelay ? outputDelay.value : 0; 813 814 size_t numInputSlots = inputDelayValue + pipelineDelayValue + kSmoothnessFactor; 815 size_t numOutputSlots = outputDelayValue + kSmoothnessFactor; 816 817 // TODO: get this from input format 818 bool secure = mComponent->getName().find(".secure") != std::string::npos; 819 820 std::shared_ptr<C2AllocatorStore> allocatorStore = GetCodec2PlatformAllocatorStore(); 821 int poolMask = property_get_int32( 822 "debug.stagefright.c2-poolmask", 823 1 << C2PlatformAllocatorStore::ION | 824 1 << C2PlatformAllocatorStore::BUFFERQUEUE); 825 826 if (inputFormat != nullptr) { 827 bool graphic = (iStreamFormat.value == C2BufferData::GRAPHIC); 828 std::shared_ptr<C2BlockPool> pool; 829 { 830 Mutexed<BlockPools>::Locked pools(mBlockPools); 831 832 // set default allocator ID. 833 pools->inputAllocatorId = (graphic) ? C2PlatformAllocatorStore::GRALLOC 834 : C2PlatformAllocatorStore::ION; 835 836 // query C2PortAllocatorsTuning::input from component. If an allocator ID is obtained 837 // from component, create the input block pool with given ID. Otherwise, use default IDs. 838 std::vector<std::unique_ptr<C2Param>> params; 839 err = mComponent->query({ }, 840 { C2PortAllocatorsTuning::input::PARAM_TYPE }, 841 C2_DONT_BLOCK, 842 ¶ms); 843 if ((err != C2_OK && err != C2_BAD_INDEX) || params.size() != 1) { 844 ALOGD("[%s] Query input allocators returned %zu params => %s (%u)", 845 mName, params.size(), asString(err), err); 846 } else if (err == C2_OK && params.size() == 1) { 847 C2PortAllocatorsTuning::input *inputAllocators = 848 C2PortAllocatorsTuning::input::From(params[0].get()); 849 if (inputAllocators && inputAllocators->flexCount() > 0) { 850 std::shared_ptr<C2Allocator> allocator; 851 // verify allocator IDs and resolve default allocator 852 allocatorStore->fetchAllocator(inputAllocators->m.values[0], &allocator); 853 if (allocator) { 854 pools->inputAllocatorId = allocator->getId(); 855 } else { 856 ALOGD("[%s] component requested invalid input allocator ID %u", 857 mName, inputAllocators->m.values[0]); 858 } 859 } 860 } 861 862 // TODO: use C2Component wrapper to associate this pool with ourselves 863 if ((poolMask >> pools->inputAllocatorId) & 1) { 864 err = CreateCodec2BlockPool(pools->inputAllocatorId, nullptr, &pool); 865 ALOGD("[%s] Created input block pool with allocatorID %u => poolID %llu - %s (%d)", 866 mName, pools->inputAllocatorId, 867 (unsigned long long)(pool ? pool->getLocalId() : 111000111), 868 asString(err), err); 869 } else { 870 err = C2_NOT_FOUND; 871 } 872 if (err != C2_OK) { 873 C2BlockPool::local_id_t inputPoolId = 874 graphic ? C2BlockPool::BASIC_GRAPHIC : C2BlockPool::BASIC_LINEAR; 875 err = GetCodec2BlockPool(inputPoolId, nullptr, &pool); 876 ALOGD("[%s] Using basic input block pool with poolID %llu => got %llu - %s (%d)", 877 mName, (unsigned long long)inputPoolId, 878 (unsigned long long)(pool ? pool->getLocalId() : 111000111), 879 asString(err), err); 880 if (err != C2_OK) { 881 return NO_MEMORY; 882 } 883 } 884 pools->inputPool = pool; 885 } 886 887 bool forceArrayMode = false; 888 Mutexed<Input>::Locked input(mInput); 889 input->inputDelay = inputDelayValue; 890 input->pipelineDelay = pipelineDelayValue; 891 input->numSlots = numInputSlots; 892 input->extraBuffers.flush(); 893 input->numExtraSlots = 0u; 894 if (graphic) { 895 if (mInputSurface) { 896 input->buffers.reset(new DummyInputBuffers(mName)); 897 } else if (mMetaMode == MODE_ANW) { 898 input->buffers.reset(new GraphicMetadataInputBuffers(mName)); 899 // This is to ensure buffers do not get released prematurely. 900 // TODO: handle this without going into array mode 901 forceArrayMode = true; 902 } else { 903 input->buffers.reset(new GraphicInputBuffers(numInputSlots, mName)); 904 } 905 } else { 906 if (hasCryptoOrDescrambler()) { 907 int32_t capacity = kLinearBufferSize; 908 (void)inputFormat->findInt32(KEY_MAX_INPUT_SIZE, &capacity); 909 if ((size_t)capacity > kMaxLinearBufferSize) { 910 ALOGD("client requested %d, capped to %zu", capacity, kMaxLinearBufferSize); 911 capacity = kMaxLinearBufferSize; 912 } 913 if (mDealer == nullptr) { 914 mDealer = new MemoryDealer( 915 align(capacity, MemoryDealer::getAllocationAlignment()) 916 * (numInputSlots + 1), 917 "EncryptedLinearInputBuffers"); 918 mDecryptDestination = mDealer->allocate((size_t)capacity); 919 } 920 if (mCrypto != nullptr && mHeapSeqNum < 0) { 921 mHeapSeqNum = mCrypto->setHeap(mDealer->getMemoryHeap()); 922 } else { 923 mHeapSeqNum = -1; 924 } 925 input->buffers.reset(new EncryptedLinearInputBuffers( 926 secure, mDealer, mCrypto, mHeapSeqNum, (size_t)capacity, 927 numInputSlots, mName)); 928 forceArrayMode = true; 929 } else { 930 input->buffers.reset(new LinearInputBuffers(mName)); 931 } 932 } 933 input->buffers->setFormat(inputFormat); 934 935 if (err == C2_OK) { 936 input->buffers->setPool(pool); 937 } else { 938 // TODO: error 939 } 940 941 if (forceArrayMode) { 942 input->buffers = input->buffers->toArrayMode(numInputSlots); 943 } 944 } 945 946 if (outputFormat != nullptr) { 947 sp<IGraphicBufferProducer> outputSurface; 948 uint32_t outputGeneration; 949 { 950 Mutexed<OutputSurface>::Locked output(mOutputSurface); 951 output->maxDequeueBuffers = numOutputSlots + reorderDepth.value + kRenderingDepth; 952 outputSurface = output->surface ? 953 output->surface->getIGraphicBufferProducer() : nullptr; 954 if (outputSurface) { 955 output->surface->setMaxDequeuedBufferCount(output->maxDequeueBuffers); 956 } 957 outputGeneration = output->generation; 958 } 959 960 bool graphic = (oStreamFormat.value == C2BufferData::GRAPHIC); 961 C2BlockPool::local_id_t outputPoolId_; 962 963 { 964 Mutexed<BlockPools>::Locked pools(mBlockPools); 965 966 // set default allocator ID. 967 pools->outputAllocatorId = (graphic) ? C2PlatformAllocatorStore::GRALLOC 968 : C2PlatformAllocatorStore::ION; 969 970 // query C2PortAllocatorsTuning::output from component, or use default allocator if 971 // unsuccessful. 972 std::vector<std::unique_ptr<C2Param>> params; 973 err = mComponent->query({ }, 974 { C2PortAllocatorsTuning::output::PARAM_TYPE }, 975 C2_DONT_BLOCK, 976 ¶ms); 977 if ((err != C2_OK && err != C2_BAD_INDEX) || params.size() != 1) { 978 ALOGD("[%s] Query output allocators returned %zu params => %s (%u)", 979 mName, params.size(), asString(err), err); 980 } else if (err == C2_OK && params.size() == 1) { 981 C2PortAllocatorsTuning::output *outputAllocators = 982 C2PortAllocatorsTuning::output::From(params[0].get()); 983 if (outputAllocators && outputAllocators->flexCount() > 0) { 984 std::shared_ptr<C2Allocator> allocator; 985 // verify allocator IDs and resolve default allocator 986 allocatorStore->fetchAllocator(outputAllocators->m.values[0], &allocator); 987 if (allocator) { 988 pools->outputAllocatorId = allocator->getId(); 989 } else { 990 ALOGD("[%s] component requested invalid output allocator ID %u", 991 mName, outputAllocators->m.values[0]); 992 } 993 } 994 } 995 996 // use bufferqueue if outputting to a surface. 997 // query C2PortSurfaceAllocatorTuning::output from component, or use default allocator 998 // if unsuccessful. 999 if (outputSurface) { 1000 params.clear(); 1001 err = mComponent->query({ }, 1002 { C2PortSurfaceAllocatorTuning::output::PARAM_TYPE }, 1003 C2_DONT_BLOCK, 1004 ¶ms); 1005 if ((err != C2_OK && err != C2_BAD_INDEX) || params.size() != 1) { 1006 ALOGD("[%s] Query output surface allocator returned %zu params => %s (%u)", 1007 mName, params.size(), asString(err), err); 1008 } else if (err == C2_OK && params.size() == 1) { 1009 C2PortSurfaceAllocatorTuning::output *surfaceAllocator = 1010 C2PortSurfaceAllocatorTuning::output::From(params[0].get()); 1011 if (surfaceAllocator) { 1012 std::shared_ptr<C2Allocator> allocator; 1013 // verify allocator IDs and resolve default allocator 1014 allocatorStore->fetchAllocator(surfaceAllocator->value, &allocator); 1015 if (allocator) { 1016 pools->outputAllocatorId = allocator->getId(); 1017 } else { 1018 ALOGD("[%s] component requested invalid surface output allocator ID %u", 1019 mName, surfaceAllocator->value); 1020 err = C2_BAD_VALUE; 1021 } 1022 } 1023 } 1024 if (pools->outputAllocatorId == C2PlatformAllocatorStore::GRALLOC 1025 && err != C2_OK 1026 && ((poolMask >> C2PlatformAllocatorStore::BUFFERQUEUE) & 1)) { 1027 pools->outputAllocatorId = C2PlatformAllocatorStore::BUFFERQUEUE; 1028 } 1029 } 1030 1031 if ((poolMask >> pools->outputAllocatorId) & 1) { 1032 err = mComponent->createBlockPool( 1033 pools->outputAllocatorId, &pools->outputPoolId, &pools->outputPoolIntf); 1034 ALOGI("[%s] Created output block pool with allocatorID %u => poolID %llu - %s", 1035 mName, pools->outputAllocatorId, 1036 (unsigned long long)pools->outputPoolId, 1037 asString(err)); 1038 } else { 1039 err = C2_NOT_FOUND; 1040 } 1041 if (err != C2_OK) { 1042 // use basic pool instead 1043 pools->outputPoolId = 1044 graphic ? C2BlockPool::BASIC_GRAPHIC : C2BlockPool::BASIC_LINEAR; 1045 } 1046 1047 // Configure output block pool ID as parameter C2PortBlockPoolsTuning::output to 1048 // component. 1049 std::unique_ptr<C2PortBlockPoolsTuning::output> poolIdsTuning = 1050 C2PortBlockPoolsTuning::output::AllocUnique({ pools->outputPoolId }); 1051 1052 std::vector<std::unique_ptr<C2SettingResult>> failures; 1053 err = mComponent->config({ poolIdsTuning.get() }, C2_MAY_BLOCK, &failures); 1054 ALOGD("[%s] Configured output block pool ids %llu => %s", 1055 mName, (unsigned long long)poolIdsTuning->m.values[0], asString(err)); 1056 outputPoolId_ = pools->outputPoolId; 1057 } 1058 1059 Mutexed<Output>::Locked output(mOutput); 1060 output->outputDelay = outputDelayValue; 1061 output->numSlots = numOutputSlots; 1062 if (graphic) { 1063 if (outputSurface) { 1064 output->buffers.reset(new GraphicOutputBuffers(mName)); 1065 } else { 1066 output->buffers.reset(new RawGraphicOutputBuffers(numOutputSlots, mName)); 1067 } 1068 } else { 1069 output->buffers.reset(new LinearOutputBuffers(mName)); 1070 } 1071 output->buffers->setFormat(outputFormat->dup()); 1072 1073 1074 // Try to set output surface to created block pool if given. 1075 if (outputSurface) { 1076 mComponent->setOutputSurface( 1077 outputPoolId_, 1078 outputSurface, 1079 outputGeneration); 1080 } 1081 1082 if (oStreamFormat.value == C2BufferData::LINEAR 1083 && mComponentName.find("c2.qti.") == std::string::npos) { 1084 // WORKAROUND: if we're using early CSD workaround we convert to 1085 // array mode, to appease apps assuming the output 1086 // buffers to be of the same size. 1087 output->buffers = output->buffers->toArrayMode(numOutputSlots); 1088 1089 int32_t channelCount; 1090 int32_t sampleRate; 1091 if (outputFormat->findInt32(KEY_CHANNEL_COUNT, &channelCount) 1092 && outputFormat->findInt32(KEY_SAMPLE_RATE, &sampleRate)) { 1093 int32_t delay = 0; 1094 int32_t padding = 0;; 1095 if (!outputFormat->findInt32("encoder-delay", &delay)) { 1096 delay = 0; 1097 } 1098 if (!outputFormat->findInt32("encoder-padding", &padding)) { 1099 padding = 0; 1100 } 1101 if (delay || padding) { 1102 // We need write access to the buffers, and we're already in 1103 // array mode. 1104 output->buffers->initSkipCutBuffer(delay, padding, sampleRate, channelCount); 1105 } 1106 } 1107 } 1108 } 1109 1110 // Set up pipeline control. This has to be done after mInputBuffers and 1111 // mOutputBuffers are initialized to make sure that lingering callbacks 1112 // about buffers from the previous generation do not interfere with the 1113 // newly initialized pipeline capacity. 1114 1115 { 1116 Mutexed<PipelineWatcher>::Locked watcher(mPipelineWatcher); 1117 watcher->inputDelay(inputDelayValue) 1118 .pipelineDelay(pipelineDelayValue) 1119 .outputDelay(outputDelayValue) 1120 .smoothnessFactor(kSmoothnessFactor); 1121 watcher->flush(); 1122 } 1123 1124 mInputMetEos = false; 1125 mSync.start(); 1126 return OK; 1127 } 1128 1129 status_t CCodecBufferChannel::requestInitialInputBuffers() { 1130 if (mInputSurface) { 1131 return OK; 1132 } 1133 1134 C2StreamBufferTypeSetting::output oStreamFormat(0u); 1135 c2_status_t err = mComponent->query({ &oStreamFormat }, {}, C2_DONT_BLOCK, nullptr); 1136 if (err != C2_OK) { 1137 return UNKNOWN_ERROR; 1138 } 1139 size_t numInputSlots = mInput.lock()->numSlots; 1140 std::vector<sp<MediaCodecBuffer>> toBeQueued; 1141 for (size_t i = 0; i < numInputSlots; ++i) { 1142 size_t index; 1143 sp<MediaCodecBuffer> buffer; 1144 { 1145 Mutexed<Input>::Locked input(mInput); 1146 if (!input->buffers->requestNewBuffer(&index, &buffer)) { 1147 if (i == 0) { 1148 ALOGW("[%s] start: cannot allocate memory at all", mName); 1149 return NO_MEMORY; 1150 } else { 1151 ALOGV("[%s] start: cannot allocate memory, only %zu buffers allocated", 1152 mName, i); 1153 } 1154 break; 1155 } 1156 } 1157 if (buffer) { 1158 Mutexed<std::list<sp<ABuffer>>>::Locked configs(mFlushedConfigs); 1159 ALOGV("[%s] input buffer %zu available", mName, index); 1160 bool post = true; 1161 if (!configs->empty()) { 1162 sp<ABuffer> config = configs->front(); 1163 configs->pop_front(); 1164 if (buffer->capacity() >= config->size()) { 1165 memcpy(buffer->base(), config->data(), config->size()); 1166 buffer->setRange(0, config->size()); 1167 buffer->meta()->clear(); 1168 buffer->meta()->setInt64("timeUs", 0); 1169 buffer->meta()->setInt32("csd", 1); 1170 post = false; 1171 } else { 1172 ALOGD("[%s] buffer capacity too small for the config (%zu < %zu)", 1173 mName, buffer->capacity(), config->size()); 1174 } 1175 } else if (oStreamFormat.value == C2BufferData::LINEAR && i == 0 1176 && mComponentName.find("c2.qti.") == std::string::npos) { 1177 // WORKAROUND: Some apps expect CSD available without queueing 1178 // any input. Queue an empty buffer to get the CSD. 1179 buffer->setRange(0, 0); 1180 buffer->meta()->clear(); 1181 buffer->meta()->setInt64("timeUs", 0); 1182 post = false; 1183 } 1184 if (post) { 1185 mCallback->onInputBufferAvailable(index, buffer); 1186 } else { 1187 toBeQueued.emplace_back(buffer); 1188 } 1189 } 1190 } 1191 for (const sp<MediaCodecBuffer> &buffer : toBeQueued) { 1192 if (queueInputBufferInternal(buffer) != OK) { 1193 ALOGV("[%s] Error while queueing initial buffers", mName); 1194 } 1195 } 1196 return OK; 1197 } 1198 1199 void CCodecBufferChannel::stop() { 1200 mSync.stop(); 1201 mFirstValidFrameIndex = mFrameIndex.load(std::memory_order_relaxed); 1202 if (mInputSurface != nullptr) { 1203 mInputSurface.reset(); 1204 } 1205 } 1206 1207 void CCodecBufferChannel::flush(const std::list<std::unique_ptr<C2Work>> &flushedWork) { 1208 ALOGV("[%s] flush", mName); 1209 { 1210 Mutexed<std::list<sp<ABuffer>>>::Locked configs(mFlushedConfigs); 1211 for (const std::unique_ptr<C2Work> &work : flushedWork) { 1212 if (!(work->input.flags & C2FrameData::FLAG_CODEC_CONFIG)) { 1213 continue; 1214 } 1215 if (work->input.buffers.empty() 1216 || work->input.buffers.front()->data().linearBlocks().empty()) { 1217 ALOGD("[%s] no linear codec config data found", mName); 1218 continue; 1219 } 1220 C2ReadView view = 1221 work->input.buffers.front()->data().linearBlocks().front().map().get(); 1222 if (view.error() != C2_OK) { 1223 ALOGD("[%s] failed to map flushed codec config data: %d", mName, view.error()); 1224 continue; 1225 } 1226 configs->push_back(ABuffer::CreateAsCopy(view.data(), view.capacity())); 1227 ALOGV("[%s] stashed flushed codec config data (size=%u)", mName, view.capacity()); 1228 } 1229 } 1230 { 1231 Mutexed<Input>::Locked input(mInput); 1232 input->buffers->flush(); 1233 input->extraBuffers.flush(); 1234 } 1235 { 1236 Mutexed<Output>::Locked output(mOutput); 1237 output->buffers->flush(flushedWork); 1238 } 1239 mReorderStash.lock()->flush(); 1240 mPipelineWatcher.lock()->flush(); 1241 } 1242 1243 void CCodecBufferChannel::onWorkDone( 1244 std::unique_ptr<C2Work> work, const sp<AMessage> &outputFormat, 1245 const C2StreamInitDataInfo::output *initData) { 1246 if (handleWork(std::move(work), outputFormat, initData)) { 1247 feedInputBufferIfAvailable(); 1248 } 1249 } 1250 1251 void CCodecBufferChannel::onInputBufferDone( 1252 uint64_t frameIndex, size_t arrayIndex) { 1253 if (mInputSurface) { 1254 return; 1255 } 1256 std::shared_ptr<C2Buffer> buffer = 1257 mPipelineWatcher.lock()->onInputBufferReleased(frameIndex, arrayIndex); 1258 bool newInputSlotAvailable; 1259 { 1260 Mutexed<Input>::Locked input(mInput); 1261 newInputSlotAvailable = input->buffers->expireComponentBuffer(buffer); 1262 if (!newInputSlotAvailable) { 1263 (void)input->extraBuffers.expireComponentBuffer(buffer); 1264 } 1265 } 1266 if (newInputSlotAvailable) { 1267 feedInputBufferIfAvailable(); 1268 } 1269 } 1270 1271 bool CCodecBufferChannel::handleWork( 1272 std::unique_ptr<C2Work> work, 1273 const sp<AMessage> &outputFormat, 1274 const C2StreamInitDataInfo::output *initData) { 1275 if ((work->input.ordinal.frameIndex - mFirstValidFrameIndex.load()).peek() < 0) { 1276 // Discard frames from previous generation. 1277 ALOGD("[%s] Discard frames from previous generation.", mName); 1278 return false; 1279 } 1280 1281 if (mInputSurface == nullptr && (work->worklets.size() != 1u 1282 || !work->worklets.front() 1283 || !(work->worklets.front()->output.flags & C2FrameData::FLAG_INCOMPLETE))) { 1284 mPipelineWatcher.lock()->onWorkDone(work->input.ordinal.frameIndex.peeku()); 1285 } 1286 1287 if (work->result == C2_NOT_FOUND) { 1288 ALOGD("[%s] flushed work; ignored.", mName); 1289 return true; 1290 } 1291 1292 if (work->result != C2_OK) { 1293 ALOGD("[%s] work failed to complete: %d", mName, work->result); 1294 mCCodecCallback->onError(work->result, ACTION_CODE_FATAL); 1295 return false; 1296 } 1297 1298 // NOTE: MediaCodec usage supposedly have only one worklet 1299 if (work->worklets.size() != 1u) { 1300 ALOGI("[%s] onWorkDone: incorrect number of worklets: %zu", 1301 mName, work->worklets.size()); 1302 mCCodecCallback->onError(UNKNOWN_ERROR, ACTION_CODE_FATAL); 1303 return false; 1304 } 1305 1306 const std::unique_ptr<C2Worklet> &worklet = work->worklets.front(); 1307 1308 std::shared_ptr<C2Buffer> buffer; 1309 // NOTE: MediaCodec usage supposedly have only one output stream. 1310 if (worklet->output.buffers.size() > 1u) { 1311 ALOGI("[%s] onWorkDone: incorrect number of output buffers: %zu", 1312 mName, worklet->output.buffers.size()); 1313 mCCodecCallback->onError(UNKNOWN_ERROR, ACTION_CODE_FATAL); 1314 return false; 1315 } else if (worklet->output.buffers.size() == 1u) { 1316 buffer = worklet->output.buffers[0]; 1317 if (!buffer) { 1318 ALOGD("[%s] onWorkDone: nullptr found in buffers; ignored.", mName); 1319 } 1320 } 1321 1322 std::optional<uint32_t> newInputDelay, newPipelineDelay; 1323 while (!worklet->output.configUpdate.empty()) { 1324 std::unique_ptr<C2Param> param; 1325 worklet->output.configUpdate.back().swap(param); 1326 worklet->output.configUpdate.pop_back(); 1327 switch (param->coreIndex().coreIndex()) { 1328 case C2PortReorderBufferDepthTuning::CORE_INDEX: { 1329 C2PortReorderBufferDepthTuning::output reorderDepth; 1330 if (reorderDepth.updateFrom(*param)) { 1331 mReorderStash.lock()->setDepth(reorderDepth.value); 1332 ALOGV("[%s] onWorkDone: updated reorder depth to %u", 1333 mName, reorderDepth.value); 1334 size_t numOutputSlots = mOutput.lock()->numSlots; 1335 Mutexed<OutputSurface>::Locked output(mOutputSurface); 1336 output->maxDequeueBuffers = 1337 numOutputSlots + reorderDepth.value + kRenderingDepth; 1338 if (output->surface) { 1339 output->surface->setMaxDequeuedBufferCount(output->maxDequeueBuffers); 1340 } 1341 } else { 1342 ALOGD("[%s] onWorkDone: failed to read reorder depth", mName); 1343 } 1344 break; 1345 } 1346 case C2PortReorderKeySetting::CORE_INDEX: { 1347 C2PortReorderKeySetting::output reorderKey; 1348 if (reorderKey.updateFrom(*param)) { 1349 mReorderStash.lock()->setKey(reorderKey.value); 1350 ALOGV("[%s] onWorkDone: updated reorder key to %u", 1351 mName, reorderKey.value); 1352 } else { 1353 ALOGD("[%s] onWorkDone: failed to read reorder key", mName); 1354 } 1355 break; 1356 } 1357 case C2PortActualDelayTuning::CORE_INDEX: { 1358 if (param->isGlobal()) { 1359 C2ActualPipelineDelayTuning pipelineDelay; 1360 if (pipelineDelay.updateFrom(*param)) { 1361 ALOGV("[%s] onWorkDone: updating pipeline delay %u", 1362 mName, pipelineDelay.value); 1363 newPipelineDelay = pipelineDelay.value; 1364 (void)mPipelineWatcher.lock()->pipelineDelay(pipelineDelay.value); 1365 } 1366 } 1367 if (param->forInput()) { 1368 C2PortActualDelayTuning::input inputDelay; 1369 if (inputDelay.updateFrom(*param)) { 1370 ALOGV("[%s] onWorkDone: updating input delay %u", 1371 mName, inputDelay.value); 1372 newInputDelay = inputDelay.value; 1373 (void)mPipelineWatcher.lock()->inputDelay(inputDelay.value); 1374 } 1375 } 1376 if (param->forOutput()) { 1377 C2PortActualDelayTuning::output outputDelay; 1378 if (outputDelay.updateFrom(*param)) { 1379 ALOGV("[%s] onWorkDone: updating output delay %u", 1380 mName, outputDelay.value); 1381 (void)mPipelineWatcher.lock()->outputDelay(outputDelay.value); 1382 1383 bool outputBuffersChanged = false; 1384 size_t numOutputSlots = 0; 1385 { 1386 Mutexed<Output>::Locked output(mOutput); 1387 output->outputDelay = outputDelay.value; 1388 numOutputSlots = outputDelay.value + kSmoothnessFactor; 1389 if (output->numSlots < numOutputSlots) { 1390 output->numSlots = numOutputSlots; 1391 if (output->buffers->isArrayMode()) { 1392 OutputBuffersArray *array = 1393 (OutputBuffersArray *)output->buffers.get(); 1394 ALOGV("[%s] onWorkDone: growing output buffer array to %zu", 1395 mName, numOutputSlots); 1396 array->grow(numOutputSlots); 1397 outputBuffersChanged = true; 1398 } 1399 } 1400 numOutputSlots = output->numSlots; 1401 } 1402 1403 if (outputBuffersChanged) { 1404 mCCodecCallback->onOutputBuffersChanged(); 1405 } 1406 1407 uint32_t depth = mReorderStash.lock()->depth(); 1408 Mutexed<OutputSurface>::Locked output(mOutputSurface); 1409 output->maxDequeueBuffers = numOutputSlots + depth + kRenderingDepth; 1410 if (output->surface) { 1411 output->surface->setMaxDequeuedBufferCount(output->maxDequeueBuffers); 1412 } 1413 } 1414 } 1415 break; 1416 } 1417 default: 1418 ALOGV("[%s] onWorkDone: unrecognized config update (%08X)", 1419 mName, param->index()); 1420 break; 1421 } 1422 } 1423 if (newInputDelay || newPipelineDelay) { 1424 Mutexed<Input>::Locked input(mInput); 1425 size_t newNumSlots = 1426 newInputDelay.value_or(input->inputDelay) + 1427 newPipelineDelay.value_or(input->pipelineDelay) + 1428 kSmoothnessFactor; 1429 if (input->buffers->isArrayMode()) { 1430 if (input->numSlots >= newNumSlots) { 1431 input->numExtraSlots = 0; 1432 } else { 1433 input->numExtraSlots = newNumSlots - input->numSlots; 1434 } 1435 ALOGV("[%s] onWorkDone: updated number of extra slots to %zu (input array mode)", 1436 mName, input->numExtraSlots); 1437 } else { 1438 input->numSlots = newNumSlots; 1439 } 1440 } 1441 1442 if (outputFormat != nullptr) { 1443 Mutexed<Output>::Locked output(mOutput); 1444 ALOGD("[%s] onWorkDone: output format changed to %s", 1445 mName, outputFormat->debugString().c_str()); 1446 output->buffers->setFormat(outputFormat); 1447 1448 AString mediaType; 1449 if (outputFormat->findString(KEY_MIME, &mediaType) 1450 && mediaType == MIMETYPE_AUDIO_RAW) { 1451 int32_t channelCount; 1452 int32_t sampleRate; 1453 if (outputFormat->findInt32(KEY_CHANNEL_COUNT, &channelCount) 1454 && outputFormat->findInt32(KEY_SAMPLE_RATE, &sampleRate)) { 1455 output->buffers->updateSkipCutBuffer(sampleRate, channelCount); 1456 } 1457 } 1458 } 1459 1460 int32_t flags = 0; 1461 if (worklet->output.flags & C2FrameData::FLAG_END_OF_STREAM) { 1462 flags |= MediaCodec::BUFFER_FLAG_EOS; 1463 ALOGV("[%s] onWorkDone: output EOS", mName); 1464 } 1465 1466 sp<MediaCodecBuffer> outBuffer; 1467 size_t index; 1468 1469 // WORKAROUND: adjust output timestamp based on client input timestamp and codec 1470 // input timestamp. Codec output timestamp (in the timestamp field) shall correspond to 1471 // the codec input timestamp, but client output timestamp should (reported in timeUs) 1472 // shall correspond to the client input timesamp (in customOrdinal). By using the 1473 // delta between the two, this allows for some timestamp deviation - e.g. if one input 1474 // produces multiple output. 1475 c2_cntr64_t timestamp = 1476 worklet->output.ordinal.timestamp + work->input.ordinal.customOrdinal 1477 - work->input.ordinal.timestamp; 1478 if (mInputSurface != nullptr) { 1479 // When using input surface we need to restore the original input timestamp. 1480 timestamp = work->input.ordinal.customOrdinal; 1481 } 1482 ALOGV("[%s] onWorkDone: input %lld, codec %lld => output %lld => %lld", 1483 mName, 1484 work->input.ordinal.customOrdinal.peekll(), 1485 work->input.ordinal.timestamp.peekll(), 1486 worklet->output.ordinal.timestamp.peekll(), 1487 timestamp.peekll()); 1488 1489 if (initData != nullptr) { 1490 Mutexed<Output>::Locked output(mOutput); 1491 if (output->buffers->registerCsd(initData, &index, &outBuffer) == OK) { 1492 outBuffer->meta()->setInt64("timeUs", timestamp.peek()); 1493 outBuffer->meta()->setInt32("flags", MediaCodec::BUFFER_FLAG_CODECCONFIG); 1494 ALOGV("[%s] onWorkDone: csd index = %zu [%p]", mName, index, outBuffer.get()); 1495 1496 output.unlock(); 1497 mCallback->onOutputBufferAvailable(index, outBuffer); 1498 } else { 1499 ALOGD("[%s] onWorkDone: unable to register csd", mName); 1500 output.unlock(); 1501 mCCodecCallback->onError(UNKNOWN_ERROR, ACTION_CODE_FATAL); 1502 return false; 1503 } 1504 } 1505 1506 if (!buffer && !flags) { 1507 ALOGV("[%s] onWorkDone: Not reporting output buffer (%lld)", 1508 mName, work->input.ordinal.frameIndex.peekull()); 1509 return true; 1510 } 1511 1512 if (buffer) { 1513 for (const std::shared_ptr<const C2Info> &info : buffer->info()) { 1514 // TODO: properly translate these to metadata 1515 switch (info->coreIndex().coreIndex()) { 1516 case C2StreamPictureTypeMaskInfo::CORE_INDEX: 1517 if (((C2StreamPictureTypeMaskInfo *)info.get())->value & C2Config::SYNC_FRAME) { 1518 flags |= MediaCodec::BUFFER_FLAG_SYNCFRAME; 1519 } 1520 break; 1521 default: 1522 break; 1523 } 1524 } 1525 } 1526 1527 { 1528 Mutexed<ReorderStash>::Locked reorder(mReorderStash); 1529 reorder->emplace(buffer, timestamp.peek(), flags, worklet->output.ordinal); 1530 if (flags & MediaCodec::BUFFER_FLAG_EOS) { 1531 // Flush reorder stash 1532 reorder->setDepth(0); 1533 } 1534 } 1535 sendOutputBuffers(); 1536 return true; 1537 } 1538 1539 void CCodecBufferChannel::sendOutputBuffers() { 1540 ReorderStash::Entry entry; 1541 sp<MediaCodecBuffer> outBuffer; 1542 size_t index; 1543 1544 while (true) { 1545 Mutexed<ReorderStash>::Locked reorder(mReorderStash); 1546 if (!reorder->hasPending()) { 1547 break; 1548 } 1549 if (!reorder->pop(&entry)) { 1550 break; 1551 } 1552 1553 Mutexed<Output>::Locked output(mOutput); 1554 status_t err = output->buffers->registerBuffer(entry.buffer, &index, &outBuffer); 1555 if (err != OK) { 1556 bool outputBuffersChanged = false; 1557 if (err != WOULD_BLOCK) { 1558 if (!output->buffers->isArrayMode()) { 1559 output->buffers = output->buffers->toArrayMode(output->numSlots); 1560 } 1561 OutputBuffersArray *array = (OutputBuffersArray *)output->buffers.get(); 1562 array->realloc(entry.buffer); 1563 outputBuffersChanged = true; 1564 } 1565 ALOGV("[%s] sendOutputBuffers: unable to register output buffer", mName); 1566 reorder->defer(entry); 1567 1568 output.unlock(); 1569 reorder.unlock(); 1570 1571 if (outputBuffersChanged) { 1572 mCCodecCallback->onOutputBuffersChanged(); 1573 } 1574 return; 1575 } 1576 output.unlock(); 1577 reorder.unlock(); 1578 1579 outBuffer->meta()->setInt64("timeUs", entry.timestamp); 1580 outBuffer->meta()->setInt32("flags", entry.flags); 1581 ALOGV("[%s] sendOutputBuffers: out buffer index = %zu [%p] => %p + %zu (%lld)", 1582 mName, index, outBuffer.get(), outBuffer->data(), outBuffer->size(), 1583 (long long)entry.timestamp); 1584 mCallback->onOutputBufferAvailable(index, outBuffer); 1585 } 1586 } 1587 1588 status_t CCodecBufferChannel::setSurface(const sp<Surface> &newSurface) { 1589 static std::atomic_uint32_t surfaceGeneration{0}; 1590 uint32_t generation = (getpid() << 10) | 1591 ((surfaceGeneration.fetch_add(1, std::memory_order_relaxed) + 1) 1592 & ((1 << 10) - 1)); 1593 1594 sp<IGraphicBufferProducer> producer; 1595 if (newSurface) { 1596 newSurface->setScalingMode(NATIVE_WINDOW_SCALING_MODE_SCALE_TO_WINDOW); 1597 newSurface->setDequeueTimeout(kDequeueTimeoutNs); 1598 newSurface->setMaxDequeuedBufferCount(mOutputSurface.lock()->maxDequeueBuffers); 1599 producer = newSurface->getIGraphicBufferProducer(); 1600 producer->setGenerationNumber(generation); 1601 } else { 1602 ALOGE("[%s] setting output surface to null", mName); 1603 return INVALID_OPERATION; 1604 } 1605 1606 std::shared_ptr<Codec2Client::Configurable> outputPoolIntf; 1607 C2BlockPool::local_id_t outputPoolId; 1608 { 1609 Mutexed<BlockPools>::Locked pools(mBlockPools); 1610 outputPoolId = pools->outputPoolId; 1611 outputPoolIntf = pools->outputPoolIntf; 1612 } 1613 1614 if (outputPoolIntf) { 1615 if (mComponent->setOutputSurface( 1616 outputPoolId, 1617 producer, 1618 generation) != C2_OK) { 1619 ALOGI("[%s] setSurface: component setOutputSurface failed", mName); 1620 return INVALID_OPERATION; 1621 } 1622 } 1623 1624 { 1625 Mutexed<OutputSurface>::Locked output(mOutputSurface); 1626 output->surface = newSurface; 1627 output->generation = generation; 1628 } 1629 1630 return OK; 1631 } 1632 1633 PipelineWatcher::Clock::duration CCodecBufferChannel::elapsed() { 1634 // When client pushed EOS, we want all the work to be done quickly. 1635 // Otherwise, component may have stalled work due to input starvation up to 1636 // the sum of the delay in the pipeline. 1637 size_t n = 0; 1638 if (!mInputMetEos) { 1639 size_t outputDelay = mOutput.lock()->outputDelay; 1640 Mutexed<Input>::Locked input(mInput); 1641 n = input->inputDelay + input->pipelineDelay + outputDelay; 1642 } 1643 return mPipelineWatcher.lock()->elapsed(PipelineWatcher::Clock::now(), n); 1644 } 1645 1646 void CCodecBufferChannel::setMetaMode(MetaMode mode) { 1647 mMetaMode = mode; 1648 } 1649 1650 status_t toStatusT(c2_status_t c2s, c2_operation_t c2op) { 1651 // C2_OK is always translated to OK. 1652 if (c2s == C2_OK) { 1653 return OK; 1654 } 1655 1656 // Operation-dependent translation 1657 // TODO: Add as necessary 1658 switch (c2op) { 1659 case C2_OPERATION_Component_start: 1660 switch (c2s) { 1661 case C2_NO_MEMORY: 1662 return NO_MEMORY; 1663 default: 1664 return UNKNOWN_ERROR; 1665 } 1666 default: 1667 break; 1668 } 1669 1670 // Backup operation-agnostic translation 1671 switch (c2s) { 1672 case C2_BAD_INDEX: 1673 return BAD_INDEX; 1674 case C2_BAD_VALUE: 1675 return BAD_VALUE; 1676 case C2_BLOCKING: 1677 return WOULD_BLOCK; 1678 case C2_DUPLICATE: 1679 return ALREADY_EXISTS; 1680 case C2_NO_INIT: 1681 return NO_INIT; 1682 case C2_NO_MEMORY: 1683 return NO_MEMORY; 1684 case C2_NOT_FOUND: 1685 return NAME_NOT_FOUND; 1686 case C2_TIMED_OUT: 1687 return TIMED_OUT; 1688 case C2_BAD_STATE: 1689 case C2_CANCELED: 1690 case C2_CANNOT_DO: 1691 case C2_CORRUPTED: 1692 case C2_OMITTED: 1693 case C2_REFUSED: 1694 return UNKNOWN_ERROR; 1695 default: 1696 return -static_cast<status_t>(c2s); 1697 } 1698 } 1699 1700 } // namespace android 1701
