1 /* 2 * Copyright 2014 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17 //#define LOG_NDEBUG 0 18 #define LOG_TAG "NuPlayerDecoder" 19 #include <utils/Log.h> 20 #include <inttypes.h> 21 22 #include <algorithm> 23 24 #include "NuPlayerCCDecoder.h" 25 #include "NuPlayerDecoder.h" 26 #include "NuPlayerDrm.h" 27 #include "NuPlayerRenderer.h" 28 #include "NuPlayerSource.h" 29 30 #include <cutils/properties.h> 31 #include <media/ICrypto.h> 32 #include <media/MediaBufferHolder.h> 33 #include <media/MediaCodecBuffer.h> 34 #include <media/stagefright/foundation/ABuffer.h> 35 #include <media/stagefright/foundation/ADebug.h> 36 #include <media/stagefright/foundation/AMessage.h> 37 #include <media/stagefright/foundation/avc_utils.h> 38 #include <media/stagefright/MediaBuffer.h> 39 #include <media/stagefright/MediaCodec.h> 40 #include <media/stagefright/MediaDefs.h> 41 #include <media/stagefright/MediaErrors.h> 42 #include <media/stagefright/SurfaceUtils.h> 43 #include <gui/Surface.h> 44 45 #include "ATSParser.h" 46 47 namespace android { 48 49 static float kDisplayRefreshingRate = 60.f; // TODO: get this from the display 50 51 // The default total video frame rate of a stream when that info is not available from 52 // the source. 53 static float kDefaultVideoFrameRateTotal = 30.f; 54 55 static inline bool getAudioDeepBufferSetting() { 56 return property_get_bool("media.stagefright.audio.deep", false /* default_value */); 57 } 58 59 NuPlayer::Decoder::Decoder( 60 const sp<AMessage> ¬ify, 61 const sp<Source> &source, 62 pid_t pid, 63 uid_t uid, 64 const sp<Renderer> &renderer, 65 const sp<Surface> &surface, 66 const sp<CCDecoder> &ccDecoder) 67 : DecoderBase(notify), 68 mSurface(surface), 69 mSource(source), 70 mRenderer(renderer), 71 mCCDecoder(ccDecoder), 72 mPid(pid), 73 mUid(uid), 74 mSkipRenderingUntilMediaTimeUs(-1LL), 75 mNumFramesTotal(0LL), 76 mNumInputFramesDropped(0LL), 77 mNumOutputFramesDropped(0LL), 78 mVideoWidth(0), 79 mVideoHeight(0), 80 mIsAudio(true), 81 mIsVideoAVC(false), 82 mIsSecure(false), 83 mIsEncrypted(false), 84 mIsEncryptedObservedEarlier(false), 85 mFormatChangePending(false), 86 mTimeChangePending(false), 87 mFrameRateTotal(kDefaultVideoFrameRateTotal), 88 mPlaybackSpeed(1.0f), 89 mNumVideoTemporalLayerTotal(1), // decode all layers 90 mNumVideoTemporalLayerAllowed(1), 91 mCurrentMaxVideoTemporalLayerId(0), 92 mResumePending(false), 93 mComponentName("decoder") { 94 mCodecLooper = new ALooper; 95 mCodecLooper->setName("NPDecoder-CL"); 96 mCodecLooper->start(false, false, ANDROID_PRIORITY_AUDIO); 97 mVideoTemporalLayerAggregateFps[0] = mFrameRateTotal; 98 } 99 100 NuPlayer::Decoder::~Decoder() { 101 // Need to stop looper first since mCodec could be accessed on the mDecoderLooper. 102 stopLooper(); 103 if (mCodec != NULL) { 104 mCodec->release(); 105 } 106 releaseAndResetMediaBuffers(); 107 } 108 109 sp<AMessage> NuPlayer::Decoder::getStats() const { 110 111 mStats->setInt64("frames-total", mNumFramesTotal); 112 mStats->setInt64("frames-dropped-input", mNumInputFramesDropped); 113 mStats->setInt64("frames-dropped-output", mNumOutputFramesDropped); 114 mStats->setFloat("frame-rate-total", mFrameRateTotal); 115 116 // i'm mutexed right now. 117 // make our own copy, so we aren't victim to any later changes. 118 sp<AMessage> copiedStats = mStats->dup(); 119 return copiedStats; 120 } 121 122 status_t NuPlayer::Decoder::setVideoSurface(const sp<Surface> &surface) { 123 if (surface == NULL || ADebug::isExperimentEnabled("legacy-setsurface")) { 124 return BAD_VALUE; 125 } 126 127 sp<AMessage> msg = new AMessage(kWhatSetVideoSurface, this); 128 129 msg->setObject("surface", surface); 130 sp<AMessage> response; 131 status_t err = msg->postAndAwaitResponse(&response); 132 if (err == OK && response != NULL) { 133 CHECK(response->findInt32("err", &err)); 134 } 135 return err; 136 } 137 138 void NuPlayer::Decoder::onMessageReceived(const sp<AMessage> &msg) { 139 ALOGV("[%s] onMessage: %s", mComponentName.c_str(), msg->debugString().c_str()); 140 141 switch (msg->what()) { 142 case kWhatCodecNotify: 143 { 144 int32_t cbID; 145 CHECK(msg->findInt32("callbackID", &cbID)); 146 147 ALOGV("[%s] kWhatCodecNotify: cbID = %d, paused = %d", 148 mIsAudio ? "audio" : "video", cbID, mPaused); 149 150 if (mPaused) { 151 break; 152 } 153 154 switch (cbID) { 155 case MediaCodec::CB_INPUT_AVAILABLE: 156 { 157 int32_t index; 158 CHECK(msg->findInt32("index", &index)); 159 160 handleAnInputBuffer(index); 161 break; 162 } 163 164 case MediaCodec::CB_OUTPUT_AVAILABLE: 165 { 166 int32_t index; 167 size_t offset; 168 size_t size; 169 int64_t timeUs; 170 int32_t flags; 171 172 CHECK(msg->findInt32("index", &index)); 173 CHECK(msg->findSize("offset", &offset)); 174 CHECK(msg->findSize("size", &size)); 175 CHECK(msg->findInt64("timeUs", &timeUs)); 176 CHECK(msg->findInt32("flags", &flags)); 177 178 handleAnOutputBuffer(index, offset, size, timeUs, flags); 179 break; 180 } 181 182 case MediaCodec::CB_OUTPUT_FORMAT_CHANGED: 183 { 184 sp<AMessage> format; 185 CHECK(msg->findMessage("format", &format)); 186 187 handleOutputFormatChange(format); 188 break; 189 } 190 191 case MediaCodec::CB_ERROR: 192 { 193 status_t err; 194 CHECK(msg->findInt32("err", &err)); 195 ALOGE("Decoder (%s) reported error : 0x%x", 196 mIsAudio ? "audio" : "video", err); 197 198 handleError(err); 199 break; 200 } 201 202 default: 203 { 204 TRESPASS(); 205 break; 206 } 207 } 208 209 break; 210 } 211 212 case kWhatRenderBuffer: 213 { 214 if (!isStaleReply(msg)) { 215 onRenderBuffer(msg); 216 } 217 break; 218 } 219 220 case kWhatAudioOutputFormatChanged: 221 { 222 if (!isStaleReply(msg)) { 223 status_t err; 224 if (msg->findInt32("err", &err) && err != OK) { 225 ALOGE("Renderer reported 0x%x when changing audio output format", err); 226 handleError(err); 227 } 228 } 229 break; 230 } 231 232 case kWhatSetVideoSurface: 233 { 234 sp<AReplyToken> replyID; 235 CHECK(msg->senderAwaitsResponse(&replyID)); 236 237 sp<RefBase> obj; 238 CHECK(msg->findObject("surface", &obj)); 239 sp<Surface> surface = static_cast<Surface *>(obj.get()); // non-null 240 int32_t err = INVALID_OPERATION; 241 // NOTE: in practice mSurface is always non-null, but checking here for completeness 242 if (mCodec != NULL && mSurface != NULL) { 243 // TODO: once AwesomePlayer is removed, remove this automatic connecting 244 // to the surface by MediaPlayerService. 245 // 246 // at this point MediaPlayerService::client has already connected to the 247 // surface, which MediaCodec does not expect 248 err = nativeWindowDisconnect(surface.get(), "kWhatSetVideoSurface(surface)"); 249 if (err == OK) { 250 err = mCodec->setSurface(surface); 251 ALOGI_IF(err, "codec setSurface returned: %d", err); 252 if (err == OK) { 253 // reconnect to the old surface as MPS::Client will expect to 254 // be able to disconnect from it. 255 (void)nativeWindowConnect(mSurface.get(), "kWhatSetVideoSurface(mSurface)"); 256 mSurface = surface; 257 } 258 } 259 if (err != OK) { 260 // reconnect to the new surface on error as MPS::Client will expect to 261 // be able to disconnect from it. 262 (void)nativeWindowConnect(surface.get(), "kWhatSetVideoSurface(err)"); 263 } 264 } 265 266 sp<AMessage> response = new AMessage; 267 response->setInt32("err", err); 268 response->postReply(replyID); 269 break; 270 } 271 272 case kWhatDrmReleaseCrypto: 273 { 274 ALOGV("kWhatDrmReleaseCrypto"); 275 onReleaseCrypto(msg); 276 break; 277 } 278 279 default: 280 DecoderBase::onMessageReceived(msg); 281 break; 282 } 283 } 284 285 void NuPlayer::Decoder::onConfigure(const sp<AMessage> &format) { 286 CHECK(mCodec == NULL); 287 288 mFormatChangePending = false; 289 mTimeChangePending = false; 290 291 ++mBufferGeneration; 292 293 AString mime; 294 CHECK(format->findString("mime", &mime)); 295 296 mIsAudio = !strncasecmp("audio/", mime.c_str(), 6); 297 mIsVideoAVC = !strcasecmp(MEDIA_MIMETYPE_VIDEO_AVC, mime.c_str()); 298 299 mComponentName = mime; 300 mComponentName.append(" decoder"); 301 ALOGV("[%s] onConfigure (surface=%p)", mComponentName.c_str(), mSurface.get()); 302 303 mCodec = MediaCodec::CreateByType( 304 mCodecLooper, mime.c_str(), false /* encoder */, NULL /* err */, mPid, mUid); 305 int32_t secure = 0; 306 if (format->findInt32("secure", &secure) && secure != 0) { 307 if (mCodec != NULL) { 308 mCodec->getName(&mComponentName); 309 mComponentName.append(".secure"); 310 mCodec->release(); 311 ALOGI("[%s] creating", mComponentName.c_str()); 312 mCodec = MediaCodec::CreateByComponentName( 313 mCodecLooper, mComponentName.c_str(), NULL /* err */, mPid, mUid); 314 } 315 } 316 if (mCodec == NULL) { 317 ALOGE("Failed to create %s%s decoder", 318 (secure ? "secure " : ""), mime.c_str()); 319 handleError(UNKNOWN_ERROR); 320 return; 321 } 322 mIsSecure = secure; 323 324 mCodec->getName(&mComponentName); 325 326 status_t err; 327 if (mSurface != NULL) { 328 // disconnect from surface as MediaCodec will reconnect 329 err = nativeWindowDisconnect(mSurface.get(), "onConfigure"); 330 // We treat this as a warning, as this is a preparatory step. 331 // Codec will try to connect to the surface, which is where 332 // any error signaling will occur. 333 ALOGW_IF(err != OK, "failed to disconnect from surface: %d", err); 334 } 335 336 // Modular DRM 337 void *pCrypto; 338 if (!format->findPointer("crypto", &pCrypto)) { 339 pCrypto = NULL; 340 } 341 sp<ICrypto> crypto = (ICrypto*)pCrypto; 342 // non-encrypted source won't have a crypto 343 mIsEncrypted = (crypto != NULL); 344 // configure is called once; still using OR in case the behavior changes. 345 mIsEncryptedObservedEarlier = mIsEncryptedObservedEarlier || mIsEncrypted; 346 ALOGV("onConfigure mCrypto: %p (%d) mIsSecure: %d", 347 crypto.get(), (crypto != NULL ? crypto->getStrongCount() : 0), mIsSecure); 348 349 err = mCodec->configure( 350 format, mSurface, crypto, 0 /* flags */); 351 352 if (err != OK) { 353 ALOGE("Failed to configure [%s] decoder (err=%d)", mComponentName.c_str(), err); 354 mCodec->release(); 355 mCodec.clear(); 356 handleError(err); 357 return; 358 } 359 rememberCodecSpecificData(format); 360 361 // the following should work in configured state 362 CHECK_EQ((status_t)OK, mCodec->getOutputFormat(&mOutputFormat)); 363 CHECK_EQ((status_t)OK, mCodec->getInputFormat(&mInputFormat)); 364 365 mStats->setString("mime", mime.c_str()); 366 mStats->setString("component-name", mComponentName.c_str()); 367 368 if (!mIsAudio) { 369 int32_t width, height; 370 if (mOutputFormat->findInt32("width", &width) 371 && mOutputFormat->findInt32("height", &height)) { 372 mStats->setInt32("width", width); 373 mStats->setInt32("height", height); 374 } 375 } 376 377 sp<AMessage> reply = new AMessage(kWhatCodecNotify, this); 378 mCodec->setCallback(reply); 379 380 err = mCodec->start(); 381 if (err != OK) { 382 ALOGE("Failed to start [%s] decoder (err=%d)", mComponentName.c_str(), err); 383 mCodec->release(); 384 mCodec.clear(); 385 handleError(err); 386 return; 387 } 388 389 releaseAndResetMediaBuffers(); 390 391 mPaused = false; 392 mResumePending = false; 393 } 394 395 void NuPlayer::Decoder::onSetParameters(const sp<AMessage> ¶ms) { 396 bool needAdjustLayers = false; 397 float frameRateTotal; 398 if (params->findFloat("frame-rate-total", &frameRateTotal) 399 && mFrameRateTotal != frameRateTotal) { 400 needAdjustLayers = true; 401 mFrameRateTotal = frameRateTotal; 402 } 403 404 int32_t numVideoTemporalLayerTotal; 405 if (params->findInt32("temporal-layer-count", &numVideoTemporalLayerTotal) 406 && numVideoTemporalLayerTotal >= 0 407 && numVideoTemporalLayerTotal <= kMaxNumVideoTemporalLayers 408 && mNumVideoTemporalLayerTotal != numVideoTemporalLayerTotal) { 409 needAdjustLayers = true; 410 mNumVideoTemporalLayerTotal = std::max(numVideoTemporalLayerTotal, 1); 411 } 412 413 if (needAdjustLayers && mNumVideoTemporalLayerTotal > 1) { 414 // TODO: For now, layer fps is calculated for some specific architectures. 415 // But it really should be extracted from the stream. 416 mVideoTemporalLayerAggregateFps[0] = 417 mFrameRateTotal / (float)(1LL << (mNumVideoTemporalLayerTotal - 1)); 418 for (int32_t i = 1; i < mNumVideoTemporalLayerTotal; ++i) { 419 mVideoTemporalLayerAggregateFps[i] = 420 mFrameRateTotal / (float)(1LL << (mNumVideoTemporalLayerTotal - i)) 421 + mVideoTemporalLayerAggregateFps[i - 1]; 422 } 423 } 424 425 float playbackSpeed; 426 if (params->findFloat("playback-speed", &playbackSpeed) 427 && mPlaybackSpeed != playbackSpeed) { 428 needAdjustLayers = true; 429 mPlaybackSpeed = playbackSpeed; 430 } 431 432 if (needAdjustLayers) { 433 float decodeFrameRate = mFrameRateTotal; 434 // enable temporal layering optimization only if we know the layering depth 435 if (mNumVideoTemporalLayerTotal > 1) { 436 int32_t layerId; 437 for (layerId = 0; layerId < mNumVideoTemporalLayerTotal - 1; ++layerId) { 438 if (mVideoTemporalLayerAggregateFps[layerId] * mPlaybackSpeed 439 >= kDisplayRefreshingRate * 0.9) { 440 break; 441 } 442 } 443 mNumVideoTemporalLayerAllowed = layerId + 1; 444 decodeFrameRate = mVideoTemporalLayerAggregateFps[layerId]; 445 } 446 ALOGV("onSetParameters: allowed layers=%d, decodeFps=%g", 447 mNumVideoTemporalLayerAllowed, decodeFrameRate); 448 449 if (mCodec == NULL) { 450 ALOGW("onSetParameters called before codec is created."); 451 return; 452 } 453 454 sp<AMessage> codecParams = new AMessage(); 455 codecParams->setFloat("operating-rate", decodeFrameRate * mPlaybackSpeed); 456 mCodec->setParameters(codecParams); 457 } 458 } 459 460 void NuPlayer::Decoder::onSetRenderer(const sp<Renderer> &renderer) { 461 mRenderer = renderer; 462 } 463 464 void NuPlayer::Decoder::onResume(bool notifyComplete) { 465 mPaused = false; 466 467 if (notifyComplete) { 468 mResumePending = true; 469 } 470 471 if (mCodec == NULL) { 472 ALOGE("[%s] onResume without a valid codec", mComponentName.c_str()); 473 handleError(NO_INIT); 474 return; 475 } 476 mCodec->start(); 477 } 478 479 void NuPlayer::Decoder::doFlush(bool notifyComplete) { 480 if (mCCDecoder != NULL) { 481 mCCDecoder->flush(); 482 } 483 484 if (mRenderer != NULL) { 485 mRenderer->flush(mIsAudio, notifyComplete); 486 mRenderer->signalTimeDiscontinuity(); 487 } 488 489 status_t err = OK; 490 if (mCodec != NULL) { 491 err = mCodec->flush(); 492 mCSDsToSubmit = mCSDsForCurrentFormat; // copy operator 493 ++mBufferGeneration; 494 } 495 496 if (err != OK) { 497 ALOGE("failed to flush [%s] (err=%d)", mComponentName.c_str(), err); 498 handleError(err); 499 // finish with posting kWhatFlushCompleted. 500 // we attempt to release the buffers even if flush fails. 501 } 502 releaseAndResetMediaBuffers(); 503 mPaused = true; 504 } 505 506 507 void NuPlayer::Decoder::onFlush() { 508 doFlush(true); 509 510 if (isDiscontinuityPending()) { 511 // This could happen if the client starts seeking/shutdown 512 // after we queued an EOS for discontinuities. 513 // We can consider discontinuity handled. 514 finishHandleDiscontinuity(false /* flushOnTimeChange */); 515 } 516 517 sp<AMessage> notify = mNotify->dup(); 518 notify->setInt32("what", kWhatFlushCompleted); 519 notify->post(); 520 } 521 522 void NuPlayer::Decoder::onShutdown(bool notifyComplete) { 523 status_t err = OK; 524 525 // if there is a pending resume request, notify complete now 526 notifyResumeCompleteIfNecessary(); 527 528 if (mCodec != NULL) { 529 err = mCodec->release(); 530 mCodec = NULL; 531 ++mBufferGeneration; 532 533 if (mSurface != NULL) { 534 // reconnect to surface as MediaCodec disconnected from it 535 status_t error = nativeWindowConnect(mSurface.get(), "onShutdown"); 536 ALOGW_IF(error != NO_ERROR, 537 "[%s] failed to connect to native window, error=%d", 538 mComponentName.c_str(), error); 539 } 540 mComponentName = "decoder"; 541 } 542 543 releaseAndResetMediaBuffers(); 544 545 if (err != OK) { 546 ALOGE("failed to release [%s] (err=%d)", mComponentName.c_str(), err); 547 handleError(err); 548 // finish with posting kWhatShutdownCompleted. 549 } 550 551 if (notifyComplete) { 552 sp<AMessage> notify = mNotify->dup(); 553 notify->setInt32("what", kWhatShutdownCompleted); 554 notify->post(); 555 mPaused = true; 556 } 557 } 558 559 /* 560 * returns true if we should request more data 561 */ 562 bool NuPlayer::Decoder::doRequestBuffers() { 563 if (isDiscontinuityPending()) { 564 return false; 565 } 566 status_t err = OK; 567 while (err == OK && !mDequeuedInputBuffers.empty()) { 568 size_t bufferIx = *mDequeuedInputBuffers.begin(); 569 sp<AMessage> msg = new AMessage(); 570 msg->setSize("buffer-ix", bufferIx); 571 err = fetchInputData(msg); 572 if (err != OK && err != ERROR_END_OF_STREAM) { 573 // if EOS, need to queue EOS buffer 574 break; 575 } 576 mDequeuedInputBuffers.erase(mDequeuedInputBuffers.begin()); 577 578 if (!mPendingInputMessages.empty() 579 || !onInputBufferFetched(msg)) { 580 mPendingInputMessages.push_back(msg); 581 } 582 } 583 584 return err == -EWOULDBLOCK 585 && mSource->feedMoreTSData() == OK; 586 } 587 588 void NuPlayer::Decoder::handleError(int32_t err) 589 { 590 // We cannot immediately release the codec due to buffers still outstanding 591 // in the renderer. We signal to the player the error so it can shutdown/release the 592 // decoder after flushing and increment the generation to discard unnecessary messages. 593 594 ++mBufferGeneration; 595 596 sp<AMessage> notify = mNotify->dup(); 597 notify->setInt32("what", kWhatError); 598 notify->setInt32("err", err); 599 notify->post(); 600 } 601 602 status_t NuPlayer::Decoder::releaseCrypto() 603 { 604 ALOGV("releaseCrypto"); 605 606 sp<AMessage> msg = new AMessage(kWhatDrmReleaseCrypto, this); 607 608 sp<AMessage> response; 609 status_t status = msg->postAndAwaitResponse(&response); 610 if (status == OK && response != NULL) { 611 CHECK(response->findInt32("status", &status)); 612 ALOGV("releaseCrypto ret: %d ", status); 613 } else { 614 ALOGE("releaseCrypto err: %d", status); 615 } 616 617 return status; 618 } 619 620 void NuPlayer::Decoder::onReleaseCrypto(const sp<AMessage>& msg) 621 { 622 status_t status = INVALID_OPERATION; 623 if (mCodec != NULL) { 624 status = mCodec->releaseCrypto(); 625 } else { 626 // returning OK if the codec has been already released 627 status = OK; 628 ALOGE("onReleaseCrypto No mCodec. err: %d", status); 629 } 630 631 sp<AMessage> response = new AMessage; 632 response->setInt32("status", status); 633 // Clearing the state as it's tied to crypto. mIsEncryptedObservedEarlier is sticky though 634 // and lasts for the lifetime of this codec. See its use in fetchInputData. 635 mIsEncrypted = false; 636 637 sp<AReplyToken> replyID; 638 CHECK(msg->senderAwaitsResponse(&replyID)); 639 response->postReply(replyID); 640 } 641 642 bool NuPlayer::Decoder::handleAnInputBuffer(size_t index) { 643 if (isDiscontinuityPending()) { 644 return false; 645 } 646 647 if (mCodec == NULL) { 648 ALOGE("[%s] handleAnInputBuffer without a valid codec", mComponentName.c_str()); 649 handleError(NO_INIT); 650 return false; 651 } 652 653 sp<MediaCodecBuffer> buffer; 654 mCodec->getInputBuffer(index, &buffer); 655 656 if (buffer == NULL) { 657 ALOGE("[%s] handleAnInputBuffer, failed to get input buffer", mComponentName.c_str()); 658 handleError(UNKNOWN_ERROR); 659 return false; 660 } 661 662 if (index >= mInputBuffers.size()) { 663 for (size_t i = mInputBuffers.size(); i <= index; ++i) { 664 mInputBuffers.add(); 665 mMediaBuffers.add(); 666 mInputBufferIsDequeued.add(); 667 mMediaBuffers.editItemAt(i) = NULL; 668 mInputBufferIsDequeued.editItemAt(i) = false; 669 } 670 } 671 mInputBuffers.editItemAt(index) = buffer; 672 673 //CHECK_LT(bufferIx, mInputBuffers.size()); 674 675 if (mMediaBuffers[index] != NULL) { 676 mMediaBuffers[index]->release(); 677 mMediaBuffers.editItemAt(index) = NULL; 678 } 679 mInputBufferIsDequeued.editItemAt(index) = true; 680 681 if (!mCSDsToSubmit.isEmpty()) { 682 sp<AMessage> msg = new AMessage(); 683 msg->setSize("buffer-ix", index); 684 685 sp<ABuffer> buffer = mCSDsToSubmit.itemAt(0); 686 ALOGV("[%s] resubmitting CSD", mComponentName.c_str()); 687 msg->setBuffer("buffer", buffer); 688 mCSDsToSubmit.removeAt(0); 689 if (!onInputBufferFetched(msg)) { 690 handleError(UNKNOWN_ERROR); 691 return false; 692 } 693 return true; 694 } 695 696 while (!mPendingInputMessages.empty()) { 697 sp<AMessage> msg = *mPendingInputMessages.begin(); 698 if (!onInputBufferFetched(msg)) { 699 break; 700 } 701 mPendingInputMessages.erase(mPendingInputMessages.begin()); 702 } 703 704 if (!mInputBufferIsDequeued.editItemAt(index)) { 705 return true; 706 } 707 708 mDequeuedInputBuffers.push_back(index); 709 710 onRequestInputBuffers(); 711 return true; 712 } 713 714 bool NuPlayer::Decoder::handleAnOutputBuffer( 715 size_t index, 716 size_t offset, 717 size_t size, 718 int64_t timeUs, 719 int32_t flags) { 720 if (mCodec == NULL) { 721 ALOGE("[%s] handleAnOutputBuffer without a valid codec", mComponentName.c_str()); 722 handleError(NO_INIT); 723 return false; 724 } 725 726 // CHECK_LT(bufferIx, mOutputBuffers.size()); 727 sp<MediaCodecBuffer> buffer; 728 mCodec->getOutputBuffer(index, &buffer); 729 730 if (buffer == NULL) { 731 ALOGE("[%s] handleAnOutputBuffer, failed to get output buffer", mComponentName.c_str()); 732 handleError(UNKNOWN_ERROR); 733 return false; 734 } 735 736 if (index >= mOutputBuffers.size()) { 737 for (size_t i = mOutputBuffers.size(); i <= index; ++i) { 738 mOutputBuffers.add(); 739 } 740 } 741 742 mOutputBuffers.editItemAt(index) = buffer; 743 744 buffer->setRange(offset, size); 745 buffer->meta()->clear(); 746 buffer->meta()->setInt64("timeUs", timeUs); 747 748 bool eos = flags & MediaCodec::BUFFER_FLAG_EOS; 749 // we do not expect CODECCONFIG or SYNCFRAME for decoder 750 751 sp<AMessage> reply = new AMessage(kWhatRenderBuffer, this); 752 reply->setSize("buffer-ix", index); 753 reply->setInt32("generation", mBufferGeneration); 754 reply->setSize("size", size); 755 756 if (eos) { 757 ALOGV("[%s] saw output EOS", mIsAudio ? "audio" : "video"); 758 759 buffer->meta()->setInt32("eos", true); 760 reply->setInt32("eos", true); 761 } 762 763 mNumFramesTotal += !mIsAudio; 764 765 if (mSkipRenderingUntilMediaTimeUs >= 0) { 766 if (timeUs < mSkipRenderingUntilMediaTimeUs) { 767 ALOGV("[%s] dropping buffer at time %lld as requested.", 768 mComponentName.c_str(), (long long)timeUs); 769 770 reply->post(); 771 if (eos) { 772 notifyResumeCompleteIfNecessary(); 773 if (mRenderer != NULL && !isDiscontinuityPending()) { 774 mRenderer->queueEOS(mIsAudio, ERROR_END_OF_STREAM); 775 } 776 } 777 return true; 778 } 779 780 mSkipRenderingUntilMediaTimeUs = -1; 781 } 782 783 // wait until 1st frame comes out to signal resume complete 784 notifyResumeCompleteIfNecessary(); 785 786 if (mRenderer != NULL) { 787 // send the buffer to renderer. 788 mRenderer->queueBuffer(mIsAudio, buffer, reply); 789 if (eos && !isDiscontinuityPending()) { 790 mRenderer->queueEOS(mIsAudio, ERROR_END_OF_STREAM); 791 } 792 } 793 794 return true; 795 } 796 797 void NuPlayer::Decoder::handleOutputFormatChange(const sp<AMessage> &format) { 798 if (!mIsAudio) { 799 int32_t width, height; 800 if (format->findInt32("width", &width) 801 && format->findInt32("height", &height)) { 802 mStats->setInt32("width", width); 803 mStats->setInt32("height", height); 804 } 805 sp<AMessage> notify = mNotify->dup(); 806 notify->setInt32("what", kWhatVideoSizeChanged); 807 notify->setMessage("format", format); 808 notify->post(); 809 } else if (mRenderer != NULL) { 810 uint32_t flags; 811 int64_t durationUs; 812 bool hasVideo = (mSource->getFormat(false /* audio */) != NULL); 813 if (getAudioDeepBufferSetting() // override regardless of source duration 814 || (mSource->getDuration(&durationUs) == OK 815 && durationUs > AUDIO_SINK_MIN_DEEP_BUFFER_DURATION_US)) { 816 flags = AUDIO_OUTPUT_FLAG_DEEP_BUFFER; 817 } else { 818 flags = AUDIO_OUTPUT_FLAG_NONE; 819 } 820 821 sp<AMessage> reply = new AMessage(kWhatAudioOutputFormatChanged, this); 822 reply->setInt32("generation", mBufferGeneration); 823 mRenderer->changeAudioFormat( 824 format, false /* offloadOnly */, hasVideo, 825 flags, mSource->isStreaming(), reply); 826 } 827 } 828 829 void NuPlayer::Decoder::releaseAndResetMediaBuffers() { 830 for (size_t i = 0; i < mMediaBuffers.size(); i++) { 831 if (mMediaBuffers[i] != NULL) { 832 mMediaBuffers[i]->release(); 833 mMediaBuffers.editItemAt(i) = NULL; 834 } 835 } 836 mMediaBuffers.resize(mInputBuffers.size()); 837 for (size_t i = 0; i < mMediaBuffers.size(); i++) { 838 mMediaBuffers.editItemAt(i) = NULL; 839 } 840 mInputBufferIsDequeued.clear(); 841 mInputBufferIsDequeued.resize(mInputBuffers.size()); 842 for (size_t i = 0; i < mInputBufferIsDequeued.size(); i++) { 843 mInputBufferIsDequeued.editItemAt(i) = false; 844 } 845 846 mPendingInputMessages.clear(); 847 mDequeuedInputBuffers.clear(); 848 mSkipRenderingUntilMediaTimeUs = -1; 849 } 850 851 void NuPlayer::Decoder::requestCodecNotification() { 852 if (mCodec != NULL) { 853 sp<AMessage> reply = new AMessage(kWhatCodecNotify, this); 854 reply->setInt32("generation", mBufferGeneration); 855 mCodec->requestActivityNotification(reply); 856 } 857 } 858 859 bool NuPlayer::Decoder::isStaleReply(const sp<AMessage> &msg) { 860 int32_t generation; 861 CHECK(msg->findInt32("generation", &generation)); 862 return generation != mBufferGeneration; 863 } 864 865 status_t NuPlayer::Decoder::fetchInputData(sp<AMessage> &reply) { 866 sp<ABuffer> accessUnit; 867 bool dropAccessUnit = true; 868 do { 869 status_t err = mSource->dequeueAccessUnit(mIsAudio, &accessUnit); 870 871 if (err == -EWOULDBLOCK) { 872 return err; 873 } else if (err != OK) { 874 if (err == INFO_DISCONTINUITY) { 875 int32_t type; 876 CHECK(accessUnit->meta()->findInt32("discontinuity", &type)); 877 878 bool formatChange = 879 (mIsAudio && 880 (type & ATSParser::DISCONTINUITY_AUDIO_FORMAT)) 881 || (!mIsAudio && 882 (type & ATSParser::DISCONTINUITY_VIDEO_FORMAT)); 883 884 bool timeChange = (type & ATSParser::DISCONTINUITY_TIME) != 0; 885 886 ALOGI("%s discontinuity (format=%d, time=%d)", 887 mIsAudio ? "audio" : "video", formatChange, timeChange); 888 889 bool seamlessFormatChange = false; 890 sp<AMessage> newFormat = mSource->getFormat(mIsAudio); 891 if (formatChange) { 892 seamlessFormatChange = 893 supportsSeamlessFormatChange(newFormat); 894 // treat seamless format change separately 895 formatChange = !seamlessFormatChange; 896 } 897 898 // For format or time change, return EOS to queue EOS input, 899 // then wait for EOS on output. 900 if (formatChange /* not seamless */) { 901 mFormatChangePending = true; 902 err = ERROR_END_OF_STREAM; 903 } else if (timeChange) { 904 rememberCodecSpecificData(newFormat); 905 mTimeChangePending = true; 906 err = ERROR_END_OF_STREAM; 907 } else if (seamlessFormatChange) { 908 // reuse existing decoder and don't flush 909 rememberCodecSpecificData(newFormat); 910 continue; 911 } else { 912 // This stream is unaffected by the discontinuity 913 return -EWOULDBLOCK; 914 } 915 } 916 917 // reply should only be returned without a buffer set 918 // when there is an error (including EOS) 919 CHECK(err != OK); 920 921 reply->setInt32("err", err); 922 return ERROR_END_OF_STREAM; 923 } 924 925 dropAccessUnit = false; 926 if (!mIsAudio && !mIsEncrypted) { 927 // Extra safeguard if higher-level behavior changes. Otherwise, not required now. 928 // Preventing the buffer from being processed (and sent to codec) if this is a later 929 // round of playback but this time without prepareDrm. Or if there is a race between 930 // stop (which is not blocking) and releaseDrm allowing buffers being processed after 931 // Crypto has been released (GenericSource currently prevents this race though). 932 // Particularly doing this check before IsAVCReferenceFrame call to prevent parsing 933 // of encrypted data. 934 if (mIsEncryptedObservedEarlier) { 935 ALOGE("fetchInputData: mismatched mIsEncrypted/mIsEncryptedObservedEarlier (0/1)"); 936 937 return INVALID_OPERATION; 938 } 939 940 int32_t layerId = 0; 941 bool haveLayerId = accessUnit->meta()->findInt32("temporal-layer-id", &layerId); 942 if (mRenderer->getVideoLateByUs() > 100000LL 943 && mIsVideoAVC 944 && !IsAVCReferenceFrame(accessUnit)) { 945 dropAccessUnit = true; 946 } else if (haveLayerId && mNumVideoTemporalLayerTotal > 1) { 947 // Add only one layer each time. 948 if (layerId > mCurrentMaxVideoTemporalLayerId + 1 949 || layerId >= mNumVideoTemporalLayerAllowed) { 950 dropAccessUnit = true; 951 ALOGV("dropping layer(%d), speed=%g, allowed layer count=%d, max layerId=%d", 952 layerId, mPlaybackSpeed, mNumVideoTemporalLayerAllowed, 953 mCurrentMaxVideoTemporalLayerId); 954 } else if (layerId > mCurrentMaxVideoTemporalLayerId) { 955 mCurrentMaxVideoTemporalLayerId = layerId; 956 } else if (layerId == 0 && mNumVideoTemporalLayerTotal > 1 957 && IsIDR(accessUnit->data(), accessUnit->size())) { 958 mCurrentMaxVideoTemporalLayerId = mNumVideoTemporalLayerTotal - 1; 959 } 960 } 961 if (dropAccessUnit) { 962 if (layerId <= mCurrentMaxVideoTemporalLayerId && layerId > 0) { 963 mCurrentMaxVideoTemporalLayerId = layerId - 1; 964 } 965 ++mNumInputFramesDropped; 966 } 967 } 968 } while (dropAccessUnit); 969 970 // ALOGV("returned a valid buffer of %s data", mIsAudio ? "mIsAudio" : "video"); 971 #if 0 972 int64_t mediaTimeUs; 973 CHECK(accessUnit->meta()->findInt64("timeUs", &mediaTimeUs)); 974 ALOGV("[%s] feeding input buffer at media time %.3f", 975 mIsAudio ? "audio" : "video", 976 mediaTimeUs / 1E6); 977 #endif 978 979 if (mCCDecoder != NULL) { 980 mCCDecoder->decode(accessUnit); 981 } 982 983 reply->setBuffer("buffer", accessUnit); 984 985 return OK; 986 } 987 988 bool NuPlayer::Decoder::onInputBufferFetched(const sp<AMessage> &msg) { 989 if (mCodec == NULL) { 990 ALOGE("[%s] onInputBufferFetched without a valid codec", mComponentName.c_str()); 991 handleError(NO_INIT); 992 return false; 993 } 994 995 size_t bufferIx; 996 CHECK(msg->findSize("buffer-ix", &bufferIx)); 997 CHECK_LT(bufferIx, mInputBuffers.size()); 998 sp<MediaCodecBuffer> codecBuffer = mInputBuffers[bufferIx]; 999 1000 sp<ABuffer> buffer; 1001 bool hasBuffer = msg->findBuffer("buffer", &buffer); 1002 bool needsCopy = true; 1003 1004 if (buffer == NULL /* includes !hasBuffer */) { 1005 int32_t streamErr = ERROR_END_OF_STREAM; 1006 CHECK(msg->findInt32("err", &streamErr) || !hasBuffer); 1007 1008 CHECK(streamErr != OK); 1009 1010 // attempt to queue EOS 1011 status_t err = mCodec->queueInputBuffer( 1012 bufferIx, 1013 0, 1014 0, 1015 0, 1016 MediaCodec::BUFFER_FLAG_EOS); 1017 if (err == OK) { 1018 mInputBufferIsDequeued.editItemAt(bufferIx) = false; 1019 } else if (streamErr == ERROR_END_OF_STREAM) { 1020 streamErr = err; 1021 // err will not be ERROR_END_OF_STREAM 1022 } 1023 1024 if (streamErr != ERROR_END_OF_STREAM) { 1025 ALOGE("Stream error for [%s] (err=%d), EOS %s queued", 1026 mComponentName.c_str(), 1027 streamErr, 1028 err == OK ? "successfully" : "unsuccessfully"); 1029 handleError(streamErr); 1030 } 1031 } else { 1032 sp<AMessage> extra; 1033 if (buffer->meta()->findMessage("extra", &extra) && extra != NULL) { 1034 int64_t resumeAtMediaTimeUs; 1035 if (extra->findInt64( 1036 "resume-at-mediaTimeUs", &resumeAtMediaTimeUs)) { 1037 ALOGV("[%s] suppressing rendering until %lld us", 1038 mComponentName.c_str(), (long long)resumeAtMediaTimeUs); 1039 mSkipRenderingUntilMediaTimeUs = resumeAtMediaTimeUs; 1040 } 1041 } 1042 1043 int64_t timeUs = 0; 1044 uint32_t flags = 0; 1045 CHECK(buffer->meta()->findInt64("timeUs", &timeUs)); 1046 1047 int32_t eos, csd; 1048 // we do not expect SYNCFRAME for decoder 1049 if (buffer->meta()->findInt32("eos", &eos) && eos) { 1050 flags |= MediaCodec::BUFFER_FLAG_EOS; 1051 } else if (buffer->meta()->findInt32("csd", &csd) && csd) { 1052 flags |= MediaCodec::BUFFER_FLAG_CODECCONFIG; 1053 } 1054 1055 // Modular DRM 1056 MediaBufferBase *mediaBuf = NULL; 1057 NuPlayerDrm::CryptoInfo *cryptInfo = NULL; 1058 1059 // copy into codec buffer 1060 if (needsCopy) { 1061 if (buffer->size() > codecBuffer->capacity()) { 1062 handleError(ERROR_BUFFER_TOO_SMALL); 1063 mDequeuedInputBuffers.push_back(bufferIx); 1064 return false; 1065 } 1066 1067 if (buffer->data() != NULL) { 1068 codecBuffer->setRange(0, buffer->size()); 1069 memcpy(codecBuffer->data(), buffer->data(), buffer->size()); 1070 } else { // No buffer->data() 1071 //Modular DRM 1072 sp<RefBase> holder; 1073 if (buffer->meta()->findObject("mediaBufferHolder", &holder)) { 1074 mediaBuf = (holder != nullptr) ? 1075 static_cast<MediaBufferHolder*>(holder.get())->mediaBuffer() : nullptr; 1076 } 1077 if (mediaBuf != NULL) { 1078 if (mediaBuf->size() > codecBuffer->capacity()) { 1079 handleError(ERROR_BUFFER_TOO_SMALL); 1080 mDequeuedInputBuffers.push_back(bufferIx); 1081 return false; 1082 } 1083 1084 codecBuffer->setRange(0, mediaBuf->size()); 1085 memcpy(codecBuffer->data(), mediaBuf->data(), mediaBuf->size()); 1086 1087 MetaDataBase &meta_data = mediaBuf->meta_data(); 1088 cryptInfo = NuPlayerDrm::getSampleCryptoInfo(meta_data); 1089 } else { // No mediaBuf 1090 ALOGE("onInputBufferFetched: buffer->data()/mediaBuf are NULL for %p", 1091 buffer.get()); 1092 handleError(UNKNOWN_ERROR); 1093 return false; 1094 } 1095 } // buffer->data() 1096 } // needsCopy 1097 1098 status_t err; 1099 AString errorDetailMsg; 1100 if (cryptInfo != NULL) { 1101 err = mCodec->queueSecureInputBuffer( 1102 bufferIx, 1103 codecBuffer->offset(), 1104 cryptInfo->subSamples, 1105 cryptInfo->numSubSamples, 1106 cryptInfo->key, 1107 cryptInfo->iv, 1108 cryptInfo->mode, 1109 cryptInfo->pattern, 1110 timeUs, 1111 flags, 1112 &errorDetailMsg); 1113 // synchronous call so done with cryptInfo here 1114 free(cryptInfo); 1115 } else { 1116 err = mCodec->queueInputBuffer( 1117 bufferIx, 1118 codecBuffer->offset(), 1119 codecBuffer->size(), 1120 timeUs, 1121 flags, 1122 &errorDetailMsg); 1123 } // no cryptInfo 1124 1125 if (err != OK) { 1126 ALOGE("onInputBufferFetched: queue%sInputBuffer failed for [%s] (err=%d, %s)", 1127 (cryptInfo != NULL ? "Secure" : ""), 1128 mComponentName.c_str(), err, errorDetailMsg.c_str()); 1129 handleError(err); 1130 } else { 1131 mInputBufferIsDequeued.editItemAt(bufferIx) = false; 1132 } 1133 1134 } // buffer != NULL 1135 return true; 1136 } 1137 1138 void NuPlayer::Decoder::onRenderBuffer(const sp<AMessage> &msg) { 1139 status_t err; 1140 int32_t render; 1141 size_t bufferIx; 1142 int32_t eos; 1143 size_t size; 1144 CHECK(msg->findSize("buffer-ix", &bufferIx)); 1145 1146 if (!mIsAudio) { 1147 int64_t timeUs; 1148 sp<MediaCodecBuffer> buffer = mOutputBuffers[bufferIx]; 1149 buffer->meta()->findInt64("timeUs", &timeUs); 1150 1151 if (mCCDecoder != NULL && mCCDecoder->isSelected()) { 1152 mCCDecoder->display(timeUs); 1153 } 1154 } 1155 1156 if (mCodec == NULL) { 1157 err = NO_INIT; 1158 } else if (msg->findInt32("render", &render) && render) { 1159 int64_t timestampNs; 1160 CHECK(msg->findInt64("timestampNs", ×tampNs)); 1161 err = mCodec->renderOutputBufferAndRelease(bufferIx, timestampNs); 1162 } else { 1163 if (!msg->findInt32("eos", &eos) || !eos || 1164 !msg->findSize("size", &size) || size) { 1165 mNumOutputFramesDropped += !mIsAudio; 1166 } 1167 err = mCodec->releaseOutputBuffer(bufferIx); 1168 } 1169 if (err != OK) { 1170 ALOGE("failed to release output buffer for [%s] (err=%d)", 1171 mComponentName.c_str(), err); 1172 handleError(err); 1173 } 1174 if (msg->findInt32("eos", &eos) && eos 1175 && isDiscontinuityPending()) { 1176 finishHandleDiscontinuity(true /* flushOnTimeChange */); 1177 } 1178 } 1179 1180 bool NuPlayer::Decoder::isDiscontinuityPending() const { 1181 return mFormatChangePending || mTimeChangePending; 1182 } 1183 1184 void NuPlayer::Decoder::finishHandleDiscontinuity(bool flushOnTimeChange) { 1185 ALOGV("finishHandleDiscontinuity: format %d, time %d, flush %d", 1186 mFormatChangePending, mTimeChangePending, flushOnTimeChange); 1187 1188 // If we have format change, pause and wait to be killed; 1189 // If we have time change only, flush and restart fetching. 1190 1191 if (mFormatChangePending) { 1192 mPaused = true; 1193 } else if (mTimeChangePending) { 1194 if (flushOnTimeChange) { 1195 doFlush(false /* notifyComplete */); 1196 signalResume(false /* notifyComplete */); 1197 } 1198 } 1199 1200 // Notify NuPlayer to either shutdown decoder, or rescan sources 1201 sp<AMessage> msg = mNotify->dup(); 1202 msg->setInt32("what", kWhatInputDiscontinuity); 1203 msg->setInt32("formatChange", mFormatChangePending); 1204 msg->post(); 1205 1206 mFormatChangePending = false; 1207 mTimeChangePending = false; 1208 } 1209 1210 bool NuPlayer::Decoder::supportsSeamlessAudioFormatChange( 1211 const sp<AMessage> &targetFormat) const { 1212 if (targetFormat == NULL) { 1213 return true; 1214 } 1215 1216 AString mime; 1217 if (!targetFormat->findString("mime", &mime)) { 1218 return false; 1219 } 1220 1221 if (!strcasecmp(mime.c_str(), MEDIA_MIMETYPE_AUDIO_AAC)) { 1222 // field-by-field comparison 1223 const char * keys[] = { "channel-count", "sample-rate", "is-adts" }; 1224 for (unsigned int i = 0; i < sizeof(keys) / sizeof(keys[0]); i++) { 1225 int32_t oldVal, newVal; 1226 if (!mInputFormat->findInt32(keys[i], &oldVal) || 1227 !targetFormat->findInt32(keys[i], &newVal) || 1228 oldVal != newVal) { 1229 return false; 1230 } 1231 } 1232 1233 sp<ABuffer> oldBuf, newBuf; 1234 if (mInputFormat->findBuffer("csd-0", &oldBuf) && 1235 targetFormat->findBuffer("csd-0", &newBuf)) { 1236 if (oldBuf->size() != newBuf->size()) { 1237 return false; 1238 } 1239 return !memcmp(oldBuf->data(), newBuf->data(), oldBuf->size()); 1240 } 1241 } 1242 return false; 1243 } 1244 1245 bool NuPlayer::Decoder::supportsSeamlessFormatChange(const sp<AMessage> &targetFormat) const { 1246 if (mInputFormat == NULL) { 1247 return false; 1248 } 1249 1250 if (targetFormat == NULL) { 1251 return true; 1252 } 1253 1254 AString oldMime, newMime; 1255 if (!mInputFormat->findString("mime", &oldMime) 1256 || !targetFormat->findString("mime", &newMime) 1257 || !(oldMime == newMime)) { 1258 return false; 1259 } 1260 1261 bool audio = !strncasecmp(oldMime.c_str(), "audio/", strlen("audio/")); 1262 bool seamless; 1263 if (audio) { 1264 seamless = supportsSeamlessAudioFormatChange(targetFormat); 1265 } else { 1266 int32_t isAdaptive; 1267 seamless = (mCodec != NULL && 1268 mInputFormat->findInt32("adaptive-playback", &isAdaptive) && 1269 isAdaptive); 1270 } 1271 1272 ALOGV("%s seamless support for %s", seamless ? "yes" : "no", oldMime.c_str()); 1273 return seamless; 1274 } 1275 1276 void NuPlayer::Decoder::rememberCodecSpecificData(const sp<AMessage> &format) { 1277 if (format == NULL) { 1278 return; 1279 } 1280 mCSDsForCurrentFormat.clear(); 1281 for (int32_t i = 0; ; ++i) { 1282 AString tag = "csd-"; 1283 tag.append(i); 1284 sp<ABuffer> buffer; 1285 if (!format->findBuffer(tag.c_str(), &buffer)) { 1286 break; 1287 } 1288 mCSDsForCurrentFormat.push(buffer); 1289 } 1290 } 1291 1292 void NuPlayer::Decoder::notifyResumeCompleteIfNecessary() { 1293 if (mResumePending) { 1294 mResumePending = false; 1295 1296 sp<AMessage> notify = mNotify->dup(); 1297 notify->setInt32("what", kWhatResumeCompleted); 1298 notify->post(); 1299 } 1300 } 1301 1302 } // namespace android 1303 1304
