1 /* 2 * Copyright 2014 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17 //#define LOG_NDEBUG 0 18 #define LOG_TAG "NuPlayerDecoder" 19 #include <utils/Log.h> 20 #include <inttypes.h> 21 22 #include <algorithm> 23 24 #include "NuPlayerCCDecoder.h" 25 #include "NuPlayerDecoder.h" 26 #include "NuPlayerDrm.h" 27 #include "NuPlayerRenderer.h" 28 #include "NuPlayerSource.h" 29 30 #include <cutils/properties.h> 31 #include <media/ICrypto.h> 32 #include <media/MediaCodecBuffer.h> 33 #include <media/stagefright/foundation/ABuffer.h> 34 #include <media/stagefright/foundation/ADebug.h> 35 #include <media/stagefright/foundation/AMessage.h> 36 #include <media/stagefright/MediaBuffer.h> 37 #include <media/stagefright/MediaCodec.h> 38 #include <media/stagefright/MediaDefs.h> 39 #include <media/stagefright/MediaErrors.h> 40 #include <media/stagefright/SurfaceUtils.h> 41 #include <gui/Surface.h> 42 43 #include "avc_utils.h" 44 #include "ATSParser.h" 45 46 namespace android { 47 48 static float kDisplayRefreshingRate = 60.f; // TODO: get this from the display 49 50 // The default total video frame rate of a stream when that info is not available from 51 // the source. 52 static float kDefaultVideoFrameRateTotal = 30.f; 53 54 static inline bool getAudioDeepBufferSetting() { 55 return property_get_bool("media.stagefright.audio.deep", false /* default_value */); 56 } 57 58 NuPlayer::Decoder::Decoder( 59 const sp<AMessage> ¬ify, 60 const sp<Source> &source, 61 pid_t pid, 62 uid_t uid, 63 const sp<Renderer> &renderer, 64 const sp<Surface> &surface, 65 const sp<CCDecoder> &ccDecoder) 66 : DecoderBase(notify), 67 mSurface(surface), 68 mSource(source), 69 mRenderer(renderer), 70 mCCDecoder(ccDecoder), 71 mPid(pid), 72 mUid(uid), 73 mSkipRenderingUntilMediaTimeUs(-1ll), 74 mNumFramesTotal(0ll), 75 mNumInputFramesDropped(0ll), 76 mNumOutputFramesDropped(0ll), 77 mVideoWidth(0), 78 mVideoHeight(0), 79 mIsAudio(true), 80 mIsVideoAVC(false), 81 mIsSecure(false), 82 mIsEncrypted(false), 83 mIsEncryptedObservedEarlier(false), 84 mFormatChangePending(false), 85 mTimeChangePending(false), 86 mFrameRateTotal(kDefaultVideoFrameRateTotal), 87 mPlaybackSpeed(1.0f), 88 mNumVideoTemporalLayerTotal(1), // decode all layers 89 mNumVideoTemporalLayerAllowed(1), 90 mCurrentMaxVideoTemporalLayerId(0), 91 mResumePending(false), 92 mComponentName("decoder") { 93 mCodecLooper = new ALooper; 94 mCodecLooper->setName("NPDecoder-CL"); 95 mCodecLooper->start(false, false, ANDROID_PRIORITY_AUDIO); 96 mVideoTemporalLayerAggregateFps[0] = mFrameRateTotal; 97 } 98 99 NuPlayer::Decoder::~Decoder() { 100 // Need to stop looper first since mCodec could be accessed on the mDecoderLooper. 101 stopLooper(); 102 if (mCodec != NULL) { 103 mCodec->release(); 104 } 105 releaseAndResetMediaBuffers(); 106 } 107 108 sp<AMessage> NuPlayer::Decoder::getStats() const { 109 mStats->setInt64("frames-total", mNumFramesTotal); 110 mStats->setInt64("frames-dropped-input", mNumInputFramesDropped); 111 mStats->setInt64("frames-dropped-output", mNumOutputFramesDropped); 112 return mStats; 113 } 114 115 status_t NuPlayer::Decoder::setVideoSurface(const sp<Surface> &surface) { 116 if (surface == NULL || ADebug::isExperimentEnabled("legacy-setsurface")) { 117 return BAD_VALUE; 118 } 119 120 sp<AMessage> msg = new AMessage(kWhatSetVideoSurface, this); 121 122 msg->setObject("surface", surface); 123 sp<AMessage> response; 124 status_t err = msg->postAndAwaitResponse(&response); 125 if (err == OK && response != NULL) { 126 CHECK(response->findInt32("err", &err)); 127 } 128 return err; 129 } 130 131 void NuPlayer::Decoder::onMessageReceived(const sp<AMessage> &msg) { 132 ALOGV("[%s] onMessage: %s", mComponentName.c_str(), msg->debugString().c_str()); 133 134 switch (msg->what()) { 135 case kWhatCodecNotify: 136 { 137 int32_t cbID; 138 CHECK(msg->findInt32("callbackID", &cbID)); 139 140 ALOGV("[%s] kWhatCodecNotify: cbID = %d, paused = %d", 141 mIsAudio ? "audio" : "video", cbID, mPaused); 142 143 if (mPaused) { 144 break; 145 } 146 147 switch (cbID) { 148 case MediaCodec::CB_INPUT_AVAILABLE: 149 { 150 int32_t index; 151 CHECK(msg->findInt32("index", &index)); 152 153 handleAnInputBuffer(index); 154 break; 155 } 156 157 case MediaCodec::CB_OUTPUT_AVAILABLE: 158 { 159 int32_t index; 160 size_t offset; 161 size_t size; 162 int64_t timeUs; 163 int32_t flags; 164 165 CHECK(msg->findInt32("index", &index)); 166 CHECK(msg->findSize("offset", &offset)); 167 CHECK(msg->findSize("size", &size)); 168 CHECK(msg->findInt64("timeUs", &timeUs)); 169 CHECK(msg->findInt32("flags", &flags)); 170 171 handleAnOutputBuffer(index, offset, size, timeUs, flags); 172 break; 173 } 174 175 case MediaCodec::CB_OUTPUT_FORMAT_CHANGED: 176 { 177 sp<AMessage> format; 178 CHECK(msg->findMessage("format", &format)); 179 180 handleOutputFormatChange(format); 181 break; 182 } 183 184 case MediaCodec::CB_ERROR: 185 { 186 status_t err; 187 CHECK(msg->findInt32("err", &err)); 188 ALOGE("Decoder (%s) reported error : 0x%x", 189 mIsAudio ? "audio" : "video", err); 190 191 handleError(err); 192 break; 193 } 194 195 default: 196 { 197 TRESPASS(); 198 break; 199 } 200 } 201 202 break; 203 } 204 205 case kWhatRenderBuffer: 206 { 207 if (!isStaleReply(msg)) { 208 onRenderBuffer(msg); 209 } 210 break; 211 } 212 213 case kWhatAudioOutputFormatChanged: 214 { 215 if (!isStaleReply(msg)) { 216 status_t err; 217 if (msg->findInt32("err", &err) && err != OK) { 218 ALOGE("Renderer reported 0x%x when changing audio output format", err); 219 handleError(err); 220 } 221 } 222 break; 223 } 224 225 case kWhatSetVideoSurface: 226 { 227 sp<AReplyToken> replyID; 228 CHECK(msg->senderAwaitsResponse(&replyID)); 229 230 sp<RefBase> obj; 231 CHECK(msg->findObject("surface", &obj)); 232 sp<Surface> surface = static_cast<Surface *>(obj.get()); // non-null 233 int32_t err = INVALID_OPERATION; 234 // NOTE: in practice mSurface is always non-null, but checking here for completeness 235 if (mCodec != NULL && mSurface != NULL) { 236 // TODO: once AwesomePlayer is removed, remove this automatic connecting 237 // to the surface by MediaPlayerService. 238 // 239 // at this point MediaPlayerService::client has already connected to the 240 // surface, which MediaCodec does not expect 241 err = nativeWindowDisconnect(surface.get(), "kWhatSetVideoSurface(surface)"); 242 if (err == OK) { 243 err = mCodec->setSurface(surface); 244 ALOGI_IF(err, "codec setSurface returned: %d", err); 245 if (err == OK) { 246 // reconnect to the old surface as MPS::Client will expect to 247 // be able to disconnect from it. 248 (void)nativeWindowConnect(mSurface.get(), "kWhatSetVideoSurface(mSurface)"); 249 mSurface = surface; 250 } 251 } 252 if (err != OK) { 253 // reconnect to the new surface on error as MPS::Client will expect to 254 // be able to disconnect from it. 255 (void)nativeWindowConnect(surface.get(), "kWhatSetVideoSurface(err)"); 256 } 257 } 258 259 sp<AMessage> response = new AMessage; 260 response->setInt32("err", err); 261 response->postReply(replyID); 262 break; 263 } 264 265 case kWhatDrmReleaseCrypto: 266 { 267 ALOGV("kWhatDrmReleaseCrypto"); 268 onReleaseCrypto(msg); 269 break; 270 } 271 272 default: 273 DecoderBase::onMessageReceived(msg); 274 break; 275 } 276 } 277 278 void NuPlayer::Decoder::onConfigure(const sp<AMessage> &format) { 279 CHECK(mCodec == NULL); 280 281 mFormatChangePending = false; 282 mTimeChangePending = false; 283 284 ++mBufferGeneration; 285 286 AString mime; 287 CHECK(format->findString("mime", &mime)); 288 289 mIsAudio = !strncasecmp("audio/", mime.c_str(), 6); 290 mIsVideoAVC = !strcasecmp(MEDIA_MIMETYPE_VIDEO_AVC, mime.c_str()); 291 292 mComponentName = mime; 293 mComponentName.append(" decoder"); 294 ALOGV("[%s] onConfigure (surface=%p)", mComponentName.c_str(), mSurface.get()); 295 296 mCodec = MediaCodec::CreateByType( 297 mCodecLooper, mime.c_str(), false /* encoder */, NULL /* err */, mPid, mUid); 298 int32_t secure = 0; 299 if (format->findInt32("secure", &secure) && secure != 0) { 300 if (mCodec != NULL) { 301 mCodec->getName(&mComponentName); 302 mComponentName.append(".secure"); 303 mCodec->release(); 304 ALOGI("[%s] creating", mComponentName.c_str()); 305 mCodec = MediaCodec::CreateByComponentName( 306 mCodecLooper, mComponentName.c_str(), NULL /* err */, mPid, mUid); 307 } 308 } 309 if (mCodec == NULL) { 310 ALOGE("Failed to create %s%s decoder", 311 (secure ? "secure " : ""), mime.c_str()); 312 handleError(UNKNOWN_ERROR); 313 return; 314 } 315 mIsSecure = secure; 316 317 mCodec->getName(&mComponentName); 318 319 status_t err; 320 if (mSurface != NULL) { 321 // disconnect from surface as MediaCodec will reconnect 322 err = nativeWindowDisconnect(mSurface.get(), "onConfigure"); 323 // We treat this as a warning, as this is a preparatory step. 324 // Codec will try to connect to the surface, which is where 325 // any error signaling will occur. 326 ALOGW_IF(err != OK, "failed to disconnect from surface: %d", err); 327 } 328 329 // Modular DRM 330 void *pCrypto; 331 if (!format->findPointer("crypto", &pCrypto)) { 332 pCrypto = NULL; 333 } 334 sp<ICrypto> crypto = (ICrypto*)pCrypto; 335 // non-encrypted source won't have a crypto 336 mIsEncrypted = (crypto != NULL); 337 // configure is called once; still using OR in case the behavior changes. 338 mIsEncryptedObservedEarlier = mIsEncryptedObservedEarlier || mIsEncrypted; 339 ALOGV("onConfigure mCrypto: %p (%d) mIsSecure: %d", 340 crypto.get(), (crypto != NULL ? crypto->getStrongCount() : 0), mIsSecure); 341 342 err = mCodec->configure( 343 format, mSurface, crypto, 0 /* flags */); 344 345 if (err != OK) { 346 ALOGE("Failed to configure [%s] decoder (err=%d)", mComponentName.c_str(), err); 347 mCodec->release(); 348 mCodec.clear(); 349 handleError(err); 350 return; 351 } 352 rememberCodecSpecificData(format); 353 354 // the following should work in configured state 355 CHECK_EQ((status_t)OK, mCodec->getOutputFormat(&mOutputFormat)); 356 CHECK_EQ((status_t)OK, mCodec->getInputFormat(&mInputFormat)); 357 358 mStats->setString("mime", mime.c_str()); 359 mStats->setString("component-name", mComponentName.c_str()); 360 361 if (!mIsAudio) { 362 int32_t width, height; 363 if (mOutputFormat->findInt32("width", &width) 364 && mOutputFormat->findInt32("height", &height)) { 365 mStats->setInt32("width", width); 366 mStats->setInt32("height", height); 367 } 368 } 369 370 sp<AMessage> reply = new AMessage(kWhatCodecNotify, this); 371 mCodec->setCallback(reply); 372 373 err = mCodec->start(); 374 if (err != OK) { 375 ALOGE("Failed to start [%s] decoder (err=%d)", mComponentName.c_str(), err); 376 mCodec->release(); 377 mCodec.clear(); 378 handleError(err); 379 return; 380 } 381 382 releaseAndResetMediaBuffers(); 383 384 mPaused = false; 385 mResumePending = false; 386 } 387 388 void NuPlayer::Decoder::onSetParameters(const sp<AMessage> ¶ms) { 389 bool needAdjustLayers = false; 390 float frameRateTotal; 391 if (params->findFloat("frame-rate-total", &frameRateTotal) 392 && mFrameRateTotal != frameRateTotal) { 393 needAdjustLayers = true; 394 mFrameRateTotal = frameRateTotal; 395 } 396 397 int32_t numVideoTemporalLayerTotal; 398 if (params->findInt32("temporal-layer-count", &numVideoTemporalLayerTotal) 399 && numVideoTemporalLayerTotal >= 0 400 && numVideoTemporalLayerTotal <= kMaxNumVideoTemporalLayers 401 && mNumVideoTemporalLayerTotal != numVideoTemporalLayerTotal) { 402 needAdjustLayers = true; 403 mNumVideoTemporalLayerTotal = std::max(numVideoTemporalLayerTotal, 1); 404 } 405 406 if (needAdjustLayers && mNumVideoTemporalLayerTotal > 1) { 407 // TODO: For now, layer fps is calculated for some specific architectures. 408 // But it really should be extracted from the stream. 409 mVideoTemporalLayerAggregateFps[0] = 410 mFrameRateTotal / (float)(1ll << (mNumVideoTemporalLayerTotal - 1)); 411 for (int32_t i = 1; i < mNumVideoTemporalLayerTotal; ++i) { 412 mVideoTemporalLayerAggregateFps[i] = 413 mFrameRateTotal / (float)(1ll << (mNumVideoTemporalLayerTotal - i)) 414 + mVideoTemporalLayerAggregateFps[i - 1]; 415 } 416 } 417 418 float playbackSpeed; 419 if (params->findFloat("playback-speed", &playbackSpeed) 420 && mPlaybackSpeed != playbackSpeed) { 421 needAdjustLayers = true; 422 mPlaybackSpeed = playbackSpeed; 423 } 424 425 if (needAdjustLayers) { 426 float decodeFrameRate = mFrameRateTotal; 427 // enable temporal layering optimization only if we know the layering depth 428 if (mNumVideoTemporalLayerTotal > 1) { 429 int32_t layerId; 430 for (layerId = 0; layerId < mNumVideoTemporalLayerTotal - 1; ++layerId) { 431 if (mVideoTemporalLayerAggregateFps[layerId] * mPlaybackSpeed 432 >= kDisplayRefreshingRate * 0.9) { 433 break; 434 } 435 } 436 mNumVideoTemporalLayerAllowed = layerId + 1; 437 decodeFrameRate = mVideoTemporalLayerAggregateFps[layerId]; 438 } 439 ALOGV("onSetParameters: allowed layers=%d, decodeFps=%g", 440 mNumVideoTemporalLayerAllowed, decodeFrameRate); 441 442 if (mCodec == NULL) { 443 ALOGW("onSetParameters called before codec is created."); 444 return; 445 } 446 447 sp<AMessage> codecParams = new AMessage(); 448 codecParams->setFloat("operating-rate", decodeFrameRate * mPlaybackSpeed); 449 mCodec->setParameters(codecParams); 450 } 451 } 452 453 void NuPlayer::Decoder::onSetRenderer(const sp<Renderer> &renderer) { 454 mRenderer = renderer; 455 } 456 457 void NuPlayer::Decoder::onResume(bool notifyComplete) { 458 mPaused = false; 459 460 if (notifyComplete) { 461 mResumePending = true; 462 } 463 464 if (mCodec == NULL) { 465 ALOGE("[%s] onResume without a valid codec", mComponentName.c_str()); 466 handleError(NO_INIT); 467 return; 468 } 469 mCodec->start(); 470 } 471 472 void NuPlayer::Decoder::doFlush(bool notifyComplete) { 473 if (mCCDecoder != NULL) { 474 mCCDecoder->flush(); 475 } 476 477 if (mRenderer != NULL) { 478 mRenderer->flush(mIsAudio, notifyComplete); 479 mRenderer->signalTimeDiscontinuity(); 480 } 481 482 status_t err = OK; 483 if (mCodec != NULL) { 484 err = mCodec->flush(); 485 mCSDsToSubmit = mCSDsForCurrentFormat; // copy operator 486 ++mBufferGeneration; 487 } 488 489 if (err != OK) { 490 ALOGE("failed to flush [%s] (err=%d)", mComponentName.c_str(), err); 491 handleError(err); 492 // finish with posting kWhatFlushCompleted. 493 // we attempt to release the buffers even if flush fails. 494 } 495 releaseAndResetMediaBuffers(); 496 mPaused = true; 497 } 498 499 500 void NuPlayer::Decoder::onFlush() { 501 doFlush(true); 502 503 if (isDiscontinuityPending()) { 504 // This could happen if the client starts seeking/shutdown 505 // after we queued an EOS for discontinuities. 506 // We can consider discontinuity handled. 507 finishHandleDiscontinuity(false /* flushOnTimeChange */); 508 } 509 510 sp<AMessage> notify = mNotify->dup(); 511 notify->setInt32("what", kWhatFlushCompleted); 512 notify->post(); 513 } 514 515 void NuPlayer::Decoder::onShutdown(bool notifyComplete) { 516 status_t err = OK; 517 518 // if there is a pending resume request, notify complete now 519 notifyResumeCompleteIfNecessary(); 520 521 if (mCodec != NULL) { 522 err = mCodec->release(); 523 mCodec = NULL; 524 ++mBufferGeneration; 525 526 if (mSurface != NULL) { 527 // reconnect to surface as MediaCodec disconnected from it 528 status_t error = nativeWindowConnect(mSurface.get(), "onShutdown"); 529 ALOGW_IF(error != NO_ERROR, 530 "[%s] failed to connect to native window, error=%d", 531 mComponentName.c_str(), error); 532 } 533 mComponentName = "decoder"; 534 } 535 536 releaseAndResetMediaBuffers(); 537 538 if (err != OK) { 539 ALOGE("failed to release [%s] (err=%d)", mComponentName.c_str(), err); 540 handleError(err); 541 // finish with posting kWhatShutdownCompleted. 542 } 543 544 if (notifyComplete) { 545 sp<AMessage> notify = mNotify->dup(); 546 notify->setInt32("what", kWhatShutdownCompleted); 547 notify->post(); 548 mPaused = true; 549 } 550 } 551 552 /* 553 * returns true if we should request more data 554 */ 555 bool NuPlayer::Decoder::doRequestBuffers() { 556 if (isDiscontinuityPending()) { 557 return false; 558 } 559 status_t err = OK; 560 while (err == OK && !mDequeuedInputBuffers.empty()) { 561 size_t bufferIx = *mDequeuedInputBuffers.begin(); 562 sp<AMessage> msg = new AMessage(); 563 msg->setSize("buffer-ix", bufferIx); 564 err = fetchInputData(msg); 565 if (err != OK && err != ERROR_END_OF_STREAM) { 566 // if EOS, need to queue EOS buffer 567 break; 568 } 569 mDequeuedInputBuffers.erase(mDequeuedInputBuffers.begin()); 570 571 if (!mPendingInputMessages.empty() 572 || !onInputBufferFetched(msg)) { 573 mPendingInputMessages.push_back(msg); 574 } 575 } 576 577 return err == -EWOULDBLOCK 578 && mSource->feedMoreTSData() == OK; 579 } 580 581 void NuPlayer::Decoder::handleError(int32_t err) 582 { 583 // We cannot immediately release the codec due to buffers still outstanding 584 // in the renderer. We signal to the player the error so it can shutdown/release the 585 // decoder after flushing and increment the generation to discard unnecessary messages. 586 587 ++mBufferGeneration; 588 589 sp<AMessage> notify = mNotify->dup(); 590 notify->setInt32("what", kWhatError); 591 notify->setInt32("err", err); 592 notify->post(); 593 } 594 595 status_t NuPlayer::Decoder::releaseCrypto() 596 { 597 ALOGV("releaseCrypto"); 598 599 sp<AMessage> msg = new AMessage(kWhatDrmReleaseCrypto, this); 600 601 sp<AMessage> response; 602 status_t status = msg->postAndAwaitResponse(&response); 603 if (status == OK && response != NULL) { 604 CHECK(response->findInt32("status", &status)); 605 ALOGV("releaseCrypto ret: %d ", status); 606 } else { 607 ALOGE("releaseCrypto err: %d", status); 608 } 609 610 return status; 611 } 612 613 void NuPlayer::Decoder::onReleaseCrypto(const sp<AMessage>& msg) 614 { 615 status_t status = INVALID_OPERATION; 616 if (mCodec != NULL) { 617 status = mCodec->releaseCrypto(); 618 } else { 619 // returning OK if the codec has been already released 620 status = OK; 621 ALOGE("onReleaseCrypto No mCodec. err: %d", status); 622 } 623 624 sp<AMessage> response = new AMessage; 625 response->setInt32("status", status); 626 // Clearing the state as it's tied to crypto. mIsEncryptedObservedEarlier is sticky though 627 // and lasts for the lifetime of this codec. See its use in fetchInputData. 628 mIsEncrypted = false; 629 630 sp<AReplyToken> replyID; 631 CHECK(msg->senderAwaitsResponse(&replyID)); 632 response->postReply(replyID); 633 } 634 635 bool NuPlayer::Decoder::handleAnInputBuffer(size_t index) { 636 if (isDiscontinuityPending()) { 637 return false; 638 } 639 640 if (mCodec == NULL) { 641 ALOGE("[%s] handleAnInputBuffer without a valid codec", mComponentName.c_str()); 642 handleError(NO_INIT); 643 return false; 644 } 645 646 sp<MediaCodecBuffer> buffer; 647 mCodec->getInputBuffer(index, &buffer); 648 649 if (buffer == NULL) { 650 ALOGE("[%s] handleAnInputBuffer, failed to get input buffer", mComponentName.c_str()); 651 handleError(UNKNOWN_ERROR); 652 return false; 653 } 654 655 if (index >= mInputBuffers.size()) { 656 for (size_t i = mInputBuffers.size(); i <= index; ++i) { 657 mInputBuffers.add(); 658 mMediaBuffers.add(); 659 mInputBufferIsDequeued.add(); 660 mMediaBuffers.editItemAt(i) = NULL; 661 mInputBufferIsDequeued.editItemAt(i) = false; 662 } 663 } 664 mInputBuffers.editItemAt(index) = buffer; 665 666 //CHECK_LT(bufferIx, mInputBuffers.size()); 667 668 if (mMediaBuffers[index] != NULL) { 669 mMediaBuffers[index]->release(); 670 mMediaBuffers.editItemAt(index) = NULL; 671 } 672 mInputBufferIsDequeued.editItemAt(index) = true; 673 674 if (!mCSDsToSubmit.isEmpty()) { 675 sp<AMessage> msg = new AMessage(); 676 msg->setSize("buffer-ix", index); 677 678 sp<ABuffer> buffer = mCSDsToSubmit.itemAt(0); 679 ALOGI("[%s] resubmitting CSD", mComponentName.c_str()); 680 msg->setBuffer("buffer", buffer); 681 mCSDsToSubmit.removeAt(0); 682 if (!onInputBufferFetched(msg)) { 683 handleError(UNKNOWN_ERROR); 684 return false; 685 } 686 return true; 687 } 688 689 while (!mPendingInputMessages.empty()) { 690 sp<AMessage> msg = *mPendingInputMessages.begin(); 691 if (!onInputBufferFetched(msg)) { 692 break; 693 } 694 mPendingInputMessages.erase(mPendingInputMessages.begin()); 695 } 696 697 if (!mInputBufferIsDequeued.editItemAt(index)) { 698 return true; 699 } 700 701 mDequeuedInputBuffers.push_back(index); 702 703 onRequestInputBuffers(); 704 return true; 705 } 706 707 bool NuPlayer::Decoder::handleAnOutputBuffer( 708 size_t index, 709 size_t offset, 710 size_t size, 711 int64_t timeUs, 712 int32_t flags) { 713 if (mCodec == NULL) { 714 ALOGE("[%s] handleAnOutputBuffer without a valid codec", mComponentName.c_str()); 715 handleError(NO_INIT); 716 return false; 717 } 718 719 // CHECK_LT(bufferIx, mOutputBuffers.size()); 720 sp<MediaCodecBuffer> buffer; 721 mCodec->getOutputBuffer(index, &buffer); 722 723 if (buffer == NULL) { 724 ALOGE("[%s] handleAnOutputBuffer, failed to get output buffer", mComponentName.c_str()); 725 handleError(UNKNOWN_ERROR); 726 return false; 727 } 728 729 if (index >= mOutputBuffers.size()) { 730 for (size_t i = mOutputBuffers.size(); i <= index; ++i) { 731 mOutputBuffers.add(); 732 } 733 } 734 735 mOutputBuffers.editItemAt(index) = buffer; 736 737 buffer->setRange(offset, size); 738 buffer->meta()->clear(); 739 buffer->meta()->setInt64("timeUs", timeUs); 740 741 bool eos = flags & MediaCodec::BUFFER_FLAG_EOS; 742 // we do not expect CODECCONFIG or SYNCFRAME for decoder 743 744 sp<AMessage> reply = new AMessage(kWhatRenderBuffer, this); 745 reply->setSize("buffer-ix", index); 746 reply->setInt32("generation", mBufferGeneration); 747 748 if (eos) { 749 ALOGI("[%s] saw output EOS", mIsAudio ? "audio" : "video"); 750 751 buffer->meta()->setInt32("eos", true); 752 reply->setInt32("eos", true); 753 } else if (mSkipRenderingUntilMediaTimeUs >= 0) { 754 if (timeUs < mSkipRenderingUntilMediaTimeUs) { 755 ALOGV("[%s] dropping buffer at time %lld as requested.", 756 mComponentName.c_str(), (long long)timeUs); 757 758 reply->post(); 759 return true; 760 } 761 762 mSkipRenderingUntilMediaTimeUs = -1; 763 } 764 765 mNumFramesTotal += !mIsAudio; 766 767 // wait until 1st frame comes out to signal resume complete 768 notifyResumeCompleteIfNecessary(); 769 770 if (mRenderer != NULL) { 771 // send the buffer to renderer. 772 mRenderer->queueBuffer(mIsAudio, buffer, reply); 773 if (eos && !isDiscontinuityPending()) { 774 mRenderer->queueEOS(mIsAudio, ERROR_END_OF_STREAM); 775 } 776 } 777 778 return true; 779 } 780 781 void NuPlayer::Decoder::handleOutputFormatChange(const sp<AMessage> &format) { 782 if (!mIsAudio) { 783 int32_t width, height; 784 if (format->findInt32("width", &width) 785 && format->findInt32("height", &height)) { 786 mStats->setInt32("width", width); 787 mStats->setInt32("height", height); 788 } 789 sp<AMessage> notify = mNotify->dup(); 790 notify->setInt32("what", kWhatVideoSizeChanged); 791 notify->setMessage("format", format); 792 notify->post(); 793 } else if (mRenderer != NULL) { 794 uint32_t flags; 795 int64_t durationUs; 796 bool hasVideo = (mSource->getFormat(false /* audio */) != NULL); 797 if (getAudioDeepBufferSetting() // override regardless of source duration 798 || (mSource->getDuration(&durationUs) == OK 799 && durationUs > AUDIO_SINK_MIN_DEEP_BUFFER_DURATION_US)) { 800 flags = AUDIO_OUTPUT_FLAG_DEEP_BUFFER; 801 } else { 802 flags = AUDIO_OUTPUT_FLAG_NONE; 803 } 804 805 sp<AMessage> reply = new AMessage(kWhatAudioOutputFormatChanged, this); 806 reply->setInt32("generation", mBufferGeneration); 807 mRenderer->changeAudioFormat( 808 format, false /* offloadOnly */, hasVideo, 809 flags, mSource->isStreaming(), reply); 810 } 811 } 812 813 void NuPlayer::Decoder::releaseAndResetMediaBuffers() { 814 for (size_t i = 0; i < mMediaBuffers.size(); i++) { 815 if (mMediaBuffers[i] != NULL) { 816 mMediaBuffers[i]->release(); 817 mMediaBuffers.editItemAt(i) = NULL; 818 } 819 } 820 mMediaBuffers.resize(mInputBuffers.size()); 821 for (size_t i = 0; i < mMediaBuffers.size(); i++) { 822 mMediaBuffers.editItemAt(i) = NULL; 823 } 824 mInputBufferIsDequeued.clear(); 825 mInputBufferIsDequeued.resize(mInputBuffers.size()); 826 for (size_t i = 0; i < mInputBufferIsDequeued.size(); i++) { 827 mInputBufferIsDequeued.editItemAt(i) = false; 828 } 829 830 mPendingInputMessages.clear(); 831 mDequeuedInputBuffers.clear(); 832 mSkipRenderingUntilMediaTimeUs = -1; 833 } 834 835 void NuPlayer::Decoder::requestCodecNotification() { 836 if (mCodec != NULL) { 837 sp<AMessage> reply = new AMessage(kWhatCodecNotify, this); 838 reply->setInt32("generation", mBufferGeneration); 839 mCodec->requestActivityNotification(reply); 840 } 841 } 842 843 bool NuPlayer::Decoder::isStaleReply(const sp<AMessage> &msg) { 844 int32_t generation; 845 CHECK(msg->findInt32("generation", &generation)); 846 return generation != mBufferGeneration; 847 } 848 849 status_t NuPlayer::Decoder::fetchInputData(sp<AMessage> &reply) { 850 sp<ABuffer> accessUnit; 851 bool dropAccessUnit = true; 852 do { 853 status_t err = mSource->dequeueAccessUnit(mIsAudio, &accessUnit); 854 855 if (err == -EWOULDBLOCK) { 856 return err; 857 } else if (err != OK) { 858 if (err == INFO_DISCONTINUITY) { 859 int32_t type; 860 CHECK(accessUnit->meta()->findInt32("discontinuity", &type)); 861 862 bool formatChange = 863 (mIsAudio && 864 (type & ATSParser::DISCONTINUITY_AUDIO_FORMAT)) 865 || (!mIsAudio && 866 (type & ATSParser::DISCONTINUITY_VIDEO_FORMAT)); 867 868 bool timeChange = (type & ATSParser::DISCONTINUITY_TIME) != 0; 869 870 ALOGI("%s discontinuity (format=%d, time=%d)", 871 mIsAudio ? "audio" : "video", formatChange, timeChange); 872 873 bool seamlessFormatChange = false; 874 sp<AMessage> newFormat = mSource->getFormat(mIsAudio); 875 if (formatChange) { 876 seamlessFormatChange = 877 supportsSeamlessFormatChange(newFormat); 878 // treat seamless format change separately 879 formatChange = !seamlessFormatChange; 880 } 881 882 // For format or time change, return EOS to queue EOS input, 883 // then wait for EOS on output. 884 if (formatChange /* not seamless */) { 885 mFormatChangePending = true; 886 err = ERROR_END_OF_STREAM; 887 } else if (timeChange) { 888 rememberCodecSpecificData(newFormat); 889 mTimeChangePending = true; 890 err = ERROR_END_OF_STREAM; 891 } else if (seamlessFormatChange) { 892 // reuse existing decoder and don't flush 893 rememberCodecSpecificData(newFormat); 894 continue; 895 } else { 896 // This stream is unaffected by the discontinuity 897 return -EWOULDBLOCK; 898 } 899 } 900 901 // reply should only be returned without a buffer set 902 // when there is an error (including EOS) 903 CHECK(err != OK); 904 905 reply->setInt32("err", err); 906 return ERROR_END_OF_STREAM; 907 } 908 909 dropAccessUnit = false; 910 if (!mIsAudio && !mIsEncrypted) { 911 // Extra safeguard if higher-level behavior changes. Otherwise, not required now. 912 // Preventing the buffer from being processed (and sent to codec) if this is a later 913 // round of playback but this time without prepareDrm. Or if there is a race between 914 // stop (which is not blocking) and releaseDrm allowing buffers being processed after 915 // Crypto has been released (GenericSource currently prevents this race though). 916 // Particularly doing this check before IsAVCReferenceFrame call to prevent parsing 917 // of encrypted data. 918 if (mIsEncryptedObservedEarlier) { 919 ALOGE("fetchInputData: mismatched mIsEncrypted/mIsEncryptedObservedEarlier (0/1)"); 920 921 return INVALID_OPERATION; 922 } 923 924 int32_t layerId = 0; 925 bool haveLayerId = accessUnit->meta()->findInt32("temporal-layer-id", &layerId); 926 if (mRenderer->getVideoLateByUs() > 100000ll 927 && mIsVideoAVC 928 && !IsAVCReferenceFrame(accessUnit)) { 929 dropAccessUnit = true; 930 } else if (haveLayerId && mNumVideoTemporalLayerTotal > 1) { 931 // Add only one layer each time. 932 if (layerId > mCurrentMaxVideoTemporalLayerId + 1 933 || layerId >= mNumVideoTemporalLayerAllowed) { 934 dropAccessUnit = true; 935 ALOGV("dropping layer(%d), speed=%g, allowed layer count=%d, max layerId=%d", 936 layerId, mPlaybackSpeed, mNumVideoTemporalLayerAllowed, 937 mCurrentMaxVideoTemporalLayerId); 938 } else if (layerId > mCurrentMaxVideoTemporalLayerId) { 939 mCurrentMaxVideoTemporalLayerId = layerId; 940 } else if (layerId == 0 && mNumVideoTemporalLayerTotal > 1 && IsIDR(accessUnit)) { 941 mCurrentMaxVideoTemporalLayerId = mNumVideoTemporalLayerTotal - 1; 942 } 943 } 944 if (dropAccessUnit) { 945 if (layerId <= mCurrentMaxVideoTemporalLayerId && layerId > 0) { 946 mCurrentMaxVideoTemporalLayerId = layerId - 1; 947 } 948 ++mNumInputFramesDropped; 949 } 950 } 951 } while (dropAccessUnit); 952 953 // ALOGV("returned a valid buffer of %s data", mIsAudio ? "mIsAudio" : "video"); 954 #if 0 955 int64_t mediaTimeUs; 956 CHECK(accessUnit->meta()->findInt64("timeUs", &mediaTimeUs)); 957 ALOGV("[%s] feeding input buffer at media time %.3f", 958 mIsAudio ? "audio" : "video", 959 mediaTimeUs / 1E6); 960 #endif 961 962 if (mCCDecoder != NULL) { 963 mCCDecoder->decode(accessUnit); 964 } 965 966 reply->setBuffer("buffer", accessUnit); 967 968 return OK; 969 } 970 971 bool NuPlayer::Decoder::onInputBufferFetched(const sp<AMessage> &msg) { 972 if (mCodec == NULL) { 973 ALOGE("[%s] onInputBufferFetched without a valid codec", mComponentName.c_str()); 974 handleError(NO_INIT); 975 return false; 976 } 977 978 size_t bufferIx; 979 CHECK(msg->findSize("buffer-ix", &bufferIx)); 980 CHECK_LT(bufferIx, mInputBuffers.size()); 981 sp<MediaCodecBuffer> codecBuffer = mInputBuffers[bufferIx]; 982 983 sp<ABuffer> buffer; 984 bool hasBuffer = msg->findBuffer("buffer", &buffer); 985 bool needsCopy = true; 986 987 if (buffer == NULL /* includes !hasBuffer */) { 988 int32_t streamErr = ERROR_END_OF_STREAM; 989 CHECK(msg->findInt32("err", &streamErr) || !hasBuffer); 990 991 CHECK(streamErr != OK); 992 993 // attempt to queue EOS 994 status_t err = mCodec->queueInputBuffer( 995 bufferIx, 996 0, 997 0, 998 0, 999 MediaCodec::BUFFER_FLAG_EOS); 1000 if (err == OK) { 1001 mInputBufferIsDequeued.editItemAt(bufferIx) = false; 1002 } else if (streamErr == ERROR_END_OF_STREAM) { 1003 streamErr = err; 1004 // err will not be ERROR_END_OF_STREAM 1005 } 1006 1007 if (streamErr != ERROR_END_OF_STREAM) { 1008 ALOGE("Stream error for [%s] (err=%d), EOS %s queued", 1009 mComponentName.c_str(), 1010 streamErr, 1011 err == OK ? "successfully" : "unsuccessfully"); 1012 handleError(streamErr); 1013 } 1014 } else { 1015 sp<AMessage> extra; 1016 if (buffer->meta()->findMessage("extra", &extra) && extra != NULL) { 1017 int64_t resumeAtMediaTimeUs; 1018 if (extra->findInt64( 1019 "resume-at-mediaTimeUs", &resumeAtMediaTimeUs)) { 1020 ALOGI("[%s] suppressing rendering until %lld us", 1021 mComponentName.c_str(), (long long)resumeAtMediaTimeUs); 1022 mSkipRenderingUntilMediaTimeUs = resumeAtMediaTimeUs; 1023 } 1024 } 1025 1026 int64_t timeUs = 0; 1027 uint32_t flags = 0; 1028 CHECK(buffer->meta()->findInt64("timeUs", &timeUs)); 1029 1030 int32_t eos, csd; 1031 // we do not expect SYNCFRAME for decoder 1032 if (buffer->meta()->findInt32("eos", &eos) && eos) { 1033 flags |= MediaCodec::BUFFER_FLAG_EOS; 1034 } else if (buffer->meta()->findInt32("csd", &csd) && csd) { 1035 flags |= MediaCodec::BUFFER_FLAG_CODECCONFIG; 1036 } 1037 1038 // Modular DRM 1039 MediaBuffer *mediaBuf = NULL; 1040 NuPlayerDrm::CryptoInfo *cryptInfo = NULL; 1041 1042 // copy into codec buffer 1043 if (needsCopy) { 1044 if (buffer->size() > codecBuffer->capacity()) { 1045 handleError(ERROR_BUFFER_TOO_SMALL); 1046 mDequeuedInputBuffers.push_back(bufferIx); 1047 return false; 1048 } 1049 1050 if (buffer->data() != NULL) { 1051 codecBuffer->setRange(0, buffer->size()); 1052 memcpy(codecBuffer->data(), buffer->data(), buffer->size()); 1053 } else { // No buffer->data() 1054 //Modular DRM 1055 mediaBuf = (MediaBuffer*)buffer->getMediaBufferBase(); 1056 if (mediaBuf != NULL) { 1057 codecBuffer->setRange(0, mediaBuf->size()); 1058 memcpy(codecBuffer->data(), mediaBuf->data(), mediaBuf->size()); 1059 1060 sp<MetaData> meta_data = mediaBuf->meta_data(); 1061 cryptInfo = NuPlayerDrm::getSampleCryptoInfo(meta_data); 1062 1063 // since getMediaBuffer() has incremented the refCount 1064 mediaBuf->release(); 1065 } else { // No mediaBuf 1066 ALOGE("onInputBufferFetched: buffer->data()/mediaBuf are NULL for %p", 1067 buffer.get()); 1068 handleError(UNKNOWN_ERROR); 1069 return false; 1070 } 1071 } // buffer->data() 1072 } // needsCopy 1073 1074 status_t err; 1075 AString errorDetailMsg; 1076 if (cryptInfo != NULL) { 1077 err = mCodec->queueSecureInputBuffer( 1078 bufferIx, 1079 codecBuffer->offset(), 1080 cryptInfo->subSamples, 1081 cryptInfo->numSubSamples, 1082 cryptInfo->key, 1083 cryptInfo->iv, 1084 cryptInfo->mode, 1085 cryptInfo->pattern, 1086 timeUs, 1087 flags, 1088 &errorDetailMsg); 1089 // synchronous call so done with cryptInfo here 1090 free(cryptInfo); 1091 } else { 1092 err = mCodec->queueInputBuffer( 1093 bufferIx, 1094 codecBuffer->offset(), 1095 codecBuffer->size(), 1096 timeUs, 1097 flags, 1098 &errorDetailMsg); 1099 } // no cryptInfo 1100 1101 if (err != OK) { 1102 ALOGE("onInputBufferFetched: queue%sInputBuffer failed for [%s] (err=%d, %s)", 1103 (cryptInfo != NULL ? "Secure" : ""), 1104 mComponentName.c_str(), err, errorDetailMsg.c_str()); 1105 handleError(err); 1106 } else { 1107 mInputBufferIsDequeued.editItemAt(bufferIx) = false; 1108 } 1109 1110 } // buffer != NULL 1111 return true; 1112 } 1113 1114 void NuPlayer::Decoder::onRenderBuffer(const sp<AMessage> &msg) { 1115 status_t err; 1116 int32_t render; 1117 size_t bufferIx; 1118 int32_t eos; 1119 CHECK(msg->findSize("buffer-ix", &bufferIx)); 1120 1121 if (!mIsAudio) { 1122 int64_t timeUs; 1123 sp<MediaCodecBuffer> buffer = mOutputBuffers[bufferIx]; 1124 buffer->meta()->findInt64("timeUs", &timeUs); 1125 1126 if (mCCDecoder != NULL && mCCDecoder->isSelected()) { 1127 mCCDecoder->display(timeUs); 1128 } 1129 } 1130 1131 if (mCodec == NULL) { 1132 err = NO_INIT; 1133 } else if (msg->findInt32("render", &render) && render) { 1134 int64_t timestampNs; 1135 CHECK(msg->findInt64("timestampNs", ×tampNs)); 1136 err = mCodec->renderOutputBufferAndRelease(bufferIx, timestampNs); 1137 } else { 1138 mNumOutputFramesDropped += !mIsAudio; 1139 err = mCodec->releaseOutputBuffer(bufferIx); 1140 } 1141 if (err != OK) { 1142 ALOGE("failed to release output buffer for [%s] (err=%d)", 1143 mComponentName.c_str(), err); 1144 handleError(err); 1145 } 1146 if (msg->findInt32("eos", &eos) && eos 1147 && isDiscontinuityPending()) { 1148 finishHandleDiscontinuity(true /* flushOnTimeChange */); 1149 } 1150 } 1151 1152 bool NuPlayer::Decoder::isDiscontinuityPending() const { 1153 return mFormatChangePending || mTimeChangePending; 1154 } 1155 1156 void NuPlayer::Decoder::finishHandleDiscontinuity(bool flushOnTimeChange) { 1157 ALOGV("finishHandleDiscontinuity: format %d, time %d, flush %d", 1158 mFormatChangePending, mTimeChangePending, flushOnTimeChange); 1159 1160 // If we have format change, pause and wait to be killed; 1161 // If we have time change only, flush and restart fetching. 1162 1163 if (mFormatChangePending) { 1164 mPaused = true; 1165 } else if (mTimeChangePending) { 1166 if (flushOnTimeChange) { 1167 doFlush(false /* notifyComplete */); 1168 signalResume(false /* notifyComplete */); 1169 } 1170 } 1171 1172 // Notify NuPlayer to either shutdown decoder, or rescan sources 1173 sp<AMessage> msg = mNotify->dup(); 1174 msg->setInt32("what", kWhatInputDiscontinuity); 1175 msg->setInt32("formatChange", mFormatChangePending); 1176 msg->post(); 1177 1178 mFormatChangePending = false; 1179 mTimeChangePending = false; 1180 } 1181 1182 bool NuPlayer::Decoder::supportsSeamlessAudioFormatChange( 1183 const sp<AMessage> &targetFormat) const { 1184 if (targetFormat == NULL) { 1185 return true; 1186 } 1187 1188 AString mime; 1189 if (!targetFormat->findString("mime", &mime)) { 1190 return false; 1191 } 1192 1193 if (!strcasecmp(mime.c_str(), MEDIA_MIMETYPE_AUDIO_AAC)) { 1194 // field-by-field comparison 1195 const char * keys[] = { "channel-count", "sample-rate", "is-adts" }; 1196 for (unsigned int i = 0; i < sizeof(keys) / sizeof(keys[0]); i++) { 1197 int32_t oldVal, newVal; 1198 if (!mInputFormat->findInt32(keys[i], &oldVal) || 1199 !targetFormat->findInt32(keys[i], &newVal) || 1200 oldVal != newVal) { 1201 return false; 1202 } 1203 } 1204 1205 sp<ABuffer> oldBuf, newBuf; 1206 if (mInputFormat->findBuffer("csd-0", &oldBuf) && 1207 targetFormat->findBuffer("csd-0", &newBuf)) { 1208 if (oldBuf->size() != newBuf->size()) { 1209 return false; 1210 } 1211 return !memcmp(oldBuf->data(), newBuf->data(), oldBuf->size()); 1212 } 1213 } 1214 return false; 1215 } 1216 1217 bool NuPlayer::Decoder::supportsSeamlessFormatChange(const sp<AMessage> &targetFormat) const { 1218 if (mInputFormat == NULL) { 1219 return false; 1220 } 1221 1222 if (targetFormat == NULL) { 1223 return true; 1224 } 1225 1226 AString oldMime, newMime; 1227 if (!mInputFormat->findString("mime", &oldMime) 1228 || !targetFormat->findString("mime", &newMime) 1229 || !(oldMime == newMime)) { 1230 return false; 1231 } 1232 1233 bool audio = !strncasecmp(oldMime.c_str(), "audio/", strlen("audio/")); 1234 bool seamless; 1235 if (audio) { 1236 seamless = supportsSeamlessAudioFormatChange(targetFormat); 1237 } else { 1238 int32_t isAdaptive; 1239 seamless = (mCodec != NULL && 1240 mInputFormat->findInt32("adaptive-playback", &isAdaptive) && 1241 isAdaptive); 1242 } 1243 1244 ALOGV("%s seamless support for %s", seamless ? "yes" : "no", oldMime.c_str()); 1245 return seamless; 1246 } 1247 1248 void NuPlayer::Decoder::rememberCodecSpecificData(const sp<AMessage> &format) { 1249 if (format == NULL) { 1250 return; 1251 } 1252 mCSDsForCurrentFormat.clear(); 1253 for (int32_t i = 0; ; ++i) { 1254 AString tag = "csd-"; 1255 tag.append(i); 1256 sp<ABuffer> buffer; 1257 if (!format->findBuffer(tag.c_str(), &buffer)) { 1258 break; 1259 } 1260 mCSDsForCurrentFormat.push(buffer); 1261 } 1262 } 1263 1264 void NuPlayer::Decoder::notifyResumeCompleteIfNecessary() { 1265 if (mResumePending) { 1266 mResumePending = false; 1267 1268 sp<AMessage> notify = mNotify->dup(); 1269 notify->setInt32("what", kWhatResumeCompleted); 1270 notify->post(); 1271 } 1272 } 1273 1274 } // namespace android 1275 1276