1 /* 2 * Copyright 2014 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17 //#define LOG_NDEBUG 0 18 #define LOG_TAG "NuPlayerDecoder" 19 #include <utils/Log.h> 20 #include <inttypes.h> 21 22 #include <algorithm> 23 24 #include "NuPlayerCCDecoder.h" 25 #include "NuPlayerDecoder.h" 26 #include "NuPlayerDrm.h" 27 #include "NuPlayerRenderer.h" 28 #include "NuPlayerSource.h" 29 30 #include <cutils/properties.h> 31 #include <media/ICrypto.h> 32 #include <media/MediaCodecBuffer.h> 33 #include <media/stagefright/foundation/ABuffer.h> 34 #include <media/stagefright/foundation/ADebug.h> 35 #include <media/stagefright/foundation/AMessage.h> 36 #include <media/stagefright/MediaBuffer.h> 37 #include <media/stagefright/MediaCodec.h> 38 #include <media/stagefright/MediaDefs.h> 39 #include <media/stagefright/MediaErrors.h> 40 #include <media/stagefright/SurfaceUtils.h> 41 #include <gui/Surface.h> 42 43 #include "avc_utils.h" 44 #include "ATSParser.h" 45 46 namespace android { 47 48 static float kDisplayRefreshingRate = 60.f; // TODO: get this from the display 49 50 // The default total video frame rate of a stream when that info is not available from 51 // the source. 52 static float kDefaultVideoFrameRateTotal = 30.f; 53 54 static inline bool getAudioDeepBufferSetting() { 55 return property_get_bool("media.stagefright.audio.deep", false /* default_value */); 56 } 57 58 NuPlayer::Decoder::Decoder( 59 const sp<AMessage> ¬ify, 60 const sp<Source> &source, 61 pid_t pid, 62 uid_t uid, 63 const sp<Renderer> &renderer, 64 const sp<Surface> &surface, 65 const sp<CCDecoder> &ccDecoder) 66 : DecoderBase(notify), 67 mSurface(surface), 68 mSource(source), 69 mRenderer(renderer), 70 mCCDecoder(ccDecoder), 71 mPid(pid), 72 mUid(uid), 73 mSkipRenderingUntilMediaTimeUs(-1ll), 74 mNumFramesTotal(0ll), 75 mNumInputFramesDropped(0ll), 76 mNumOutputFramesDropped(0ll), 77 mVideoWidth(0), 78 mVideoHeight(0), 79 mIsAudio(true), 80 mIsVideoAVC(false), 81 mIsSecure(false), 82 mIsEncrypted(false), 83 mIsEncryptedObservedEarlier(false), 84 mFormatChangePending(false), 85 mTimeChangePending(false), 86 mFrameRateTotal(kDefaultVideoFrameRateTotal), 87 mPlaybackSpeed(1.0f), 88 mNumVideoTemporalLayerTotal(1), // decode all layers 89 mNumVideoTemporalLayerAllowed(1), 90 mCurrentMaxVideoTemporalLayerId(0), 91 mResumePending(false), 92 mComponentName("decoder") { 93 mCodecLooper = new ALooper; 94 mCodecLooper->setName("NPDecoder-CL"); 95 mCodecLooper->start(false, false, ANDROID_PRIORITY_AUDIO); 96 mVideoTemporalLayerAggregateFps[0] = mFrameRateTotal; 97 } 98 99 NuPlayer::Decoder::~Decoder() { 100 // Need to stop looper first since mCodec could be accessed on the mDecoderLooper. 101 stopLooper(); 102 if (mCodec != NULL) { 103 mCodec->release(); 104 } 105 releaseAndResetMediaBuffers(); 106 } 107 108 sp<AMessage> NuPlayer::Decoder::getStats() const { 109 mStats->setInt64("frames-total", mNumFramesTotal); 110 mStats->setInt64("frames-dropped-input", mNumInputFramesDropped); 111 mStats->setInt64("frames-dropped-output", mNumOutputFramesDropped); 112 return mStats; 113 } 114 115 status_t NuPlayer::Decoder::setVideoSurface(const sp<Surface> &surface) { 116 if (surface == NULL || ADebug::isExperimentEnabled("legacy-setsurface")) { 117 return BAD_VALUE; 118 } 119 120 sp<AMessage> msg = new AMessage(kWhatSetVideoSurface, this); 121 122 msg->setObject("surface", surface); 123 sp<AMessage> response; 124 status_t err = msg->postAndAwaitResponse(&response); 125 if (err == OK && response != NULL) { 126 CHECK(response->findInt32("err", &err)); 127 } 128 return err; 129 } 130 131 void NuPlayer::Decoder::onMessageReceived(const sp<AMessage> &msg) { 132 ALOGV("[%s] onMessage: %s", mComponentName.c_str(), msg->debugString().c_str()); 133 134 switch (msg->what()) { 135 case kWhatCodecNotify: 136 { 137 int32_t cbID; 138 CHECK(msg->findInt32("callbackID", &cbID)); 139 140 ALOGV("[%s] kWhatCodecNotify: cbID = %d, paused = %d", 141 mIsAudio ? "audio" : "video", cbID, mPaused); 142 143 if (mPaused) { 144 break; 145 } 146 147 switch (cbID) { 148 case MediaCodec::CB_INPUT_AVAILABLE: 149 { 150 int32_t index; 151 CHECK(msg->findInt32("index", &index)); 152 153 handleAnInputBuffer(index); 154 break; 155 } 156 157 case MediaCodec::CB_OUTPUT_AVAILABLE: 158 { 159 int32_t index; 160 size_t offset; 161 size_t size; 162 int64_t timeUs; 163 int32_t flags; 164 165 CHECK(msg->findInt32("index", &index)); 166 CHECK(msg->findSize("offset", &offset)); 167 CHECK(msg->findSize("size", &size)); 168 CHECK(msg->findInt64("timeUs", &timeUs)); 169 CHECK(msg->findInt32("flags", &flags)); 170 171 handleAnOutputBuffer(index, offset, size, timeUs, flags); 172 break; 173 } 174 175 case MediaCodec::CB_OUTPUT_FORMAT_CHANGED: 176 { 177 sp<AMessage> format; 178 CHECK(msg->findMessage("format", &format)); 179 180 handleOutputFormatChange(format); 181 break; 182 } 183 184 case MediaCodec::CB_ERROR: 185 { 186 status_t err; 187 CHECK(msg->findInt32("err", &err)); 188 ALOGE("Decoder (%s) reported error : 0x%x", 189 mIsAudio ? "audio" : "video", err); 190 191 handleError(err); 192 break; 193 } 194 195 default: 196 { 197 TRESPASS(); 198 break; 199 } 200 } 201 202 break; 203 } 204 205 case kWhatRenderBuffer: 206 { 207 if (!isStaleReply(msg)) { 208 onRenderBuffer(msg); 209 } 210 break; 211 } 212 213 case kWhatAudioOutputFormatChanged: 214 { 215 if (!isStaleReply(msg)) { 216 status_t err; 217 if (msg->findInt32("err", &err) && err != OK) { 218 ALOGE("Renderer reported 0x%x when changing audio output format", err); 219 handleError(err); 220 } 221 } 222 break; 223 } 224 225 case kWhatSetVideoSurface: 226 { 227 sp<AReplyToken> replyID; 228 CHECK(msg->senderAwaitsResponse(&replyID)); 229 230 sp<RefBase> obj; 231 CHECK(msg->findObject("surface", &obj)); 232 sp<Surface> surface = static_cast<Surface *>(obj.get()); // non-null 233 int32_t err = INVALID_OPERATION; 234 // NOTE: in practice mSurface is always non-null, but checking here for completeness 235 if (mCodec != NULL && mSurface != NULL) { 236 // TODO: once AwesomePlayer is removed, remove this automatic connecting 237 // to the surface by MediaPlayerService. 238 // 239 // at this point MediaPlayerService::client has already connected to the 240 // surface, which MediaCodec does not expect 241 err = nativeWindowDisconnect(surface.get(), "kWhatSetVideoSurface(surface)"); 242 if (err == OK) { 243 err = mCodec->setSurface(surface); 244 ALOGI_IF(err, "codec setSurface returned: %d", err); 245 if (err == OK) { 246 // reconnect to the old surface as MPS::Client will expect to 247 // be able to disconnect from it. 248 (void)nativeWindowConnect(mSurface.get(), "kWhatSetVideoSurface(mSurface)"); 249 mSurface = surface; 250 } 251 } 252 if (err != OK) { 253 // reconnect to the new surface on error as MPS::Client will expect to 254 // be able to disconnect from it. 255 (void)nativeWindowConnect(surface.get(), "kWhatSetVideoSurface(err)"); 256 } 257 } 258 259 sp<AMessage> response = new AMessage; 260 response->setInt32("err", err); 261 response->postReply(replyID); 262 break; 263 } 264 265 case kWhatDrmReleaseCrypto: 266 { 267 ALOGV("kWhatDrmReleaseCrypto"); 268 onReleaseCrypto(msg); 269 break; 270 } 271 272 default: 273 DecoderBase::onMessageReceived(msg); 274 break; 275 } 276 } 277 278 void NuPlayer::Decoder::onConfigure(const sp<AMessage> &format) { 279 CHECK(mCodec == NULL); 280 281 mFormatChangePending = false; 282 mTimeChangePending = false; 283 284 ++mBufferGeneration; 285 286 AString mime; 287 CHECK(format->findString("mime", &mime)); 288 289 mIsAudio = !strncasecmp("audio/", mime.c_str(), 6); 290 mIsVideoAVC = !strcasecmp(MEDIA_MIMETYPE_VIDEO_AVC, mime.c_str()); 291 292 mComponentName = mime; 293 mComponentName.append(" decoder"); 294 ALOGV("[%s] onConfigure (surface=%p)", mComponentName.c_str(), mSurface.get()); 295 296 mCodec = MediaCodec::CreateByType( 297 mCodecLooper, mime.c_str(), false /* encoder */, NULL /* err */, mPid, mUid); 298 int32_t secure = 0; 299 if (format->findInt32("secure", &secure) && secure != 0) { 300 if (mCodec != NULL) { 301 mCodec->getName(&mComponentName); 302 mComponentName.append(".secure"); 303 mCodec->release(); 304 ALOGI("[%s] creating", mComponentName.c_str()); 305 mCodec = MediaCodec::CreateByComponentName( 306 mCodecLooper, mComponentName.c_str(), NULL /* err */, mPid, mUid); 307 } 308 } 309 if (mCodec == NULL) { 310 ALOGE("Failed to create %s%s decoder", 311 (secure ? "secure " : ""), mime.c_str()); 312 handleError(UNKNOWN_ERROR); 313 return; 314 } 315 mIsSecure = secure; 316 317 mCodec->getName(&mComponentName); 318 319 status_t err; 320 if (mSurface != NULL) { 321 // disconnect from surface as MediaCodec will reconnect 322 err = nativeWindowDisconnect(mSurface.get(), "onConfigure"); 323 // We treat this as a warning, as this is a preparatory step. 324 // Codec will try to connect to the surface, which is where 325 // any error signaling will occur. 326 ALOGW_IF(err != OK, "failed to disconnect from surface: %d", err); 327 } 328 329 // Modular DRM 330 void *pCrypto; 331 if (!format->findPointer("crypto", &pCrypto)) { 332 pCrypto = NULL; 333 } 334 sp<ICrypto> crypto = (ICrypto*)pCrypto; 335 // non-encrypted source won't have a crypto 336 mIsEncrypted = (crypto != NULL); 337 // configure is called once; still using OR in case the behavior changes. 338 mIsEncryptedObservedEarlier = mIsEncryptedObservedEarlier || mIsEncrypted; 339 ALOGV("onConfigure mCrypto: %p (%d) mIsSecure: %d", 340 crypto.get(), (crypto != NULL ? crypto->getStrongCount() : 0), mIsSecure); 341 342 err = mCodec->configure( 343 format, mSurface, crypto, 0 /* flags */); 344 345 if (err != OK) { 346 ALOGE("Failed to configure %s decoder (err=%d)", mComponentName.c_str(), err); 347 mCodec->release(); 348 mCodec.clear(); 349 handleError(err); 350 return; 351 } 352 rememberCodecSpecificData(format); 353 354 // the following should work in configured state 355 CHECK_EQ((status_t)OK, mCodec->getOutputFormat(&mOutputFormat)); 356 CHECK_EQ((status_t)OK, mCodec->getInputFormat(&mInputFormat)); 357 358 mStats->setString("mime", mime.c_str()); 359 mStats->setString("component-name", mComponentName.c_str()); 360 361 if (!mIsAudio) { 362 int32_t width, height; 363 if (mOutputFormat->findInt32("width", &width) 364 && mOutputFormat->findInt32("height", &height)) { 365 mStats->setInt32("width", width); 366 mStats->setInt32("height", height); 367 } 368 } 369 370 sp<AMessage> reply = new AMessage(kWhatCodecNotify, this); 371 mCodec->setCallback(reply); 372 373 err = mCodec->start(); 374 if (err != OK) { 375 ALOGE("Failed to start %s decoder (err=%d)", mComponentName.c_str(), err); 376 mCodec->release(); 377 mCodec.clear(); 378 handleError(err); 379 return; 380 } 381 382 releaseAndResetMediaBuffers(); 383 384 mPaused = false; 385 mResumePending = false; 386 } 387 388 void NuPlayer::Decoder::onSetParameters(const sp<AMessage> ¶ms) { 389 bool needAdjustLayers = false; 390 float frameRateTotal; 391 if (params->findFloat("frame-rate-total", &frameRateTotal) 392 && mFrameRateTotal != frameRateTotal) { 393 needAdjustLayers = true; 394 mFrameRateTotal = frameRateTotal; 395 } 396 397 int32_t numVideoTemporalLayerTotal; 398 if (params->findInt32("temporal-layer-count", &numVideoTemporalLayerTotal) 399 && numVideoTemporalLayerTotal >= 0 400 && numVideoTemporalLayerTotal <= kMaxNumVideoTemporalLayers 401 && mNumVideoTemporalLayerTotal != numVideoTemporalLayerTotal) { 402 needAdjustLayers = true; 403 mNumVideoTemporalLayerTotal = std::max(numVideoTemporalLayerTotal, 1); 404 } 405 406 if (needAdjustLayers && mNumVideoTemporalLayerTotal > 1) { 407 // TODO: For now, layer fps is calculated for some specific architectures. 408 // But it really should be extracted from the stream. 409 mVideoTemporalLayerAggregateFps[0] = 410 mFrameRateTotal / (float)(1ll << (mNumVideoTemporalLayerTotal - 1)); 411 for (int32_t i = 1; i < mNumVideoTemporalLayerTotal; ++i) { 412 mVideoTemporalLayerAggregateFps[i] = 413 mFrameRateTotal / (float)(1ll << (mNumVideoTemporalLayerTotal - i)) 414 + mVideoTemporalLayerAggregateFps[i - 1]; 415 } 416 } 417 418 float playbackSpeed; 419 if (params->findFloat("playback-speed", &playbackSpeed) 420 && mPlaybackSpeed != playbackSpeed) { 421 needAdjustLayers = true; 422 mPlaybackSpeed = playbackSpeed; 423 } 424 425 if (needAdjustLayers) { 426 float decodeFrameRate = mFrameRateTotal; 427 // enable temporal layering optimization only if we know the layering depth 428 if (mNumVideoTemporalLayerTotal > 1) { 429 int32_t layerId; 430 for (layerId = 0; layerId < mNumVideoTemporalLayerTotal - 1; ++layerId) { 431 if (mVideoTemporalLayerAggregateFps[layerId] * mPlaybackSpeed 432 >= kDisplayRefreshingRate * 0.9) { 433 break; 434 } 435 } 436 mNumVideoTemporalLayerAllowed = layerId + 1; 437 decodeFrameRate = mVideoTemporalLayerAggregateFps[layerId]; 438 } 439 ALOGV("onSetParameters: allowed layers=%d, decodeFps=%g", 440 mNumVideoTemporalLayerAllowed, decodeFrameRate); 441 442 if (mCodec == NULL) { 443 ALOGW("onSetParameters called before codec is created."); 444 return; 445 } 446 447 sp<AMessage> codecParams = new AMessage(); 448 codecParams->setFloat("operating-rate", decodeFrameRate * mPlaybackSpeed); 449 mCodec->setParameters(codecParams); 450 } 451 } 452 453 void NuPlayer::Decoder::onSetRenderer(const sp<Renderer> &renderer) { 454 mRenderer = renderer; 455 } 456 457 void NuPlayer::Decoder::onResume(bool notifyComplete) { 458 mPaused = false; 459 460 if (notifyComplete) { 461 mResumePending = true; 462 } 463 mCodec->start(); 464 } 465 466 void NuPlayer::Decoder::doFlush(bool notifyComplete) { 467 if (mCCDecoder != NULL) { 468 mCCDecoder->flush(); 469 } 470 471 if (mRenderer != NULL) { 472 mRenderer->flush(mIsAudio, notifyComplete); 473 mRenderer->signalTimeDiscontinuity(); 474 } 475 476 status_t err = OK; 477 if (mCodec != NULL) { 478 err = mCodec->flush(); 479 mCSDsToSubmit = mCSDsForCurrentFormat; // copy operator 480 ++mBufferGeneration; 481 } 482 483 if (err != OK) { 484 ALOGE("failed to flush %s (err=%d)", mComponentName.c_str(), err); 485 handleError(err); 486 // finish with posting kWhatFlushCompleted. 487 // we attempt to release the buffers even if flush fails. 488 } 489 releaseAndResetMediaBuffers(); 490 mPaused = true; 491 } 492 493 494 void NuPlayer::Decoder::onFlush() { 495 doFlush(true); 496 497 if (isDiscontinuityPending()) { 498 // This could happen if the client starts seeking/shutdown 499 // after we queued an EOS for discontinuities. 500 // We can consider discontinuity handled. 501 finishHandleDiscontinuity(false /* flushOnTimeChange */); 502 } 503 504 sp<AMessage> notify = mNotify->dup(); 505 notify->setInt32("what", kWhatFlushCompleted); 506 notify->post(); 507 } 508 509 void NuPlayer::Decoder::onShutdown(bool notifyComplete) { 510 status_t err = OK; 511 512 // if there is a pending resume request, notify complete now 513 notifyResumeCompleteIfNecessary(); 514 515 if (mCodec != NULL) { 516 err = mCodec->release(); 517 mCodec = NULL; 518 ++mBufferGeneration; 519 520 if (mSurface != NULL) { 521 // reconnect to surface as MediaCodec disconnected from it 522 status_t error = nativeWindowConnect(mSurface.get(), "onShutdown"); 523 ALOGW_IF(error != NO_ERROR, 524 "[%s] failed to connect to native window, error=%d", 525 mComponentName.c_str(), error); 526 } 527 mComponentName = "decoder"; 528 } 529 530 releaseAndResetMediaBuffers(); 531 532 if (err != OK) { 533 ALOGE("failed to release %s (err=%d)", mComponentName.c_str(), err); 534 handleError(err); 535 // finish with posting kWhatShutdownCompleted. 536 } 537 538 if (notifyComplete) { 539 sp<AMessage> notify = mNotify->dup(); 540 notify->setInt32("what", kWhatShutdownCompleted); 541 notify->post(); 542 mPaused = true; 543 } 544 } 545 546 /* 547 * returns true if we should request more data 548 */ 549 bool NuPlayer::Decoder::doRequestBuffers() { 550 if (isDiscontinuityPending()) { 551 return false; 552 } 553 status_t err = OK; 554 while (err == OK && !mDequeuedInputBuffers.empty()) { 555 size_t bufferIx = *mDequeuedInputBuffers.begin(); 556 sp<AMessage> msg = new AMessage(); 557 msg->setSize("buffer-ix", bufferIx); 558 err = fetchInputData(msg); 559 if (err != OK && err != ERROR_END_OF_STREAM) { 560 // if EOS, need to queue EOS buffer 561 break; 562 } 563 mDequeuedInputBuffers.erase(mDequeuedInputBuffers.begin()); 564 565 if (!mPendingInputMessages.empty() 566 || !onInputBufferFetched(msg)) { 567 mPendingInputMessages.push_back(msg); 568 } 569 } 570 571 return err == -EWOULDBLOCK 572 && mSource->feedMoreTSData() == OK; 573 } 574 575 void NuPlayer::Decoder::handleError(int32_t err) 576 { 577 // We cannot immediately release the codec due to buffers still outstanding 578 // in the renderer. We signal to the player the error so it can shutdown/release the 579 // decoder after flushing and increment the generation to discard unnecessary messages. 580 581 ++mBufferGeneration; 582 583 sp<AMessage> notify = mNotify->dup(); 584 notify->setInt32("what", kWhatError); 585 notify->setInt32("err", err); 586 notify->post(); 587 } 588 589 status_t NuPlayer::Decoder::releaseCrypto() 590 { 591 ALOGV("releaseCrypto"); 592 593 sp<AMessage> msg = new AMessage(kWhatDrmReleaseCrypto, this); 594 595 sp<AMessage> response; 596 status_t status = msg->postAndAwaitResponse(&response); 597 if (status == OK && response != NULL) { 598 CHECK(response->findInt32("status", &status)); 599 ALOGV("releaseCrypto ret: %d ", status); 600 } else { 601 ALOGE("releaseCrypto err: %d", status); 602 } 603 604 return status; 605 } 606 607 void NuPlayer::Decoder::onReleaseCrypto(const sp<AMessage>& msg) 608 { 609 status_t status = INVALID_OPERATION; 610 if (mCodec != NULL) { 611 status = mCodec->releaseCrypto(); 612 } else { 613 // returning OK if the codec has been already released 614 status = OK; 615 ALOGE("onReleaseCrypto No mCodec. err: %d", status); 616 } 617 618 sp<AMessage> response = new AMessage; 619 response->setInt32("status", status); 620 // Clearing the state as it's tied to crypto. mIsEncryptedObservedEarlier is sticky though 621 // and lasts for the lifetime of this codec. See its use in fetchInputData. 622 mIsEncrypted = false; 623 624 sp<AReplyToken> replyID; 625 CHECK(msg->senderAwaitsResponse(&replyID)); 626 response->postReply(replyID); 627 } 628 629 bool NuPlayer::Decoder::handleAnInputBuffer(size_t index) { 630 if (isDiscontinuityPending()) { 631 return false; 632 } 633 634 sp<MediaCodecBuffer> buffer; 635 mCodec->getInputBuffer(index, &buffer); 636 637 if (buffer == NULL) { 638 handleError(UNKNOWN_ERROR); 639 return false; 640 } 641 642 if (index >= mInputBuffers.size()) { 643 for (size_t i = mInputBuffers.size(); i <= index; ++i) { 644 mInputBuffers.add(); 645 mMediaBuffers.add(); 646 mInputBufferIsDequeued.add(); 647 mMediaBuffers.editItemAt(i) = NULL; 648 mInputBufferIsDequeued.editItemAt(i) = false; 649 } 650 } 651 mInputBuffers.editItemAt(index) = buffer; 652 653 //CHECK_LT(bufferIx, mInputBuffers.size()); 654 655 if (mMediaBuffers[index] != NULL) { 656 mMediaBuffers[index]->release(); 657 mMediaBuffers.editItemAt(index) = NULL; 658 } 659 mInputBufferIsDequeued.editItemAt(index) = true; 660 661 if (!mCSDsToSubmit.isEmpty()) { 662 sp<AMessage> msg = new AMessage(); 663 msg->setSize("buffer-ix", index); 664 665 sp<ABuffer> buffer = mCSDsToSubmit.itemAt(0); 666 ALOGI("[%s] resubmitting CSD", mComponentName.c_str()); 667 msg->setBuffer("buffer", buffer); 668 mCSDsToSubmit.removeAt(0); 669 if (!onInputBufferFetched(msg)) { 670 handleError(UNKNOWN_ERROR); 671 return false; 672 } 673 return true; 674 } 675 676 while (!mPendingInputMessages.empty()) { 677 sp<AMessage> msg = *mPendingInputMessages.begin(); 678 if (!onInputBufferFetched(msg)) { 679 break; 680 } 681 mPendingInputMessages.erase(mPendingInputMessages.begin()); 682 } 683 684 if (!mInputBufferIsDequeued.editItemAt(index)) { 685 return true; 686 } 687 688 mDequeuedInputBuffers.push_back(index); 689 690 onRequestInputBuffers(); 691 return true; 692 } 693 694 bool NuPlayer::Decoder::handleAnOutputBuffer( 695 size_t index, 696 size_t offset, 697 size_t size, 698 int64_t timeUs, 699 int32_t flags) { 700 // CHECK_LT(bufferIx, mOutputBuffers.size()); 701 sp<MediaCodecBuffer> buffer; 702 mCodec->getOutputBuffer(index, &buffer); 703 704 if (buffer == NULL) { 705 handleError(UNKNOWN_ERROR); 706 return false; 707 } 708 709 if (index >= mOutputBuffers.size()) { 710 for (size_t i = mOutputBuffers.size(); i <= index; ++i) { 711 mOutputBuffers.add(); 712 } 713 } 714 715 mOutputBuffers.editItemAt(index) = buffer; 716 717 buffer->setRange(offset, size); 718 buffer->meta()->clear(); 719 buffer->meta()->setInt64("timeUs", timeUs); 720 721 bool eos = flags & MediaCodec::BUFFER_FLAG_EOS; 722 // we do not expect CODECCONFIG or SYNCFRAME for decoder 723 724 sp<AMessage> reply = new AMessage(kWhatRenderBuffer, this); 725 reply->setSize("buffer-ix", index); 726 reply->setInt32("generation", mBufferGeneration); 727 728 if (eos) { 729 ALOGI("[%s] saw output EOS", mIsAudio ? "audio" : "video"); 730 731 buffer->meta()->setInt32("eos", true); 732 reply->setInt32("eos", true); 733 } else if (mSkipRenderingUntilMediaTimeUs >= 0) { 734 if (timeUs < mSkipRenderingUntilMediaTimeUs) { 735 ALOGV("[%s] dropping buffer at time %lld as requested.", 736 mComponentName.c_str(), (long long)timeUs); 737 738 reply->post(); 739 return true; 740 } 741 742 mSkipRenderingUntilMediaTimeUs = -1; 743 } 744 745 mNumFramesTotal += !mIsAudio; 746 747 // wait until 1st frame comes out to signal resume complete 748 notifyResumeCompleteIfNecessary(); 749 750 if (mRenderer != NULL) { 751 // send the buffer to renderer. 752 mRenderer->queueBuffer(mIsAudio, buffer, reply); 753 if (eos && !isDiscontinuityPending()) { 754 mRenderer->queueEOS(mIsAudio, ERROR_END_OF_STREAM); 755 } 756 } 757 758 return true; 759 } 760 761 void NuPlayer::Decoder::handleOutputFormatChange(const sp<AMessage> &format) { 762 if (!mIsAudio) { 763 int32_t width, height; 764 if (format->findInt32("width", &width) 765 && format->findInt32("height", &height)) { 766 mStats->setInt32("width", width); 767 mStats->setInt32("height", height); 768 } 769 sp<AMessage> notify = mNotify->dup(); 770 notify->setInt32("what", kWhatVideoSizeChanged); 771 notify->setMessage("format", format); 772 notify->post(); 773 } else if (mRenderer != NULL) { 774 uint32_t flags; 775 int64_t durationUs; 776 bool hasVideo = (mSource->getFormat(false /* audio */) != NULL); 777 if (getAudioDeepBufferSetting() // override regardless of source duration 778 || (mSource->getDuration(&durationUs) == OK 779 && durationUs > AUDIO_SINK_MIN_DEEP_BUFFER_DURATION_US)) { 780 flags = AUDIO_OUTPUT_FLAG_DEEP_BUFFER; 781 } else { 782 flags = AUDIO_OUTPUT_FLAG_NONE; 783 } 784 785 sp<AMessage> reply = new AMessage(kWhatAudioOutputFormatChanged, this); 786 reply->setInt32("generation", mBufferGeneration); 787 mRenderer->changeAudioFormat( 788 format, false /* offloadOnly */, hasVideo, 789 flags, mSource->isStreaming(), reply); 790 } 791 } 792 793 void NuPlayer::Decoder::releaseAndResetMediaBuffers() { 794 for (size_t i = 0; i < mMediaBuffers.size(); i++) { 795 if (mMediaBuffers[i] != NULL) { 796 mMediaBuffers[i]->release(); 797 mMediaBuffers.editItemAt(i) = NULL; 798 } 799 } 800 mMediaBuffers.resize(mInputBuffers.size()); 801 for (size_t i = 0; i < mMediaBuffers.size(); i++) { 802 mMediaBuffers.editItemAt(i) = NULL; 803 } 804 mInputBufferIsDequeued.clear(); 805 mInputBufferIsDequeued.resize(mInputBuffers.size()); 806 for (size_t i = 0; i < mInputBufferIsDequeued.size(); i++) { 807 mInputBufferIsDequeued.editItemAt(i) = false; 808 } 809 810 mPendingInputMessages.clear(); 811 mDequeuedInputBuffers.clear(); 812 mSkipRenderingUntilMediaTimeUs = -1; 813 } 814 815 void NuPlayer::Decoder::requestCodecNotification() { 816 if (mCodec != NULL) { 817 sp<AMessage> reply = new AMessage(kWhatCodecNotify, this); 818 reply->setInt32("generation", mBufferGeneration); 819 mCodec->requestActivityNotification(reply); 820 } 821 } 822 823 bool NuPlayer::Decoder::isStaleReply(const sp<AMessage> &msg) { 824 int32_t generation; 825 CHECK(msg->findInt32("generation", &generation)); 826 return generation != mBufferGeneration; 827 } 828 829 status_t NuPlayer::Decoder::fetchInputData(sp<AMessage> &reply) { 830 sp<ABuffer> accessUnit; 831 bool dropAccessUnit = true; 832 do { 833 status_t err = mSource->dequeueAccessUnit(mIsAudio, &accessUnit); 834 835 if (err == -EWOULDBLOCK) { 836 return err; 837 } else if (err != OK) { 838 if (err == INFO_DISCONTINUITY) { 839 int32_t type; 840 CHECK(accessUnit->meta()->findInt32("discontinuity", &type)); 841 842 bool formatChange = 843 (mIsAudio && 844 (type & ATSParser::DISCONTINUITY_AUDIO_FORMAT)) 845 || (!mIsAudio && 846 (type & ATSParser::DISCONTINUITY_VIDEO_FORMAT)); 847 848 bool timeChange = (type & ATSParser::DISCONTINUITY_TIME) != 0; 849 850 ALOGI("%s discontinuity (format=%d, time=%d)", 851 mIsAudio ? "audio" : "video", formatChange, timeChange); 852 853 bool seamlessFormatChange = false; 854 sp<AMessage> newFormat = mSource->getFormat(mIsAudio); 855 if (formatChange) { 856 seamlessFormatChange = 857 supportsSeamlessFormatChange(newFormat); 858 // treat seamless format change separately 859 formatChange = !seamlessFormatChange; 860 } 861 862 // For format or time change, return EOS to queue EOS input, 863 // then wait for EOS on output. 864 if (formatChange /* not seamless */) { 865 mFormatChangePending = true; 866 err = ERROR_END_OF_STREAM; 867 } else if (timeChange) { 868 rememberCodecSpecificData(newFormat); 869 mTimeChangePending = true; 870 err = ERROR_END_OF_STREAM; 871 } else if (seamlessFormatChange) { 872 // reuse existing decoder and don't flush 873 rememberCodecSpecificData(newFormat); 874 continue; 875 } else { 876 // This stream is unaffected by the discontinuity 877 return -EWOULDBLOCK; 878 } 879 } 880 881 // reply should only be returned without a buffer set 882 // when there is an error (including EOS) 883 CHECK(err != OK); 884 885 reply->setInt32("err", err); 886 return ERROR_END_OF_STREAM; 887 } 888 889 dropAccessUnit = false; 890 if (!mIsAudio && !mIsEncrypted) { 891 // Extra safeguard if higher-level behavior changes. Otherwise, not required now. 892 // Preventing the buffer from being processed (and sent to codec) if this is a later 893 // round of playback but this time without prepareDrm. Or if there is a race between 894 // stop (which is not blocking) and releaseDrm allowing buffers being processed after 895 // Crypto has been released (GenericSource currently prevents this race though). 896 // Particularly doing this check before IsAVCReferenceFrame call to prevent parsing 897 // of encrypted data. 898 if (mIsEncryptedObservedEarlier) { 899 ALOGE("fetchInputData: mismatched mIsEncrypted/mIsEncryptedObservedEarlier (0/1)"); 900 901 return INVALID_OPERATION; 902 } 903 904 int32_t layerId = 0; 905 bool haveLayerId = accessUnit->meta()->findInt32("temporal-layer-id", &layerId); 906 if (mRenderer->getVideoLateByUs() > 100000ll 907 && mIsVideoAVC 908 && !IsAVCReferenceFrame(accessUnit)) { 909 dropAccessUnit = true; 910 } else if (haveLayerId && mNumVideoTemporalLayerTotal > 1) { 911 // Add only one layer each time. 912 if (layerId > mCurrentMaxVideoTemporalLayerId + 1 913 || layerId >= mNumVideoTemporalLayerAllowed) { 914 dropAccessUnit = true; 915 ALOGV("dropping layer(%d), speed=%g, allowed layer count=%d, max layerId=%d", 916 layerId, mPlaybackSpeed, mNumVideoTemporalLayerAllowed, 917 mCurrentMaxVideoTemporalLayerId); 918 } else if (layerId > mCurrentMaxVideoTemporalLayerId) { 919 mCurrentMaxVideoTemporalLayerId = layerId; 920 } else if (layerId == 0 && mNumVideoTemporalLayerTotal > 1 && IsIDR(accessUnit)) { 921 mCurrentMaxVideoTemporalLayerId = mNumVideoTemporalLayerTotal - 1; 922 } 923 } 924 if (dropAccessUnit) { 925 if (layerId <= mCurrentMaxVideoTemporalLayerId && layerId > 0) { 926 mCurrentMaxVideoTemporalLayerId = layerId - 1; 927 } 928 ++mNumInputFramesDropped; 929 } 930 } 931 } while (dropAccessUnit); 932 933 // ALOGV("returned a valid buffer of %s data", mIsAudio ? "mIsAudio" : "video"); 934 #if 0 935 int64_t mediaTimeUs; 936 CHECK(accessUnit->meta()->findInt64("timeUs", &mediaTimeUs)); 937 ALOGV("[%s] feeding input buffer at media time %.3f", 938 mIsAudio ? "audio" : "video", 939 mediaTimeUs / 1E6); 940 #endif 941 942 if (mCCDecoder != NULL) { 943 mCCDecoder->decode(accessUnit); 944 } 945 946 reply->setBuffer("buffer", accessUnit); 947 948 return OK; 949 } 950 951 bool NuPlayer::Decoder::onInputBufferFetched(const sp<AMessage> &msg) { 952 size_t bufferIx; 953 CHECK(msg->findSize("buffer-ix", &bufferIx)); 954 CHECK_LT(bufferIx, mInputBuffers.size()); 955 sp<MediaCodecBuffer> codecBuffer = mInputBuffers[bufferIx]; 956 957 sp<ABuffer> buffer; 958 bool hasBuffer = msg->findBuffer("buffer", &buffer); 959 bool needsCopy = true; 960 961 if (buffer == NULL /* includes !hasBuffer */) { 962 int32_t streamErr = ERROR_END_OF_STREAM; 963 CHECK(msg->findInt32("err", &streamErr) || !hasBuffer); 964 965 CHECK(streamErr != OK); 966 967 // attempt to queue EOS 968 status_t err = mCodec->queueInputBuffer( 969 bufferIx, 970 0, 971 0, 972 0, 973 MediaCodec::BUFFER_FLAG_EOS); 974 if (err == OK) { 975 mInputBufferIsDequeued.editItemAt(bufferIx) = false; 976 } else if (streamErr == ERROR_END_OF_STREAM) { 977 streamErr = err; 978 // err will not be ERROR_END_OF_STREAM 979 } 980 981 if (streamErr != ERROR_END_OF_STREAM) { 982 ALOGE("Stream error for %s (err=%d), EOS %s queued", 983 mComponentName.c_str(), 984 streamErr, 985 err == OK ? "successfully" : "unsuccessfully"); 986 handleError(streamErr); 987 } 988 } else { 989 sp<AMessage> extra; 990 if (buffer->meta()->findMessage("extra", &extra) && extra != NULL) { 991 int64_t resumeAtMediaTimeUs; 992 if (extra->findInt64( 993 "resume-at-mediaTimeUs", &resumeAtMediaTimeUs)) { 994 ALOGI("[%s] suppressing rendering until %lld us", 995 mComponentName.c_str(), (long long)resumeAtMediaTimeUs); 996 mSkipRenderingUntilMediaTimeUs = resumeAtMediaTimeUs; 997 } 998 } 999 1000 int64_t timeUs = 0; 1001 uint32_t flags = 0; 1002 CHECK(buffer->meta()->findInt64("timeUs", &timeUs)); 1003 1004 int32_t eos, csd; 1005 // we do not expect SYNCFRAME for decoder 1006 if (buffer->meta()->findInt32("eos", &eos) && eos) { 1007 flags |= MediaCodec::BUFFER_FLAG_EOS; 1008 } else if (buffer->meta()->findInt32("csd", &csd) && csd) { 1009 flags |= MediaCodec::BUFFER_FLAG_CODECCONFIG; 1010 } 1011 1012 // Modular DRM 1013 MediaBuffer *mediaBuf = NULL; 1014 NuPlayerDrm::CryptoInfo *cryptInfo = NULL; 1015 1016 // copy into codec buffer 1017 if (needsCopy) { 1018 if (buffer->size() > codecBuffer->capacity()) { 1019 handleError(ERROR_BUFFER_TOO_SMALL); 1020 mDequeuedInputBuffers.push_back(bufferIx); 1021 return false; 1022 } 1023 1024 if (buffer->data() != NULL) { 1025 codecBuffer->setRange(0, buffer->size()); 1026 memcpy(codecBuffer->data(), buffer->data(), buffer->size()); 1027 } else { // No buffer->data() 1028 //Modular DRM 1029 mediaBuf = (MediaBuffer*)buffer->getMediaBufferBase(); 1030 if (mediaBuf != NULL) { 1031 codecBuffer->setRange(0, mediaBuf->size()); 1032 memcpy(codecBuffer->data(), mediaBuf->data(), mediaBuf->size()); 1033 1034 sp<MetaData> meta_data = mediaBuf->meta_data(); 1035 cryptInfo = NuPlayerDrm::getSampleCryptoInfo(meta_data); 1036 1037 // since getMediaBuffer() has incremented the refCount 1038 mediaBuf->release(); 1039 } else { // No mediaBuf 1040 ALOGE("onInputBufferFetched: buffer->data()/mediaBuf are NULL for %p", 1041 buffer.get()); 1042 handleError(UNKNOWN_ERROR); 1043 return false; 1044 } 1045 } // buffer->data() 1046 } // needsCopy 1047 1048 status_t err; 1049 AString errorDetailMsg; 1050 if (cryptInfo != NULL) { 1051 err = mCodec->queueSecureInputBuffer( 1052 bufferIx, 1053 codecBuffer->offset(), 1054 cryptInfo->subSamples, 1055 cryptInfo->numSubSamples, 1056 cryptInfo->key, 1057 cryptInfo->iv, 1058 cryptInfo->mode, 1059 cryptInfo->pattern, 1060 timeUs, 1061 flags, 1062 &errorDetailMsg); 1063 // synchronous call so done with cryptInfo here 1064 free(cryptInfo); 1065 } else { 1066 err = mCodec->queueInputBuffer( 1067 bufferIx, 1068 codecBuffer->offset(), 1069 codecBuffer->size(), 1070 timeUs, 1071 flags, 1072 &errorDetailMsg); 1073 } // no cryptInfo 1074 1075 if (err != OK) { 1076 ALOGE("onInputBufferFetched: queue%sInputBuffer failed for %s (err=%d, %s)", 1077 (cryptInfo != NULL ? "Secure" : ""), 1078 mComponentName.c_str(), err, errorDetailMsg.c_str()); 1079 handleError(err); 1080 } else { 1081 mInputBufferIsDequeued.editItemAt(bufferIx) = false; 1082 } 1083 1084 } // buffer != NULL 1085 return true; 1086 } 1087 1088 void NuPlayer::Decoder::onRenderBuffer(const sp<AMessage> &msg) { 1089 status_t err; 1090 int32_t render; 1091 size_t bufferIx; 1092 int32_t eos; 1093 CHECK(msg->findSize("buffer-ix", &bufferIx)); 1094 1095 if (!mIsAudio) { 1096 int64_t timeUs; 1097 sp<MediaCodecBuffer> buffer = mOutputBuffers[bufferIx]; 1098 buffer->meta()->findInt64("timeUs", &timeUs); 1099 1100 if (mCCDecoder != NULL && mCCDecoder->isSelected()) { 1101 mCCDecoder->display(timeUs); 1102 } 1103 } 1104 1105 if (msg->findInt32("render", &render) && render) { 1106 int64_t timestampNs; 1107 CHECK(msg->findInt64("timestampNs", ×tampNs)); 1108 err = mCodec->renderOutputBufferAndRelease(bufferIx, timestampNs); 1109 } else { 1110 mNumOutputFramesDropped += !mIsAudio; 1111 err = mCodec->releaseOutputBuffer(bufferIx); 1112 } 1113 if (err != OK) { 1114 ALOGE("failed to release output buffer for %s (err=%d)", 1115 mComponentName.c_str(), err); 1116 handleError(err); 1117 } 1118 if (msg->findInt32("eos", &eos) && eos 1119 && isDiscontinuityPending()) { 1120 finishHandleDiscontinuity(true /* flushOnTimeChange */); 1121 } 1122 } 1123 1124 bool NuPlayer::Decoder::isDiscontinuityPending() const { 1125 return mFormatChangePending || mTimeChangePending; 1126 } 1127 1128 void NuPlayer::Decoder::finishHandleDiscontinuity(bool flushOnTimeChange) { 1129 ALOGV("finishHandleDiscontinuity: format %d, time %d, flush %d", 1130 mFormatChangePending, mTimeChangePending, flushOnTimeChange); 1131 1132 // If we have format change, pause and wait to be killed; 1133 // If we have time change only, flush and restart fetching. 1134 1135 if (mFormatChangePending) { 1136 mPaused = true; 1137 } else if (mTimeChangePending) { 1138 if (flushOnTimeChange) { 1139 doFlush(false /* notifyComplete */); 1140 signalResume(false /* notifyComplete */); 1141 } 1142 } 1143 1144 // Notify NuPlayer to either shutdown decoder, or rescan sources 1145 sp<AMessage> msg = mNotify->dup(); 1146 msg->setInt32("what", kWhatInputDiscontinuity); 1147 msg->setInt32("formatChange", mFormatChangePending); 1148 msg->post(); 1149 1150 mFormatChangePending = false; 1151 mTimeChangePending = false; 1152 } 1153 1154 bool NuPlayer::Decoder::supportsSeamlessAudioFormatChange( 1155 const sp<AMessage> &targetFormat) const { 1156 if (targetFormat == NULL) { 1157 return true; 1158 } 1159 1160 AString mime; 1161 if (!targetFormat->findString("mime", &mime)) { 1162 return false; 1163 } 1164 1165 if (!strcasecmp(mime.c_str(), MEDIA_MIMETYPE_AUDIO_AAC)) { 1166 // field-by-field comparison 1167 const char * keys[] = { "channel-count", "sample-rate", "is-adts" }; 1168 for (unsigned int i = 0; i < sizeof(keys) / sizeof(keys[0]); i++) { 1169 int32_t oldVal, newVal; 1170 if (!mInputFormat->findInt32(keys[i], &oldVal) || 1171 !targetFormat->findInt32(keys[i], &newVal) || 1172 oldVal != newVal) { 1173 return false; 1174 } 1175 } 1176 1177 sp<ABuffer> oldBuf, newBuf; 1178 if (mInputFormat->findBuffer("csd-0", &oldBuf) && 1179 targetFormat->findBuffer("csd-0", &newBuf)) { 1180 if (oldBuf->size() != newBuf->size()) { 1181 return false; 1182 } 1183 return !memcmp(oldBuf->data(), newBuf->data(), oldBuf->size()); 1184 } 1185 } 1186 return false; 1187 } 1188 1189 bool NuPlayer::Decoder::supportsSeamlessFormatChange(const sp<AMessage> &targetFormat) const { 1190 if (mInputFormat == NULL) { 1191 return false; 1192 } 1193 1194 if (targetFormat == NULL) { 1195 return true; 1196 } 1197 1198 AString oldMime, newMime; 1199 if (!mInputFormat->findString("mime", &oldMime) 1200 || !targetFormat->findString("mime", &newMime) 1201 || !(oldMime == newMime)) { 1202 return false; 1203 } 1204 1205 bool audio = !strncasecmp(oldMime.c_str(), "audio/", strlen("audio/")); 1206 bool seamless; 1207 if (audio) { 1208 seamless = supportsSeamlessAudioFormatChange(targetFormat); 1209 } else { 1210 int32_t isAdaptive; 1211 seamless = (mCodec != NULL && 1212 mInputFormat->findInt32("adaptive-playback", &isAdaptive) && 1213 isAdaptive); 1214 } 1215 1216 ALOGV("%s seamless support for %s", seamless ? "yes" : "no", oldMime.c_str()); 1217 return seamless; 1218 } 1219 1220 void NuPlayer::Decoder::rememberCodecSpecificData(const sp<AMessage> &format) { 1221 if (format == NULL) { 1222 return; 1223 } 1224 mCSDsForCurrentFormat.clear(); 1225 for (int32_t i = 0; ; ++i) { 1226 AString tag = "csd-"; 1227 tag.append(i); 1228 sp<ABuffer> buffer; 1229 if (!format->findBuffer(tag.c_str(), &buffer)) { 1230 break; 1231 } 1232 mCSDsForCurrentFormat.push(buffer); 1233 } 1234 } 1235 1236 void NuPlayer::Decoder::notifyResumeCompleteIfNecessary() { 1237 if (mResumePending) { 1238 mResumePending = false; 1239 1240 sp<AMessage> notify = mNotify->dup(); 1241 notify->setInt32("what", kWhatResumeCompleted); 1242 notify->post(); 1243 } 1244 } 1245 1246 } // namespace android 1247 1248
