1 /* 2 * Copyright 2014 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17 //#define LOG_NDEBUG 0 18 #define LOG_TAG "NuPlayerDecoder" 19 #include <utils/Log.h> 20 #include <inttypes.h> 21 22 #include <algorithm> 23 24 #include "NuPlayerCCDecoder.h" 25 #include "NuPlayerDecoder.h" 26 #include "NuPlayerRenderer.h" 27 #include "NuPlayerSource.h" 28 29 #include <cutils/properties.h> 30 #include <media/ICrypto.h> 31 #include <media/stagefright/foundation/ABuffer.h> 32 #include <media/stagefright/foundation/ADebug.h> 33 #include <media/stagefright/foundation/AMessage.h> 34 #include <media/stagefright/MediaBuffer.h> 35 #include <media/stagefright/MediaCodec.h> 36 #include <media/stagefright/MediaDefs.h> 37 #include <media/stagefright/MediaErrors.h> 38 39 #include <gui/Surface.h> 40 41 #include "avc_utils.h" 42 #include "ATSParser.h" 43 44 namespace android { 45 46 static float kDisplayRefreshingRate = 60.f; // TODO: get this from the display 47 48 // The default total video frame rate of a stream when that info is not available from 49 // the source. 50 static float kDefaultVideoFrameRateTotal = 30.f; 51 52 static inline bool getAudioDeepBufferSetting() { 53 return property_get_bool("media.stagefright.audio.deep", false /* default_value */); 54 } 55 56 NuPlayer::Decoder::Decoder( 57 const sp<AMessage> ¬ify, 58 const sp<Source> &source, 59 pid_t pid, 60 const sp<Renderer> &renderer, 61 const sp<Surface> &surface, 62 const sp<CCDecoder> &ccDecoder) 63 : DecoderBase(notify), 64 mSurface(surface), 65 mSource(source), 66 mRenderer(renderer), 67 mCCDecoder(ccDecoder), 68 mPid(pid), 69 mSkipRenderingUntilMediaTimeUs(-1ll), 70 mNumFramesTotal(0ll), 71 mNumInputFramesDropped(0ll), 72 mNumOutputFramesDropped(0ll), 73 mVideoWidth(0), 74 mVideoHeight(0), 75 mIsAudio(true), 76 mIsVideoAVC(false), 77 mIsSecure(false), 78 mFormatChangePending(false), 79 mTimeChangePending(false), 80 mFrameRateTotal(kDefaultVideoFrameRateTotal), 81 mPlaybackSpeed(1.0f), 82 mNumVideoTemporalLayerTotal(1), // decode all layers 83 mNumVideoTemporalLayerAllowed(1), 84 mCurrentMaxVideoTemporalLayerId(0), 85 mResumePending(false), 86 mComponentName("decoder") { 87 mCodecLooper = new ALooper; 88 mCodecLooper->setName("NPDecoder-CL"); 89 mCodecLooper->start(false, false, ANDROID_PRIORITY_AUDIO); 90 mVideoTemporalLayerAggregateFps[0] = mFrameRateTotal; 91 } 92 93 NuPlayer::Decoder::~Decoder() { 94 mCodec->release(); 95 releaseAndResetMediaBuffers(); 96 } 97 98 sp<AMessage> NuPlayer::Decoder::getStats() const { 99 mStats->setInt64("frames-total", mNumFramesTotal); 100 mStats->setInt64("frames-dropped-input", mNumInputFramesDropped); 101 mStats->setInt64("frames-dropped-output", mNumOutputFramesDropped); 102 return mStats; 103 } 104 105 status_t NuPlayer::Decoder::setVideoSurface(const sp<Surface> &surface) { 106 if (surface == NULL || ADebug::isExperimentEnabled("legacy-setsurface")) { 107 return BAD_VALUE; 108 } 109 110 sp<AMessage> msg = new AMessage(kWhatSetVideoSurface, this); 111 112 msg->setObject("surface", surface); 113 sp<AMessage> response; 114 status_t err = msg->postAndAwaitResponse(&response); 115 if (err == OK && response != NULL) { 116 CHECK(response->findInt32("err", &err)); 117 } 118 return err; 119 } 120 121 void NuPlayer::Decoder::onMessageReceived(const sp<AMessage> &msg) { 122 ALOGV("[%s] onMessage: %s", mComponentName.c_str(), msg->debugString().c_str()); 123 124 switch (msg->what()) { 125 case kWhatCodecNotify: 126 { 127 int32_t cbID; 128 CHECK(msg->findInt32("callbackID", &cbID)); 129 130 ALOGV("[%s] kWhatCodecNotify: cbID = %d, paused = %d", 131 mIsAudio ? "audio" : "video", cbID, mPaused); 132 133 if (mPaused) { 134 break; 135 } 136 137 switch (cbID) { 138 case MediaCodec::CB_INPUT_AVAILABLE: 139 { 140 int32_t index; 141 CHECK(msg->findInt32("index", &index)); 142 143 handleAnInputBuffer(index); 144 break; 145 } 146 147 case MediaCodec::CB_OUTPUT_AVAILABLE: 148 { 149 int32_t index; 150 size_t offset; 151 size_t size; 152 int64_t timeUs; 153 int32_t flags; 154 155 CHECK(msg->findInt32("index", &index)); 156 CHECK(msg->findSize("offset", &offset)); 157 CHECK(msg->findSize("size", &size)); 158 CHECK(msg->findInt64("timeUs", &timeUs)); 159 CHECK(msg->findInt32("flags", &flags)); 160 161 handleAnOutputBuffer(index, offset, size, timeUs, flags); 162 break; 163 } 164 165 case MediaCodec::CB_OUTPUT_FORMAT_CHANGED: 166 { 167 sp<AMessage> format; 168 CHECK(msg->findMessage("format", &format)); 169 170 handleOutputFormatChange(format); 171 break; 172 } 173 174 case MediaCodec::CB_ERROR: 175 { 176 status_t err; 177 CHECK(msg->findInt32("err", &err)); 178 ALOGE("Decoder (%s) reported error : 0x%x", 179 mIsAudio ? "audio" : "video", err); 180 181 handleError(err); 182 break; 183 } 184 185 default: 186 { 187 TRESPASS(); 188 break; 189 } 190 } 191 192 break; 193 } 194 195 case kWhatRenderBuffer: 196 { 197 if (!isStaleReply(msg)) { 198 onRenderBuffer(msg); 199 } 200 break; 201 } 202 203 case kWhatSetVideoSurface: 204 { 205 sp<AReplyToken> replyID; 206 CHECK(msg->senderAwaitsResponse(&replyID)); 207 208 sp<RefBase> obj; 209 CHECK(msg->findObject("surface", &obj)); 210 sp<Surface> surface = static_cast<Surface *>(obj.get()); // non-null 211 int32_t err = INVALID_OPERATION; 212 // NOTE: in practice mSurface is always non-null, but checking here for completeness 213 if (mCodec != NULL && mSurface != NULL) { 214 // TODO: once AwesomePlayer is removed, remove this automatic connecting 215 // to the surface by MediaPlayerService. 216 // 217 // at this point MediaPlayerService::client has already connected to the 218 // surface, which MediaCodec does not expect 219 err = native_window_api_disconnect(surface.get(), NATIVE_WINDOW_API_MEDIA); 220 if (err == OK) { 221 err = mCodec->setSurface(surface); 222 ALOGI_IF(err, "codec setSurface returned: %d", err); 223 if (err == OK) { 224 // reconnect to the old surface as MPS::Client will expect to 225 // be able to disconnect from it. 226 (void)native_window_api_connect(mSurface.get(), NATIVE_WINDOW_API_MEDIA); 227 mSurface = surface; 228 } 229 } 230 if (err != OK) { 231 // reconnect to the new surface on error as MPS::Client will expect to 232 // be able to disconnect from it. 233 (void)native_window_api_connect(surface.get(), NATIVE_WINDOW_API_MEDIA); 234 } 235 } 236 237 sp<AMessage> response = new AMessage; 238 response->setInt32("err", err); 239 response->postReply(replyID); 240 break; 241 } 242 243 default: 244 DecoderBase::onMessageReceived(msg); 245 break; 246 } 247 } 248 249 void NuPlayer::Decoder::onConfigure(const sp<AMessage> &format) { 250 CHECK(mCodec == NULL); 251 252 mFormatChangePending = false; 253 mTimeChangePending = false; 254 255 ++mBufferGeneration; 256 257 AString mime; 258 CHECK(format->findString("mime", &mime)); 259 260 mIsAudio = !strncasecmp("audio/", mime.c_str(), 6); 261 mIsVideoAVC = !strcasecmp(MEDIA_MIMETYPE_VIDEO_AVC, mime.c_str()); 262 263 mComponentName = mime; 264 mComponentName.append(" decoder"); 265 ALOGV("[%s] onConfigure (surface=%p)", mComponentName.c_str(), mSurface.get()); 266 267 mCodec = MediaCodec::CreateByType( 268 mCodecLooper, mime.c_str(), false /* encoder */, NULL /* err */, mPid); 269 int32_t secure = 0; 270 if (format->findInt32("secure", &secure) && secure != 0) { 271 if (mCodec != NULL) { 272 mCodec->getName(&mComponentName); 273 mComponentName.append(".secure"); 274 mCodec->release(); 275 ALOGI("[%s] creating", mComponentName.c_str()); 276 mCodec = MediaCodec::CreateByComponentName( 277 mCodecLooper, mComponentName.c_str(), NULL /* err */, mPid); 278 } 279 } 280 if (mCodec == NULL) { 281 ALOGE("Failed to create %s%s decoder", 282 (secure ? "secure " : ""), mime.c_str()); 283 handleError(UNKNOWN_ERROR); 284 return; 285 } 286 mIsSecure = secure; 287 288 mCodec->getName(&mComponentName); 289 290 status_t err; 291 if (mSurface != NULL) { 292 // disconnect from surface as MediaCodec will reconnect 293 err = native_window_api_disconnect( 294 mSurface.get(), NATIVE_WINDOW_API_MEDIA); 295 // We treat this as a warning, as this is a preparatory step. 296 // Codec will try to connect to the surface, which is where 297 // any error signaling will occur. 298 ALOGW_IF(err != OK, "failed to disconnect from surface: %d", err); 299 } 300 err = mCodec->configure( 301 format, mSurface, NULL /* crypto */, 0 /* flags */); 302 if (err != OK) { 303 ALOGE("Failed to configure %s decoder (err=%d)", mComponentName.c_str(), err); 304 mCodec->release(); 305 mCodec.clear(); 306 handleError(err); 307 return; 308 } 309 rememberCodecSpecificData(format); 310 311 // the following should work in configured state 312 CHECK_EQ((status_t)OK, mCodec->getOutputFormat(&mOutputFormat)); 313 CHECK_EQ((status_t)OK, mCodec->getInputFormat(&mInputFormat)); 314 315 mStats->setString("mime", mime.c_str()); 316 mStats->setString("component-name", mComponentName.c_str()); 317 318 if (!mIsAudio) { 319 int32_t width, height; 320 if (mOutputFormat->findInt32("width", &width) 321 && mOutputFormat->findInt32("height", &height)) { 322 mStats->setInt32("width", width); 323 mStats->setInt32("height", height); 324 } 325 } 326 327 sp<AMessage> reply = new AMessage(kWhatCodecNotify, this); 328 mCodec->setCallback(reply); 329 330 err = mCodec->start(); 331 if (err != OK) { 332 ALOGE("Failed to start %s decoder (err=%d)", mComponentName.c_str(), err); 333 mCodec->release(); 334 mCodec.clear(); 335 handleError(err); 336 return; 337 } 338 339 releaseAndResetMediaBuffers(); 340 341 mPaused = false; 342 mResumePending = false; 343 } 344 345 void NuPlayer::Decoder::onSetParameters(const sp<AMessage> ¶ms) { 346 bool needAdjustLayers = false; 347 float frameRateTotal; 348 if (params->findFloat("frame-rate-total", &frameRateTotal) 349 && mFrameRateTotal != frameRateTotal) { 350 needAdjustLayers = true; 351 mFrameRateTotal = frameRateTotal; 352 } 353 354 int32_t numVideoTemporalLayerTotal; 355 if (params->findInt32("temporal-layer-count", &numVideoTemporalLayerTotal) 356 && numVideoTemporalLayerTotal >= 0 357 && numVideoTemporalLayerTotal <= kMaxNumVideoTemporalLayers 358 && mNumVideoTemporalLayerTotal != numVideoTemporalLayerTotal) { 359 needAdjustLayers = true; 360 mNumVideoTemporalLayerTotal = std::max(numVideoTemporalLayerTotal, 1); 361 } 362 363 if (needAdjustLayers && mNumVideoTemporalLayerTotal > 1) { 364 // TODO: For now, layer fps is calculated for some specific architectures. 365 // But it really should be extracted from the stream. 366 mVideoTemporalLayerAggregateFps[0] = 367 mFrameRateTotal / (float)(1ll << (mNumVideoTemporalLayerTotal - 1)); 368 for (int32_t i = 1; i < mNumVideoTemporalLayerTotal; ++i) { 369 mVideoTemporalLayerAggregateFps[i] = 370 mFrameRateTotal / (float)(1ll << (mNumVideoTemporalLayerTotal - i)) 371 + mVideoTemporalLayerAggregateFps[i - 1]; 372 } 373 } 374 375 float playbackSpeed; 376 if (params->findFloat("playback-speed", &playbackSpeed) 377 && mPlaybackSpeed != playbackSpeed) { 378 needAdjustLayers = true; 379 mPlaybackSpeed = playbackSpeed; 380 } 381 382 if (needAdjustLayers) { 383 float decodeFrameRate = mFrameRateTotal; 384 // enable temporal layering optimization only if we know the layering depth 385 if (mNumVideoTemporalLayerTotal > 1) { 386 int32_t layerId; 387 for (layerId = 0; layerId < mNumVideoTemporalLayerTotal - 1; ++layerId) { 388 if (mVideoTemporalLayerAggregateFps[layerId] * mPlaybackSpeed 389 >= kDisplayRefreshingRate * 0.9) { 390 break; 391 } 392 } 393 mNumVideoTemporalLayerAllowed = layerId + 1; 394 decodeFrameRate = mVideoTemporalLayerAggregateFps[layerId]; 395 } 396 ALOGV("onSetParameters: allowed layers=%d, decodeFps=%g", 397 mNumVideoTemporalLayerAllowed, decodeFrameRate); 398 399 if (mCodec == NULL) { 400 ALOGW("onSetParameters called before codec is created."); 401 return; 402 } 403 404 sp<AMessage> codecParams = new AMessage(); 405 codecParams->setFloat("operating-rate", decodeFrameRate * mPlaybackSpeed); 406 mCodec->setParameters(codecParams); 407 } 408 } 409 410 void NuPlayer::Decoder::onSetRenderer(const sp<Renderer> &renderer) { 411 bool hadNoRenderer = (mRenderer == NULL); 412 mRenderer = renderer; 413 if (hadNoRenderer && mRenderer != NULL) { 414 // this means that the widevine legacy source is ready 415 onRequestInputBuffers(); 416 } 417 } 418 419 void NuPlayer::Decoder::onGetInputBuffers( 420 Vector<sp<ABuffer> > *dstBuffers) { 421 CHECK_EQ((status_t)OK, mCodec->getWidevineLegacyBuffers(dstBuffers)); 422 } 423 424 void NuPlayer::Decoder::onResume(bool notifyComplete) { 425 mPaused = false; 426 427 if (notifyComplete) { 428 mResumePending = true; 429 } 430 mCodec->start(); 431 } 432 433 void NuPlayer::Decoder::doFlush(bool notifyComplete) { 434 if (mCCDecoder != NULL) { 435 mCCDecoder->flush(); 436 } 437 438 if (mRenderer != NULL) { 439 mRenderer->flush(mIsAudio, notifyComplete); 440 mRenderer->signalTimeDiscontinuity(); 441 } 442 443 status_t err = OK; 444 if (mCodec != NULL) { 445 err = mCodec->flush(); 446 mCSDsToSubmit = mCSDsForCurrentFormat; // copy operator 447 ++mBufferGeneration; 448 } 449 450 if (err != OK) { 451 ALOGE("failed to flush %s (err=%d)", mComponentName.c_str(), err); 452 handleError(err); 453 // finish with posting kWhatFlushCompleted. 454 // we attempt to release the buffers even if flush fails. 455 } 456 releaseAndResetMediaBuffers(); 457 mPaused = true; 458 } 459 460 461 void NuPlayer::Decoder::onFlush() { 462 doFlush(true); 463 464 if (isDiscontinuityPending()) { 465 // This could happen if the client starts seeking/shutdown 466 // after we queued an EOS for discontinuities. 467 // We can consider discontinuity handled. 468 finishHandleDiscontinuity(false /* flushOnTimeChange */); 469 } 470 471 sp<AMessage> notify = mNotify->dup(); 472 notify->setInt32("what", kWhatFlushCompleted); 473 notify->post(); 474 } 475 476 void NuPlayer::Decoder::onShutdown(bool notifyComplete) { 477 status_t err = OK; 478 479 // if there is a pending resume request, notify complete now 480 notifyResumeCompleteIfNecessary(); 481 482 if (mCodec != NULL) { 483 err = mCodec->release(); 484 mCodec = NULL; 485 ++mBufferGeneration; 486 487 if (mSurface != NULL) { 488 // reconnect to surface as MediaCodec disconnected from it 489 status_t error = 490 native_window_api_connect(mSurface.get(), NATIVE_WINDOW_API_MEDIA); 491 ALOGW_IF(error != NO_ERROR, 492 "[%s] failed to connect to native window, error=%d", 493 mComponentName.c_str(), error); 494 } 495 mComponentName = "decoder"; 496 } 497 498 releaseAndResetMediaBuffers(); 499 500 if (err != OK) { 501 ALOGE("failed to release %s (err=%d)", mComponentName.c_str(), err); 502 handleError(err); 503 // finish with posting kWhatShutdownCompleted. 504 } 505 506 if (notifyComplete) { 507 sp<AMessage> notify = mNotify->dup(); 508 notify->setInt32("what", kWhatShutdownCompleted); 509 notify->post(); 510 mPaused = true; 511 } 512 } 513 514 /* 515 * returns true if we should request more data 516 */ 517 bool NuPlayer::Decoder::doRequestBuffers() { 518 // mRenderer is only NULL if we have a legacy widevine source that 519 // is not yet ready. In this case we must not fetch input. 520 if (isDiscontinuityPending() || mRenderer == NULL) { 521 return false; 522 } 523 status_t err = OK; 524 while (err == OK && !mDequeuedInputBuffers.empty()) { 525 size_t bufferIx = *mDequeuedInputBuffers.begin(); 526 sp<AMessage> msg = new AMessage(); 527 msg->setSize("buffer-ix", bufferIx); 528 err = fetchInputData(msg); 529 if (err != OK && err != ERROR_END_OF_STREAM) { 530 // if EOS, need to queue EOS buffer 531 break; 532 } 533 mDequeuedInputBuffers.erase(mDequeuedInputBuffers.begin()); 534 535 if (!mPendingInputMessages.empty() 536 || !onInputBufferFetched(msg)) { 537 mPendingInputMessages.push_back(msg); 538 } 539 } 540 541 return err == -EWOULDBLOCK 542 && mSource->feedMoreTSData() == OK; 543 } 544 545 void NuPlayer::Decoder::handleError(int32_t err) 546 { 547 // We cannot immediately release the codec due to buffers still outstanding 548 // in the renderer. We signal to the player the error so it can shutdown/release the 549 // decoder after flushing and increment the generation to discard unnecessary messages. 550 551 ++mBufferGeneration; 552 553 sp<AMessage> notify = mNotify->dup(); 554 notify->setInt32("what", kWhatError); 555 notify->setInt32("err", err); 556 notify->post(); 557 } 558 559 bool NuPlayer::Decoder::handleAnInputBuffer(size_t index) { 560 if (isDiscontinuityPending()) { 561 return false; 562 } 563 564 sp<ABuffer> buffer; 565 mCodec->getInputBuffer(index, &buffer); 566 567 if (buffer == NULL) { 568 handleError(UNKNOWN_ERROR); 569 return false; 570 } 571 572 if (index >= mInputBuffers.size()) { 573 for (size_t i = mInputBuffers.size(); i <= index; ++i) { 574 mInputBuffers.add(); 575 mMediaBuffers.add(); 576 mInputBufferIsDequeued.add(); 577 mMediaBuffers.editItemAt(i) = NULL; 578 mInputBufferIsDequeued.editItemAt(i) = false; 579 } 580 } 581 mInputBuffers.editItemAt(index) = buffer; 582 583 //CHECK_LT(bufferIx, mInputBuffers.size()); 584 585 if (mMediaBuffers[index] != NULL) { 586 mMediaBuffers[index]->release(); 587 mMediaBuffers.editItemAt(index) = NULL; 588 } 589 mInputBufferIsDequeued.editItemAt(index) = true; 590 591 if (!mCSDsToSubmit.isEmpty()) { 592 sp<AMessage> msg = new AMessage(); 593 msg->setSize("buffer-ix", index); 594 595 sp<ABuffer> buffer = mCSDsToSubmit.itemAt(0); 596 ALOGI("[%s] resubmitting CSD", mComponentName.c_str()); 597 msg->setBuffer("buffer", buffer); 598 mCSDsToSubmit.removeAt(0); 599 if (!onInputBufferFetched(msg)) { 600 handleError(UNKNOWN_ERROR); 601 return false; 602 } 603 return true; 604 } 605 606 while (!mPendingInputMessages.empty()) { 607 sp<AMessage> msg = *mPendingInputMessages.begin(); 608 if (!onInputBufferFetched(msg)) { 609 break; 610 } 611 mPendingInputMessages.erase(mPendingInputMessages.begin()); 612 } 613 614 if (!mInputBufferIsDequeued.editItemAt(index)) { 615 return true; 616 } 617 618 mDequeuedInputBuffers.push_back(index); 619 620 onRequestInputBuffers(); 621 return true; 622 } 623 624 bool NuPlayer::Decoder::handleAnOutputBuffer( 625 size_t index, 626 size_t offset, 627 size_t size, 628 int64_t timeUs, 629 int32_t flags) { 630 // CHECK_LT(bufferIx, mOutputBuffers.size()); 631 sp<ABuffer> buffer; 632 mCodec->getOutputBuffer(index, &buffer); 633 634 if (index >= mOutputBuffers.size()) { 635 for (size_t i = mOutputBuffers.size(); i <= index; ++i) { 636 mOutputBuffers.add(); 637 } 638 } 639 640 mOutputBuffers.editItemAt(index) = buffer; 641 642 buffer->setRange(offset, size); 643 buffer->meta()->clear(); 644 buffer->meta()->setInt64("timeUs", timeUs); 645 646 bool eos = flags & MediaCodec::BUFFER_FLAG_EOS; 647 // we do not expect CODECCONFIG or SYNCFRAME for decoder 648 649 sp<AMessage> reply = new AMessage(kWhatRenderBuffer, this); 650 reply->setSize("buffer-ix", index); 651 reply->setInt32("generation", mBufferGeneration); 652 653 if (eos) { 654 ALOGI("[%s] saw output EOS", mIsAudio ? "audio" : "video"); 655 656 buffer->meta()->setInt32("eos", true); 657 reply->setInt32("eos", true); 658 } else if (mSkipRenderingUntilMediaTimeUs >= 0) { 659 if (timeUs < mSkipRenderingUntilMediaTimeUs) { 660 ALOGV("[%s] dropping buffer at time %lld as requested.", 661 mComponentName.c_str(), (long long)timeUs); 662 663 reply->post(); 664 return true; 665 } 666 667 mSkipRenderingUntilMediaTimeUs = -1; 668 } 669 670 mNumFramesTotal += !mIsAudio; 671 672 // wait until 1st frame comes out to signal resume complete 673 notifyResumeCompleteIfNecessary(); 674 675 if (mRenderer != NULL) { 676 // send the buffer to renderer. 677 mRenderer->queueBuffer(mIsAudio, buffer, reply); 678 if (eos && !isDiscontinuityPending()) { 679 mRenderer->queueEOS(mIsAudio, ERROR_END_OF_STREAM); 680 } 681 } 682 683 return true; 684 } 685 686 void NuPlayer::Decoder::handleOutputFormatChange(const sp<AMessage> &format) { 687 if (!mIsAudio) { 688 int32_t width, height; 689 if (format->findInt32("width", &width) 690 && format->findInt32("height", &height)) { 691 mStats->setInt32("width", width); 692 mStats->setInt32("height", height); 693 } 694 sp<AMessage> notify = mNotify->dup(); 695 notify->setInt32("what", kWhatVideoSizeChanged); 696 notify->setMessage("format", format); 697 notify->post(); 698 } else if (mRenderer != NULL) { 699 uint32_t flags; 700 int64_t durationUs; 701 bool hasVideo = (mSource->getFormat(false /* audio */) != NULL); 702 if (getAudioDeepBufferSetting() // override regardless of source duration 703 || (!hasVideo 704 && mSource->getDuration(&durationUs) == OK 705 && durationUs > AUDIO_SINK_MIN_DEEP_BUFFER_DURATION_US)) { 706 flags = AUDIO_OUTPUT_FLAG_DEEP_BUFFER; 707 } else { 708 flags = AUDIO_OUTPUT_FLAG_NONE; 709 } 710 711 status_t err = mRenderer->openAudioSink( 712 format, false /* offloadOnly */, hasVideo, flags, NULL /* isOffloaed */); 713 if (err != OK) { 714 handleError(err); 715 } 716 } 717 } 718 719 void NuPlayer::Decoder::releaseAndResetMediaBuffers() { 720 for (size_t i = 0; i < mMediaBuffers.size(); i++) { 721 if (mMediaBuffers[i] != NULL) { 722 mMediaBuffers[i]->release(); 723 mMediaBuffers.editItemAt(i) = NULL; 724 } 725 } 726 mMediaBuffers.resize(mInputBuffers.size()); 727 for (size_t i = 0; i < mMediaBuffers.size(); i++) { 728 mMediaBuffers.editItemAt(i) = NULL; 729 } 730 mInputBufferIsDequeued.clear(); 731 mInputBufferIsDequeued.resize(mInputBuffers.size()); 732 for (size_t i = 0; i < mInputBufferIsDequeued.size(); i++) { 733 mInputBufferIsDequeued.editItemAt(i) = false; 734 } 735 736 mPendingInputMessages.clear(); 737 mDequeuedInputBuffers.clear(); 738 mSkipRenderingUntilMediaTimeUs = -1; 739 } 740 741 void NuPlayer::Decoder::requestCodecNotification() { 742 if (mCodec != NULL) { 743 sp<AMessage> reply = new AMessage(kWhatCodecNotify, this); 744 reply->setInt32("generation", mBufferGeneration); 745 mCodec->requestActivityNotification(reply); 746 } 747 } 748 749 bool NuPlayer::Decoder::isStaleReply(const sp<AMessage> &msg) { 750 int32_t generation; 751 CHECK(msg->findInt32("generation", &generation)); 752 return generation != mBufferGeneration; 753 } 754 755 status_t NuPlayer::Decoder::fetchInputData(sp<AMessage> &reply) { 756 sp<ABuffer> accessUnit; 757 bool dropAccessUnit = true; 758 do { 759 status_t err = mSource->dequeueAccessUnit(mIsAudio, &accessUnit); 760 761 if (err == -EWOULDBLOCK) { 762 return err; 763 } else if (err != OK) { 764 if (err == INFO_DISCONTINUITY) { 765 int32_t type; 766 CHECK(accessUnit->meta()->findInt32("discontinuity", &type)); 767 768 bool formatChange = 769 (mIsAudio && 770 (type & ATSParser::DISCONTINUITY_AUDIO_FORMAT)) 771 || (!mIsAudio && 772 (type & ATSParser::DISCONTINUITY_VIDEO_FORMAT)); 773 774 bool timeChange = (type & ATSParser::DISCONTINUITY_TIME) != 0; 775 776 ALOGI("%s discontinuity (format=%d, time=%d)", 777 mIsAudio ? "audio" : "video", formatChange, timeChange); 778 779 bool seamlessFormatChange = false; 780 sp<AMessage> newFormat = mSource->getFormat(mIsAudio); 781 if (formatChange) { 782 seamlessFormatChange = 783 supportsSeamlessFormatChange(newFormat); 784 // treat seamless format change separately 785 formatChange = !seamlessFormatChange; 786 } 787 788 // For format or time change, return EOS to queue EOS input, 789 // then wait for EOS on output. 790 if (formatChange /* not seamless */) { 791 mFormatChangePending = true; 792 err = ERROR_END_OF_STREAM; 793 } else if (timeChange) { 794 rememberCodecSpecificData(newFormat); 795 mTimeChangePending = true; 796 err = ERROR_END_OF_STREAM; 797 } else if (seamlessFormatChange) { 798 // reuse existing decoder and don't flush 799 rememberCodecSpecificData(newFormat); 800 continue; 801 } else { 802 // This stream is unaffected by the discontinuity 803 return -EWOULDBLOCK; 804 } 805 } 806 807 // reply should only be returned without a buffer set 808 // when there is an error (including EOS) 809 CHECK(err != OK); 810 811 reply->setInt32("err", err); 812 return ERROR_END_OF_STREAM; 813 } 814 815 dropAccessUnit = false; 816 if (!mIsAudio && !mIsSecure) { 817 int32_t layerId = 0; 818 bool haveLayerId = accessUnit->meta()->findInt32("temporal-layer-id", &layerId); 819 if (mRenderer->getVideoLateByUs() > 100000ll 820 && mIsVideoAVC 821 && !IsAVCReferenceFrame(accessUnit)) { 822 dropAccessUnit = true; 823 } else if (haveLayerId && mNumVideoTemporalLayerTotal > 1) { 824 // Add only one layer each time. 825 if (layerId > mCurrentMaxVideoTemporalLayerId + 1 826 || layerId >= mNumVideoTemporalLayerAllowed) { 827 dropAccessUnit = true; 828 ALOGV("dropping layer(%d), speed=%g, allowed layer count=%d, max layerId=%d", 829 layerId, mPlaybackSpeed, mNumVideoTemporalLayerAllowed, 830 mCurrentMaxVideoTemporalLayerId); 831 } else if (layerId > mCurrentMaxVideoTemporalLayerId) { 832 mCurrentMaxVideoTemporalLayerId = layerId; 833 } else if (layerId == 0 && mNumVideoTemporalLayerTotal > 1 && IsIDR(accessUnit)) { 834 mCurrentMaxVideoTemporalLayerId = mNumVideoTemporalLayerTotal - 1; 835 } 836 } 837 if (dropAccessUnit) { 838 if (layerId <= mCurrentMaxVideoTemporalLayerId && layerId > 0) { 839 mCurrentMaxVideoTemporalLayerId = layerId - 1; 840 } 841 ++mNumInputFramesDropped; 842 } 843 } 844 } while (dropAccessUnit); 845 846 // ALOGV("returned a valid buffer of %s data", mIsAudio ? "mIsAudio" : "video"); 847 #if 0 848 int64_t mediaTimeUs; 849 CHECK(accessUnit->meta()->findInt64("timeUs", &mediaTimeUs)); 850 ALOGV("[%s] feeding input buffer at media time %.3f", 851 mIsAudio ? "audio" : "video", 852 mediaTimeUs / 1E6); 853 #endif 854 855 if (mCCDecoder != NULL) { 856 mCCDecoder->decode(accessUnit); 857 } 858 859 reply->setBuffer("buffer", accessUnit); 860 861 return OK; 862 } 863 864 bool NuPlayer::Decoder::onInputBufferFetched(const sp<AMessage> &msg) { 865 size_t bufferIx; 866 CHECK(msg->findSize("buffer-ix", &bufferIx)); 867 CHECK_LT(bufferIx, mInputBuffers.size()); 868 sp<ABuffer> codecBuffer = mInputBuffers[bufferIx]; 869 870 sp<ABuffer> buffer; 871 bool hasBuffer = msg->findBuffer("buffer", &buffer); 872 873 // handle widevine classic source - that fills an arbitrary input buffer 874 MediaBuffer *mediaBuffer = NULL; 875 if (hasBuffer) { 876 mediaBuffer = (MediaBuffer *)(buffer->getMediaBufferBase()); 877 if (mediaBuffer != NULL) { 878 // likely filled another buffer than we requested: adjust buffer index 879 size_t ix; 880 for (ix = 0; ix < mInputBuffers.size(); ix++) { 881 const sp<ABuffer> &buf = mInputBuffers[ix]; 882 if (buf->data() == mediaBuffer->data()) { 883 // all input buffers are dequeued on start, hence the check 884 if (!mInputBufferIsDequeued[ix]) { 885 ALOGV("[%s] received MediaBuffer for #%zu instead of #%zu", 886 mComponentName.c_str(), ix, bufferIx); 887 mediaBuffer->release(); 888 return false; 889 } 890 891 // TRICKY: need buffer for the metadata, so instead, set 892 // codecBuffer to the same (though incorrect) buffer to 893 // avoid a memcpy into the codecBuffer 894 codecBuffer = buffer; 895 codecBuffer->setRange( 896 mediaBuffer->range_offset(), 897 mediaBuffer->range_length()); 898 bufferIx = ix; 899 break; 900 } 901 } 902 CHECK(ix < mInputBuffers.size()); 903 } 904 } 905 906 if (buffer == NULL /* includes !hasBuffer */) { 907 int32_t streamErr = ERROR_END_OF_STREAM; 908 CHECK(msg->findInt32("err", &streamErr) || !hasBuffer); 909 910 CHECK(streamErr != OK); 911 912 // attempt to queue EOS 913 status_t err = mCodec->queueInputBuffer( 914 bufferIx, 915 0, 916 0, 917 0, 918 MediaCodec::BUFFER_FLAG_EOS); 919 if (err == OK) { 920 mInputBufferIsDequeued.editItemAt(bufferIx) = false; 921 } else if (streamErr == ERROR_END_OF_STREAM) { 922 streamErr = err; 923 // err will not be ERROR_END_OF_STREAM 924 } 925 926 if (streamErr != ERROR_END_OF_STREAM) { 927 ALOGE("Stream error for %s (err=%d), EOS %s queued", 928 mComponentName.c_str(), 929 streamErr, 930 err == OK ? "successfully" : "unsuccessfully"); 931 handleError(streamErr); 932 } 933 } else { 934 sp<AMessage> extra; 935 if (buffer->meta()->findMessage("extra", &extra) && extra != NULL) { 936 int64_t resumeAtMediaTimeUs; 937 if (extra->findInt64( 938 "resume-at-mediaTimeUs", &resumeAtMediaTimeUs)) { 939 ALOGI("[%s] suppressing rendering until %lld us", 940 mComponentName.c_str(), (long long)resumeAtMediaTimeUs); 941 mSkipRenderingUntilMediaTimeUs = resumeAtMediaTimeUs; 942 } 943 } 944 945 int64_t timeUs = 0; 946 uint32_t flags = 0; 947 CHECK(buffer->meta()->findInt64("timeUs", &timeUs)); 948 949 int32_t eos, csd; 950 // we do not expect SYNCFRAME for decoder 951 if (buffer->meta()->findInt32("eos", &eos) && eos) { 952 flags |= MediaCodec::BUFFER_FLAG_EOS; 953 } else if (buffer->meta()->findInt32("csd", &csd) && csd) { 954 flags |= MediaCodec::BUFFER_FLAG_CODECCONFIG; 955 } 956 957 // copy into codec buffer 958 if (buffer != codecBuffer) { 959 if (buffer->size() > codecBuffer->capacity()) { 960 handleError(ERROR_BUFFER_TOO_SMALL); 961 mDequeuedInputBuffers.push_back(bufferIx); 962 return false; 963 } 964 codecBuffer->setRange(0, buffer->size()); 965 memcpy(codecBuffer->data(), buffer->data(), buffer->size()); 966 } 967 968 status_t err = mCodec->queueInputBuffer( 969 bufferIx, 970 codecBuffer->offset(), 971 codecBuffer->size(), 972 timeUs, 973 flags); 974 if (err != OK) { 975 if (mediaBuffer != NULL) { 976 mediaBuffer->release(); 977 } 978 ALOGE("Failed to queue input buffer for %s (err=%d)", 979 mComponentName.c_str(), err); 980 handleError(err); 981 } else { 982 mInputBufferIsDequeued.editItemAt(bufferIx) = false; 983 if (mediaBuffer != NULL) { 984 CHECK(mMediaBuffers[bufferIx] == NULL); 985 mMediaBuffers.editItemAt(bufferIx) = mediaBuffer; 986 } 987 } 988 } 989 return true; 990 } 991 992 void NuPlayer::Decoder::onRenderBuffer(const sp<AMessage> &msg) { 993 status_t err; 994 int32_t render; 995 size_t bufferIx; 996 int32_t eos; 997 CHECK(msg->findSize("buffer-ix", &bufferIx)); 998 999 if (!mIsAudio) { 1000 int64_t timeUs; 1001 sp<ABuffer> buffer = mOutputBuffers[bufferIx]; 1002 buffer->meta()->findInt64("timeUs", &timeUs); 1003 1004 if (mCCDecoder != NULL && mCCDecoder->isSelected()) { 1005 mCCDecoder->display(timeUs); 1006 } 1007 } 1008 1009 if (msg->findInt32("render", &render) && render) { 1010 int64_t timestampNs; 1011 CHECK(msg->findInt64("timestampNs", ×tampNs)); 1012 err = mCodec->renderOutputBufferAndRelease(bufferIx, timestampNs); 1013 } else { 1014 mNumOutputFramesDropped += !mIsAudio; 1015 err = mCodec->releaseOutputBuffer(bufferIx); 1016 } 1017 if (err != OK) { 1018 ALOGE("failed to release output buffer for %s (err=%d)", 1019 mComponentName.c_str(), err); 1020 handleError(err); 1021 } 1022 if (msg->findInt32("eos", &eos) && eos 1023 && isDiscontinuityPending()) { 1024 finishHandleDiscontinuity(true /* flushOnTimeChange */); 1025 } 1026 } 1027 1028 bool NuPlayer::Decoder::isDiscontinuityPending() const { 1029 return mFormatChangePending || mTimeChangePending; 1030 } 1031 1032 void NuPlayer::Decoder::finishHandleDiscontinuity(bool flushOnTimeChange) { 1033 ALOGV("finishHandleDiscontinuity: format %d, time %d, flush %d", 1034 mFormatChangePending, mTimeChangePending, flushOnTimeChange); 1035 1036 // If we have format change, pause and wait to be killed; 1037 // If we have time change only, flush and restart fetching. 1038 1039 if (mFormatChangePending) { 1040 mPaused = true; 1041 } else if (mTimeChangePending) { 1042 if (flushOnTimeChange) { 1043 doFlush(false /* notifyComplete */); 1044 signalResume(false /* notifyComplete */); 1045 } 1046 } 1047 1048 // Notify NuPlayer to either shutdown decoder, or rescan sources 1049 sp<AMessage> msg = mNotify->dup(); 1050 msg->setInt32("what", kWhatInputDiscontinuity); 1051 msg->setInt32("formatChange", mFormatChangePending); 1052 msg->post(); 1053 1054 mFormatChangePending = false; 1055 mTimeChangePending = false; 1056 } 1057 1058 bool NuPlayer::Decoder::supportsSeamlessAudioFormatChange( 1059 const sp<AMessage> &targetFormat) const { 1060 if (targetFormat == NULL) { 1061 return true; 1062 } 1063 1064 AString mime; 1065 if (!targetFormat->findString("mime", &mime)) { 1066 return false; 1067 } 1068 1069 if (!strcasecmp(mime.c_str(), MEDIA_MIMETYPE_AUDIO_AAC)) { 1070 // field-by-field comparison 1071 const char * keys[] = { "channel-count", "sample-rate", "is-adts" }; 1072 for (unsigned int i = 0; i < sizeof(keys) / sizeof(keys[0]); i++) { 1073 int32_t oldVal, newVal; 1074 if (!mInputFormat->findInt32(keys[i], &oldVal) || 1075 !targetFormat->findInt32(keys[i], &newVal) || 1076 oldVal != newVal) { 1077 return false; 1078 } 1079 } 1080 1081 sp<ABuffer> oldBuf, newBuf; 1082 if (mInputFormat->findBuffer("csd-0", &oldBuf) && 1083 targetFormat->findBuffer("csd-0", &newBuf)) { 1084 if (oldBuf->size() != newBuf->size()) { 1085 return false; 1086 } 1087 return !memcmp(oldBuf->data(), newBuf->data(), oldBuf->size()); 1088 } 1089 } 1090 return false; 1091 } 1092 1093 bool NuPlayer::Decoder::supportsSeamlessFormatChange(const sp<AMessage> &targetFormat) const { 1094 if (mInputFormat == NULL) { 1095 return false; 1096 } 1097 1098 if (targetFormat == NULL) { 1099 return true; 1100 } 1101 1102 AString oldMime, newMime; 1103 if (!mInputFormat->findString("mime", &oldMime) 1104 || !targetFormat->findString("mime", &newMime) 1105 || !(oldMime == newMime)) { 1106 return false; 1107 } 1108 1109 bool audio = !strncasecmp(oldMime.c_str(), "audio/", strlen("audio/")); 1110 bool seamless; 1111 if (audio) { 1112 seamless = supportsSeamlessAudioFormatChange(targetFormat); 1113 } else { 1114 int32_t isAdaptive; 1115 seamless = (mCodec != NULL && 1116 mInputFormat->findInt32("adaptive-playback", &isAdaptive) && 1117 isAdaptive); 1118 } 1119 1120 ALOGV("%s seamless support for %s", seamless ? "yes" : "no", oldMime.c_str()); 1121 return seamless; 1122 } 1123 1124 void NuPlayer::Decoder::rememberCodecSpecificData(const sp<AMessage> &format) { 1125 if (format == NULL) { 1126 return; 1127 } 1128 mCSDsForCurrentFormat.clear(); 1129 for (int32_t i = 0; ; ++i) { 1130 AString tag = "csd-"; 1131 tag.append(i); 1132 sp<ABuffer> buffer; 1133 if (!format->findBuffer(tag.c_str(), &buffer)) { 1134 break; 1135 } 1136 mCSDsForCurrentFormat.push(buffer); 1137 } 1138 } 1139 1140 void NuPlayer::Decoder::notifyResumeCompleteIfNecessary() { 1141 if (mResumePending) { 1142 mResumePending = false; 1143 1144 sp<AMessage> notify = mNotify->dup(); 1145 notify->setInt32("what", kWhatResumeCompleted); 1146 notify->post(); 1147 } 1148 } 1149 1150 } // namespace android 1151 1152