1 /* 2 * Copyright (C) 2010 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17 //#define LOG_NDEBUG 0 18 #define LOG_TAG "NuPlayerRenderer" 19 #include <utils/Log.h> 20 21 #include "NuPlayerRenderer.h" 22 #include <algorithm> 23 #include <cutils/properties.h> 24 #include <media/stagefright/foundation/ADebug.h> 25 #include <media/stagefright/foundation/AMessage.h> 26 #include <media/stagefright/foundation/AUtils.h> 27 #include <media/stagefright/foundation/AWakeLock.h> 28 #include <media/stagefright/MediaClock.h> 29 #include <media/stagefright/MediaErrors.h> 30 #include <media/stagefright/MetaData.h> 31 #include <media/stagefright/Utils.h> 32 #include <media/stagefright/VideoFrameScheduler.h> 33 #include <media/MediaCodecBuffer.h> 34 35 #include <inttypes.h> 36 37 namespace android { 38 39 /* 40 * Example of common configuration settings in shell script form 41 42 #Turn offload audio off (use PCM for Play Music) -- AudioPolicyManager 43 adb shell setprop audio.offload.disable 1 44 45 #Allow offload audio with video (requires offloading to be enabled) -- AudioPolicyManager 46 adb shell setprop audio.offload.video 1 47 48 #Use audio callbacks for PCM data 49 adb shell setprop media.stagefright.audio.cbk 1 50 51 #Use deep buffer for PCM data with video (it is generally enabled for audio-only) 52 adb shell setprop media.stagefright.audio.deep 1 53 54 #Set size of buffers for pcm audio sink in msec (example: 1000 msec) 55 adb shell setprop media.stagefright.audio.sink 1000 56 57 * These configurations take effect for the next track played (not the current track). 58 */ 59 60 static inline bool getUseAudioCallbackSetting() { 61 return property_get_bool("media.stagefright.audio.cbk", false /* default_value */); 62 } 63 64 static inline int32_t getAudioSinkPcmMsSetting() { 65 return property_get_int32( 66 "media.stagefright.audio.sink", 500 /* default_value */); 67 } 68 69 // Maximum time in paused state when offloading audio decompression. When elapsed, the AudioSink 70 // is closed to allow the audio DSP to power down. 71 static const int64_t kOffloadPauseMaxUs = 10000000ll; 72 73 // Maximum allowed delay from AudioSink, 1.5 seconds. 74 static const int64_t kMaxAllowedAudioSinkDelayUs = 1500000ll; 75 76 static const int64_t kMinimumAudioClockUpdatePeriodUs = 20 /* msec */ * 1000; 77 78 // static 79 const NuPlayer::Renderer::PcmInfo NuPlayer::Renderer::AUDIO_PCMINFO_INITIALIZER = { 80 AUDIO_CHANNEL_NONE, 81 AUDIO_OUTPUT_FLAG_NONE, 82 AUDIO_FORMAT_INVALID, 83 0, // mNumChannels 84 0 // mSampleRate 85 }; 86 87 // static 88 const int64_t NuPlayer::Renderer::kMinPositionUpdateDelayUs = 100000ll; 89 90 NuPlayer::Renderer::Renderer( 91 const sp<MediaPlayerBase::AudioSink> &sink, 92 const sp<AMessage> ¬ify, 93 uint32_t flags) 94 : mAudioSink(sink), 95 mUseVirtualAudioSink(false), 96 mNotify(notify), 97 mFlags(flags), 98 mNumFramesWritten(0), 99 mDrainAudioQueuePending(false), 100 mDrainVideoQueuePending(false), 101 mAudioQueueGeneration(0), 102 mVideoQueueGeneration(0), 103 mAudioDrainGeneration(0), 104 mVideoDrainGeneration(0), 105 mAudioEOSGeneration(0), 106 mPlaybackSettings(AUDIO_PLAYBACK_RATE_DEFAULT), 107 mAudioFirstAnchorTimeMediaUs(-1), 108 mAnchorTimeMediaUs(-1), 109 mAnchorNumFramesWritten(-1), 110 mVideoLateByUs(0ll), 111 mHasAudio(false), 112 mHasVideo(false), 113 mNotifyCompleteAudio(false), 114 mNotifyCompleteVideo(false), 115 mSyncQueues(false), 116 mPaused(false), 117 mPauseDrainAudioAllowedUs(0), 118 mVideoSampleReceived(false), 119 mVideoRenderingStarted(false), 120 mVideoRenderingStartGeneration(0), 121 mAudioRenderingStartGeneration(0), 122 mRenderingDataDelivered(false), 123 mNextAudioClockUpdateTimeUs(-1), 124 mLastAudioMediaTimeUs(-1), 125 mAudioOffloadPauseTimeoutGeneration(0), 126 mAudioTornDown(false), 127 mCurrentOffloadInfo(AUDIO_INFO_INITIALIZER), 128 mCurrentPcmInfo(AUDIO_PCMINFO_INITIALIZER), 129 mTotalBuffersQueued(0), 130 mLastAudioBufferDrained(0), 131 mUseAudioCallback(false), 132 mWakeLock(new AWakeLock()) { 133 mMediaClock = new MediaClock; 134 mPlaybackRate = mPlaybackSettings.mSpeed; 135 mMediaClock->setPlaybackRate(mPlaybackRate); 136 } 137 138 NuPlayer::Renderer::~Renderer() { 139 if (offloadingAudio()) { 140 mAudioSink->stop(); 141 mAudioSink->flush(); 142 mAudioSink->close(); 143 } 144 145 // Try to avoid racing condition in case callback is still on. 146 Mutex::Autolock autoLock(mLock); 147 if (mUseAudioCallback) { 148 flushQueue(&mAudioQueue); 149 flushQueue(&mVideoQueue); 150 } 151 mWakeLock.clear(); 152 mMediaClock.clear(); 153 mVideoScheduler.clear(); 154 mNotify.clear(); 155 mAudioSink.clear(); 156 } 157 158 void NuPlayer::Renderer::queueBuffer( 159 bool audio, 160 const sp<MediaCodecBuffer> &buffer, 161 const sp<AMessage> ¬ifyConsumed) { 162 sp<AMessage> msg = new AMessage(kWhatQueueBuffer, this); 163 msg->setInt32("queueGeneration", getQueueGeneration(audio)); 164 msg->setInt32("audio", static_cast<int32_t>(audio)); 165 msg->setObject("buffer", buffer); 166 msg->setMessage("notifyConsumed", notifyConsumed); 167 msg->post(); 168 } 169 170 void NuPlayer::Renderer::queueEOS(bool audio, status_t finalResult) { 171 CHECK_NE(finalResult, (status_t)OK); 172 173 sp<AMessage> msg = new AMessage(kWhatQueueEOS, this); 174 msg->setInt32("queueGeneration", getQueueGeneration(audio)); 175 msg->setInt32("audio", static_cast<int32_t>(audio)); 176 msg->setInt32("finalResult", finalResult); 177 msg->post(); 178 } 179 180 status_t NuPlayer::Renderer::setPlaybackSettings(const AudioPlaybackRate &rate) { 181 sp<AMessage> msg = new AMessage(kWhatConfigPlayback, this); 182 writeToAMessage(msg, rate); 183 sp<AMessage> response; 184 status_t err = msg->postAndAwaitResponse(&response); 185 if (err == OK && response != NULL) { 186 CHECK(response->findInt32("err", &err)); 187 } 188 return err; 189 } 190 191 status_t NuPlayer::Renderer::onConfigPlayback(const AudioPlaybackRate &rate /* sanitized */) { 192 if (rate.mSpeed == 0.f) { 193 onPause(); 194 // don't call audiosink's setPlaybackRate if pausing, as pitch does not 195 // have to correspond to the any non-0 speed (e.g old speed). Keep 196 // settings nonetheless, using the old speed, in case audiosink changes. 197 AudioPlaybackRate newRate = rate; 198 newRate.mSpeed = mPlaybackSettings.mSpeed; 199 mPlaybackSettings = newRate; 200 return OK; 201 } 202 203 if (mAudioSink != NULL && mAudioSink->ready()) { 204 status_t err = mAudioSink->setPlaybackRate(rate); 205 if (err != OK) { 206 return err; 207 } 208 } 209 mPlaybackSettings = rate; 210 mPlaybackRate = rate.mSpeed; 211 mMediaClock->setPlaybackRate(mPlaybackRate); 212 return OK; 213 } 214 215 status_t NuPlayer::Renderer::getPlaybackSettings(AudioPlaybackRate *rate /* nonnull */) { 216 sp<AMessage> msg = new AMessage(kWhatGetPlaybackSettings, this); 217 sp<AMessage> response; 218 status_t err = msg->postAndAwaitResponse(&response); 219 if (err == OK && response != NULL) { 220 CHECK(response->findInt32("err", &err)); 221 if (err == OK) { 222 readFromAMessage(response, rate); 223 } 224 } 225 return err; 226 } 227 228 status_t NuPlayer::Renderer::onGetPlaybackSettings(AudioPlaybackRate *rate /* nonnull */) { 229 if (mAudioSink != NULL && mAudioSink->ready()) { 230 status_t err = mAudioSink->getPlaybackRate(rate); 231 if (err == OK) { 232 if (!isAudioPlaybackRateEqual(*rate, mPlaybackSettings)) { 233 ALOGW("correcting mismatch in internal/external playback rate"); 234 } 235 // get playback settings used by audiosink, as it may be 236 // slightly off due to audiosink not taking small changes. 237 mPlaybackSettings = *rate; 238 if (mPaused) { 239 rate->mSpeed = 0.f; 240 } 241 } 242 return err; 243 } 244 *rate = mPlaybackSettings; 245 return OK; 246 } 247 248 status_t NuPlayer::Renderer::setSyncSettings(const AVSyncSettings &sync, float videoFpsHint) { 249 sp<AMessage> msg = new AMessage(kWhatConfigSync, this); 250 writeToAMessage(msg, sync, videoFpsHint); 251 sp<AMessage> response; 252 status_t err = msg->postAndAwaitResponse(&response); 253 if (err == OK && response != NULL) { 254 CHECK(response->findInt32("err", &err)); 255 } 256 return err; 257 } 258 259 status_t NuPlayer::Renderer::onConfigSync(const AVSyncSettings &sync, float videoFpsHint __unused) { 260 if (sync.mSource != AVSYNC_SOURCE_DEFAULT) { 261 return BAD_VALUE; 262 } 263 // TODO: support sync sources 264 return INVALID_OPERATION; 265 } 266 267 status_t NuPlayer::Renderer::getSyncSettings(AVSyncSettings *sync, float *videoFps) { 268 sp<AMessage> msg = new AMessage(kWhatGetSyncSettings, this); 269 sp<AMessage> response; 270 status_t err = msg->postAndAwaitResponse(&response); 271 if (err == OK && response != NULL) { 272 CHECK(response->findInt32("err", &err)); 273 if (err == OK) { 274 readFromAMessage(response, sync, videoFps); 275 } 276 } 277 return err; 278 } 279 280 status_t NuPlayer::Renderer::onGetSyncSettings( 281 AVSyncSettings *sync /* nonnull */, float *videoFps /* nonnull */) { 282 *sync = mSyncSettings; 283 *videoFps = -1.f; 284 return OK; 285 } 286 287 void NuPlayer::Renderer::flush(bool audio, bool notifyComplete) { 288 { 289 Mutex::Autolock autoLock(mLock); 290 if (audio) { 291 mNotifyCompleteAudio |= notifyComplete; 292 clearAudioFirstAnchorTime_l(); 293 ++mAudioQueueGeneration; 294 ++mAudioDrainGeneration; 295 } else { 296 mNotifyCompleteVideo |= notifyComplete; 297 ++mVideoQueueGeneration; 298 ++mVideoDrainGeneration; 299 } 300 301 mMediaClock->clearAnchor(); 302 mVideoLateByUs = 0; 303 mSyncQueues = false; 304 } 305 306 sp<AMessage> msg = new AMessage(kWhatFlush, this); 307 msg->setInt32("audio", static_cast<int32_t>(audio)); 308 msg->post(); 309 } 310 311 void NuPlayer::Renderer::signalTimeDiscontinuity() { 312 } 313 314 void NuPlayer::Renderer::signalDisableOffloadAudio() { 315 (new AMessage(kWhatDisableOffloadAudio, this))->post(); 316 } 317 318 void NuPlayer::Renderer::signalEnableOffloadAudio() { 319 (new AMessage(kWhatEnableOffloadAudio, this))->post(); 320 } 321 322 void NuPlayer::Renderer::pause() { 323 (new AMessage(kWhatPause, this))->post(); 324 } 325 326 void NuPlayer::Renderer::resume() { 327 (new AMessage(kWhatResume, this))->post(); 328 } 329 330 void NuPlayer::Renderer::setVideoFrameRate(float fps) { 331 sp<AMessage> msg = new AMessage(kWhatSetVideoFrameRate, this); 332 msg->setFloat("frame-rate", fps); 333 msg->post(); 334 } 335 336 // Called on any threads without mLock acquired. 337 status_t NuPlayer::Renderer::getCurrentPosition(int64_t *mediaUs) { 338 status_t result = mMediaClock->getMediaTime(ALooper::GetNowUs(), mediaUs); 339 if (result == OK) { 340 return result; 341 } 342 343 // MediaClock has not started yet. Try to start it if possible. 344 { 345 Mutex::Autolock autoLock(mLock); 346 if (mAudioFirstAnchorTimeMediaUs == -1) { 347 return result; 348 } 349 350 AudioTimestamp ts; 351 status_t res = mAudioSink->getTimestamp(ts); 352 if (res != OK) { 353 return result; 354 } 355 356 // AudioSink has rendered some frames. 357 int64_t nowUs = ALooper::GetNowUs(); 358 int64_t nowMediaUs = mAudioSink->getPlayedOutDurationUs(nowUs) 359 + mAudioFirstAnchorTimeMediaUs; 360 mMediaClock->updateAnchor(nowMediaUs, nowUs, -1); 361 } 362 363 return mMediaClock->getMediaTime(ALooper::GetNowUs(), mediaUs); 364 } 365 366 void NuPlayer::Renderer::clearAudioFirstAnchorTime_l() { 367 mAudioFirstAnchorTimeMediaUs = -1; 368 mMediaClock->setStartingTimeMedia(-1); 369 } 370 371 void NuPlayer::Renderer::setAudioFirstAnchorTimeIfNeeded_l(int64_t mediaUs) { 372 if (mAudioFirstAnchorTimeMediaUs == -1) { 373 mAudioFirstAnchorTimeMediaUs = mediaUs; 374 mMediaClock->setStartingTimeMedia(mediaUs); 375 } 376 } 377 378 // Called on renderer looper. 379 void NuPlayer::Renderer::clearAnchorTime() { 380 mMediaClock->clearAnchor(); 381 mAnchorTimeMediaUs = -1; 382 mAnchorNumFramesWritten = -1; 383 } 384 385 void NuPlayer::Renderer::setVideoLateByUs(int64_t lateUs) { 386 Mutex::Autolock autoLock(mLock); 387 mVideoLateByUs = lateUs; 388 } 389 390 int64_t NuPlayer::Renderer::getVideoLateByUs() { 391 Mutex::Autolock autoLock(mLock); 392 return mVideoLateByUs; 393 } 394 395 status_t NuPlayer::Renderer::openAudioSink( 396 const sp<AMessage> &format, 397 bool offloadOnly, 398 bool hasVideo, 399 uint32_t flags, 400 bool *isOffloaded, 401 bool isStreaming) { 402 sp<AMessage> msg = new AMessage(kWhatOpenAudioSink, this); 403 msg->setMessage("format", format); 404 msg->setInt32("offload-only", offloadOnly); 405 msg->setInt32("has-video", hasVideo); 406 msg->setInt32("flags", flags); 407 msg->setInt32("isStreaming", isStreaming); 408 409 sp<AMessage> response; 410 status_t postStatus = msg->postAndAwaitResponse(&response); 411 412 int32_t err; 413 if (postStatus != OK || response.get() == nullptr || !response->findInt32("err", &err)) { 414 err = INVALID_OPERATION; 415 } else if (err == OK && isOffloaded != NULL) { 416 int32_t offload; 417 CHECK(response->findInt32("offload", &offload)); 418 *isOffloaded = (offload != 0); 419 } 420 return err; 421 } 422 423 void NuPlayer::Renderer::closeAudioSink() { 424 sp<AMessage> msg = new AMessage(kWhatCloseAudioSink, this); 425 426 sp<AMessage> response; 427 msg->postAndAwaitResponse(&response); 428 } 429 430 void NuPlayer::Renderer::changeAudioFormat( 431 const sp<AMessage> &format, 432 bool offloadOnly, 433 bool hasVideo, 434 uint32_t flags, 435 bool isStreaming, 436 const sp<AMessage> ¬ify) { 437 sp<AMessage> meta = new AMessage; 438 meta->setMessage("format", format); 439 meta->setInt32("offload-only", offloadOnly); 440 meta->setInt32("has-video", hasVideo); 441 meta->setInt32("flags", flags); 442 meta->setInt32("isStreaming", isStreaming); 443 444 sp<AMessage> msg = new AMessage(kWhatChangeAudioFormat, this); 445 msg->setInt32("queueGeneration", getQueueGeneration(true /* audio */)); 446 msg->setMessage("notify", notify); 447 msg->setMessage("meta", meta); 448 msg->post(); 449 } 450 451 void NuPlayer::Renderer::onMessageReceived(const sp<AMessage> &msg) { 452 switch (msg->what()) { 453 case kWhatOpenAudioSink: 454 { 455 sp<AMessage> format; 456 CHECK(msg->findMessage("format", &format)); 457 458 int32_t offloadOnly; 459 CHECK(msg->findInt32("offload-only", &offloadOnly)); 460 461 int32_t hasVideo; 462 CHECK(msg->findInt32("has-video", &hasVideo)); 463 464 uint32_t flags; 465 CHECK(msg->findInt32("flags", (int32_t *)&flags)); 466 467 uint32_t isStreaming; 468 CHECK(msg->findInt32("isStreaming", (int32_t *)&isStreaming)); 469 470 status_t err = onOpenAudioSink(format, offloadOnly, hasVideo, flags, isStreaming); 471 472 sp<AMessage> response = new AMessage; 473 response->setInt32("err", err); 474 response->setInt32("offload", offloadingAudio()); 475 476 sp<AReplyToken> replyID; 477 CHECK(msg->senderAwaitsResponse(&replyID)); 478 response->postReply(replyID); 479 480 break; 481 } 482 483 case kWhatCloseAudioSink: 484 { 485 sp<AReplyToken> replyID; 486 CHECK(msg->senderAwaitsResponse(&replyID)); 487 488 onCloseAudioSink(); 489 490 sp<AMessage> response = new AMessage; 491 response->postReply(replyID); 492 break; 493 } 494 495 case kWhatStopAudioSink: 496 { 497 mAudioSink->stop(); 498 break; 499 } 500 501 case kWhatChangeAudioFormat: 502 { 503 int32_t queueGeneration; 504 CHECK(msg->findInt32("queueGeneration", &queueGeneration)); 505 506 sp<AMessage> notify; 507 CHECK(msg->findMessage("notify", ¬ify)); 508 509 if (offloadingAudio()) { 510 ALOGW("changeAudioFormat should NOT be called in offload mode"); 511 notify->setInt32("err", INVALID_OPERATION); 512 notify->post(); 513 break; 514 } 515 516 sp<AMessage> meta; 517 CHECK(msg->findMessage("meta", &meta)); 518 519 if (queueGeneration != getQueueGeneration(true /* audio */) 520 || mAudioQueue.empty()) { 521 onChangeAudioFormat(meta, notify); 522 break; 523 } 524 525 QueueEntry entry; 526 entry.mNotifyConsumed = notify; 527 entry.mMeta = meta; 528 529 Mutex::Autolock autoLock(mLock); 530 mAudioQueue.push_back(entry); 531 postDrainAudioQueue_l(); 532 533 break; 534 } 535 536 case kWhatDrainAudioQueue: 537 { 538 mDrainAudioQueuePending = false; 539 540 int32_t generation; 541 CHECK(msg->findInt32("drainGeneration", &generation)); 542 if (generation != getDrainGeneration(true /* audio */)) { 543 break; 544 } 545 546 if (onDrainAudioQueue()) { 547 uint32_t numFramesPlayed; 548 CHECK_EQ(mAudioSink->getPosition(&numFramesPlayed), 549 (status_t)OK); 550 551 uint32_t numFramesPendingPlayout = 552 mNumFramesWritten - numFramesPlayed; 553 554 // This is how long the audio sink will have data to 555 // play back. 556 int64_t delayUs = 557 mAudioSink->msecsPerFrame() 558 * numFramesPendingPlayout * 1000ll; 559 if (mPlaybackRate > 1.0f) { 560 delayUs /= mPlaybackRate; 561 } 562 563 // Let's give it more data after about half that time 564 // has elapsed. 565 delayUs /= 2; 566 // check the buffer size to estimate maximum delay permitted. 567 const int64_t maxDrainDelayUs = std::max( 568 mAudioSink->getBufferDurationInUs(), (int64_t)500000 /* half second */); 569 ALOGD_IF(delayUs > maxDrainDelayUs, "postDrainAudioQueue long delay: %lld > %lld", 570 (long long)delayUs, (long long)maxDrainDelayUs); 571 Mutex::Autolock autoLock(mLock); 572 postDrainAudioQueue_l(delayUs); 573 } 574 break; 575 } 576 577 case kWhatDrainVideoQueue: 578 { 579 int32_t generation; 580 CHECK(msg->findInt32("drainGeneration", &generation)); 581 if (generation != getDrainGeneration(false /* audio */)) { 582 break; 583 } 584 585 mDrainVideoQueuePending = false; 586 587 onDrainVideoQueue(); 588 589 postDrainVideoQueue(); 590 break; 591 } 592 593 case kWhatPostDrainVideoQueue: 594 { 595 int32_t generation; 596 CHECK(msg->findInt32("drainGeneration", &generation)); 597 if (generation != getDrainGeneration(false /* audio */)) { 598 break; 599 } 600 601 mDrainVideoQueuePending = false; 602 postDrainVideoQueue(); 603 break; 604 } 605 606 case kWhatQueueBuffer: 607 { 608 onQueueBuffer(msg); 609 break; 610 } 611 612 case kWhatQueueEOS: 613 { 614 onQueueEOS(msg); 615 break; 616 } 617 618 case kWhatEOS: 619 { 620 int32_t generation; 621 CHECK(msg->findInt32("audioEOSGeneration", &generation)); 622 if (generation != mAudioEOSGeneration) { 623 break; 624 } 625 status_t finalResult; 626 CHECK(msg->findInt32("finalResult", &finalResult)); 627 notifyEOS(true /* audio */, finalResult); 628 break; 629 } 630 631 case kWhatConfigPlayback: 632 { 633 sp<AReplyToken> replyID; 634 CHECK(msg->senderAwaitsResponse(&replyID)); 635 AudioPlaybackRate rate; 636 readFromAMessage(msg, &rate); 637 status_t err = onConfigPlayback(rate); 638 sp<AMessage> response = new AMessage; 639 response->setInt32("err", err); 640 response->postReply(replyID); 641 break; 642 } 643 644 case kWhatGetPlaybackSettings: 645 { 646 sp<AReplyToken> replyID; 647 CHECK(msg->senderAwaitsResponse(&replyID)); 648 AudioPlaybackRate rate = AUDIO_PLAYBACK_RATE_DEFAULT; 649 status_t err = onGetPlaybackSettings(&rate); 650 sp<AMessage> response = new AMessage; 651 if (err == OK) { 652 writeToAMessage(response, rate); 653 } 654 response->setInt32("err", err); 655 response->postReply(replyID); 656 break; 657 } 658 659 case kWhatConfigSync: 660 { 661 sp<AReplyToken> replyID; 662 CHECK(msg->senderAwaitsResponse(&replyID)); 663 AVSyncSettings sync; 664 float videoFpsHint; 665 readFromAMessage(msg, &sync, &videoFpsHint); 666 status_t err = onConfigSync(sync, videoFpsHint); 667 sp<AMessage> response = new AMessage; 668 response->setInt32("err", err); 669 response->postReply(replyID); 670 break; 671 } 672 673 case kWhatGetSyncSettings: 674 { 675 sp<AReplyToken> replyID; 676 CHECK(msg->senderAwaitsResponse(&replyID)); 677 678 ALOGV("kWhatGetSyncSettings"); 679 AVSyncSettings sync; 680 float videoFps = -1.f; 681 status_t err = onGetSyncSettings(&sync, &videoFps); 682 sp<AMessage> response = new AMessage; 683 if (err == OK) { 684 writeToAMessage(response, sync, videoFps); 685 } 686 response->setInt32("err", err); 687 response->postReply(replyID); 688 break; 689 } 690 691 case kWhatFlush: 692 { 693 onFlush(msg); 694 break; 695 } 696 697 case kWhatDisableOffloadAudio: 698 { 699 onDisableOffloadAudio(); 700 break; 701 } 702 703 case kWhatEnableOffloadAudio: 704 { 705 onEnableOffloadAudio(); 706 break; 707 } 708 709 case kWhatPause: 710 { 711 onPause(); 712 break; 713 } 714 715 case kWhatResume: 716 { 717 onResume(); 718 break; 719 } 720 721 case kWhatSetVideoFrameRate: 722 { 723 float fps; 724 CHECK(msg->findFloat("frame-rate", &fps)); 725 onSetVideoFrameRate(fps); 726 break; 727 } 728 729 case kWhatAudioTearDown: 730 { 731 int32_t reason; 732 CHECK(msg->findInt32("reason", &reason)); 733 734 onAudioTearDown((AudioTearDownReason)reason); 735 break; 736 } 737 738 case kWhatAudioOffloadPauseTimeout: 739 { 740 int32_t generation; 741 CHECK(msg->findInt32("drainGeneration", &generation)); 742 if (generation != mAudioOffloadPauseTimeoutGeneration) { 743 break; 744 } 745 ALOGV("Audio Offload tear down due to pause timeout."); 746 onAudioTearDown(kDueToTimeout); 747 mWakeLock->release(); 748 break; 749 } 750 751 default: 752 TRESPASS(); 753 break; 754 } 755 } 756 757 void NuPlayer::Renderer::postDrainAudioQueue_l(int64_t delayUs) { 758 if (mDrainAudioQueuePending || mSyncQueues || mUseAudioCallback) { 759 return; 760 } 761 762 if (mAudioQueue.empty()) { 763 return; 764 } 765 766 // FIXME: if paused, wait until AudioTrack stop() is complete before delivering data. 767 if (mPaused) { 768 const int64_t diffUs = mPauseDrainAudioAllowedUs - ALooper::GetNowUs(); 769 if (diffUs > delayUs) { 770 delayUs = diffUs; 771 } 772 } 773 774 mDrainAudioQueuePending = true; 775 sp<AMessage> msg = new AMessage(kWhatDrainAudioQueue, this); 776 msg->setInt32("drainGeneration", mAudioDrainGeneration); 777 msg->post(delayUs); 778 } 779 780 void NuPlayer::Renderer::prepareForMediaRenderingStart_l() { 781 mAudioRenderingStartGeneration = mAudioDrainGeneration; 782 mVideoRenderingStartGeneration = mVideoDrainGeneration; 783 mRenderingDataDelivered = false; 784 } 785 786 void NuPlayer::Renderer::notifyIfMediaRenderingStarted_l() { 787 if (mVideoRenderingStartGeneration == mVideoDrainGeneration && 788 mAudioRenderingStartGeneration == mAudioDrainGeneration) { 789 mRenderingDataDelivered = true; 790 if (mPaused) { 791 return; 792 } 793 mVideoRenderingStartGeneration = -1; 794 mAudioRenderingStartGeneration = -1; 795 796 sp<AMessage> notify = mNotify->dup(); 797 notify->setInt32("what", kWhatMediaRenderingStart); 798 notify->post(); 799 } 800 } 801 802 // static 803 size_t NuPlayer::Renderer::AudioSinkCallback( 804 MediaPlayerBase::AudioSink * /* audioSink */, 805 void *buffer, 806 size_t size, 807 void *cookie, 808 MediaPlayerBase::AudioSink::cb_event_t event) { 809 NuPlayer::Renderer *me = (NuPlayer::Renderer *)cookie; 810 811 switch (event) { 812 case MediaPlayerBase::AudioSink::CB_EVENT_FILL_BUFFER: 813 { 814 return me->fillAudioBuffer(buffer, size); 815 break; 816 } 817 818 case MediaPlayerBase::AudioSink::CB_EVENT_STREAM_END: 819 { 820 ALOGV("AudioSink::CB_EVENT_STREAM_END"); 821 me->notifyEOSCallback(); 822 break; 823 } 824 825 case MediaPlayerBase::AudioSink::CB_EVENT_TEAR_DOWN: 826 { 827 ALOGV("AudioSink::CB_EVENT_TEAR_DOWN"); 828 me->notifyAudioTearDown(kDueToError); 829 break; 830 } 831 } 832 833 return 0; 834 } 835 836 void NuPlayer::Renderer::notifyEOSCallback() { 837 Mutex::Autolock autoLock(mLock); 838 839 if (!mUseAudioCallback) { 840 return; 841 } 842 843 notifyEOS_l(true /* audio */, ERROR_END_OF_STREAM); 844 } 845 846 size_t NuPlayer::Renderer::fillAudioBuffer(void *buffer, size_t size) { 847 Mutex::Autolock autoLock(mLock); 848 849 if (!mUseAudioCallback) { 850 return 0; 851 } 852 853 bool hasEOS = false; 854 855 size_t sizeCopied = 0; 856 bool firstEntry = true; 857 QueueEntry *entry; // will be valid after while loop if hasEOS is set. 858 while (sizeCopied < size && !mAudioQueue.empty()) { 859 entry = &*mAudioQueue.begin(); 860 861 if (entry->mBuffer == NULL) { // EOS 862 hasEOS = true; 863 mAudioQueue.erase(mAudioQueue.begin()); 864 break; 865 } 866 867 if (firstEntry && entry->mOffset == 0) { 868 firstEntry = false; 869 int64_t mediaTimeUs; 870 CHECK(entry->mBuffer->meta()->findInt64("timeUs", &mediaTimeUs)); 871 ALOGV("fillAudioBuffer: rendering audio at media time %.2f secs", mediaTimeUs / 1E6); 872 setAudioFirstAnchorTimeIfNeeded_l(mediaTimeUs); 873 } 874 875 size_t copy = entry->mBuffer->size() - entry->mOffset; 876 size_t sizeRemaining = size - sizeCopied; 877 if (copy > sizeRemaining) { 878 copy = sizeRemaining; 879 } 880 881 memcpy((char *)buffer + sizeCopied, 882 entry->mBuffer->data() + entry->mOffset, 883 copy); 884 885 entry->mOffset += copy; 886 if (entry->mOffset == entry->mBuffer->size()) { 887 entry->mNotifyConsumed->post(); 888 mAudioQueue.erase(mAudioQueue.begin()); 889 entry = NULL; 890 } 891 sizeCopied += copy; 892 893 notifyIfMediaRenderingStarted_l(); 894 } 895 896 if (mAudioFirstAnchorTimeMediaUs >= 0) { 897 int64_t nowUs = ALooper::GetNowUs(); 898 int64_t nowMediaUs = 899 mAudioFirstAnchorTimeMediaUs + mAudioSink->getPlayedOutDurationUs(nowUs); 900 // we don't know how much data we are queueing for offloaded tracks. 901 mMediaClock->updateAnchor(nowMediaUs, nowUs, INT64_MAX); 902 } 903 904 // for non-offloaded audio, we need to compute the frames written because 905 // there is no EVENT_STREAM_END notification. The frames written gives 906 // an estimate on the pending played out duration. 907 if (!offloadingAudio()) { 908 mNumFramesWritten += sizeCopied / mAudioSink->frameSize(); 909 } 910 911 if (hasEOS) { 912 (new AMessage(kWhatStopAudioSink, this))->post(); 913 // As there is currently no EVENT_STREAM_END callback notification for 914 // non-offloaded audio tracks, we need to post the EOS ourselves. 915 if (!offloadingAudio()) { 916 int64_t postEOSDelayUs = 0; 917 if (mAudioSink->needsTrailingPadding()) { 918 postEOSDelayUs = getPendingAudioPlayoutDurationUs(ALooper::GetNowUs()); 919 } 920 ALOGV("fillAudioBuffer: notifyEOS_l " 921 "mNumFramesWritten:%u finalResult:%d postEOSDelay:%lld", 922 mNumFramesWritten, entry->mFinalResult, (long long)postEOSDelayUs); 923 notifyEOS_l(true /* audio */, entry->mFinalResult, postEOSDelayUs); 924 } 925 } 926 return sizeCopied; 927 } 928 929 void NuPlayer::Renderer::drainAudioQueueUntilLastEOS() { 930 List<QueueEntry>::iterator it = mAudioQueue.begin(), itEOS = it; 931 bool foundEOS = false; 932 while (it != mAudioQueue.end()) { 933 int32_t eos; 934 QueueEntry *entry = &*it++; 935 if ((entry->mBuffer == nullptr && entry->mNotifyConsumed == nullptr) 936 || (entry->mNotifyConsumed->findInt32("eos", &eos) && eos != 0)) { 937 itEOS = it; 938 foundEOS = true; 939 } 940 } 941 942 if (foundEOS) { 943 // post all replies before EOS and drop the samples 944 for (it = mAudioQueue.begin(); it != itEOS; it++) { 945 if (it->mBuffer == nullptr) { 946 if (it->mNotifyConsumed == nullptr) { 947 // delay doesn't matter as we don't even have an AudioTrack 948 notifyEOS(true /* audio */, it->mFinalResult); 949 } else { 950 // TAG for re-opening audio sink. 951 onChangeAudioFormat(it->mMeta, it->mNotifyConsumed); 952 } 953 } else { 954 it->mNotifyConsumed->post(); 955 } 956 } 957 mAudioQueue.erase(mAudioQueue.begin(), itEOS); 958 } 959 } 960 961 bool NuPlayer::Renderer::onDrainAudioQueue() { 962 // do not drain audio during teardown as queued buffers may be invalid. 963 if (mAudioTornDown) { 964 return false; 965 } 966 // TODO: This call to getPosition checks if AudioTrack has been created 967 // in AudioSink before draining audio. If AudioTrack doesn't exist, then 968 // CHECKs on getPosition will fail. 969 // We still need to figure out why AudioTrack is not created when 970 // this function is called. One possible reason could be leftover 971 // audio. Another possible place is to check whether decoder 972 // has received INFO_FORMAT_CHANGED as the first buffer since 973 // AudioSink is opened there, and possible interactions with flush 974 // immediately after start. Investigate error message 975 // "vorbis_dsp_synthesis returned -135", along with RTSP. 976 uint32_t numFramesPlayed; 977 if (mAudioSink->getPosition(&numFramesPlayed) != OK) { 978 // When getPosition fails, renderer will not reschedule the draining 979 // unless new samples are queued. 980 // If we have pending EOS (or "eos" marker for discontinuities), we need 981 // to post these now as NuPlayerDecoder might be waiting for it. 982 drainAudioQueueUntilLastEOS(); 983 984 ALOGW("onDrainAudioQueue(): audio sink is not ready"); 985 return false; 986 } 987 988 #if 0 989 ssize_t numFramesAvailableToWrite = 990 mAudioSink->frameCount() - (mNumFramesWritten - numFramesPlayed); 991 992 if (numFramesAvailableToWrite == mAudioSink->frameCount()) { 993 ALOGI("audio sink underrun"); 994 } else { 995 ALOGV("audio queue has %d frames left to play", 996 mAudioSink->frameCount() - numFramesAvailableToWrite); 997 } 998 #endif 999 1000 uint32_t prevFramesWritten = mNumFramesWritten; 1001 while (!mAudioQueue.empty()) { 1002 QueueEntry *entry = &*mAudioQueue.begin(); 1003 1004 if (entry->mBuffer == NULL) { 1005 if (entry->mNotifyConsumed != nullptr) { 1006 // TAG for re-open audio sink. 1007 onChangeAudioFormat(entry->mMeta, entry->mNotifyConsumed); 1008 mAudioQueue.erase(mAudioQueue.begin()); 1009 continue; 1010 } 1011 1012 // EOS 1013 if (mPaused) { 1014 // Do not notify EOS when paused. 1015 // This is needed to avoid switch to next clip while in pause. 1016 ALOGV("onDrainAudioQueue(): Do not notify EOS when paused"); 1017 return false; 1018 } 1019 1020 int64_t postEOSDelayUs = 0; 1021 if (mAudioSink->needsTrailingPadding()) { 1022 postEOSDelayUs = getPendingAudioPlayoutDurationUs(ALooper::GetNowUs()); 1023 } 1024 notifyEOS(true /* audio */, entry->mFinalResult, postEOSDelayUs); 1025 mLastAudioMediaTimeUs = getDurationUsIfPlayedAtSampleRate(mNumFramesWritten); 1026 1027 mAudioQueue.erase(mAudioQueue.begin()); 1028 entry = NULL; 1029 if (mAudioSink->needsTrailingPadding()) { 1030 // If we're not in gapless playback (i.e. through setNextPlayer), we 1031 // need to stop the track here, because that will play out the last 1032 // little bit at the end of the file. Otherwise short files won't play. 1033 mAudioSink->stop(); 1034 mNumFramesWritten = 0; 1035 } 1036 return false; 1037 } 1038 1039 mLastAudioBufferDrained = entry->mBufferOrdinal; 1040 1041 // ignore 0-sized buffer which could be EOS marker with no data 1042 if (entry->mOffset == 0 && entry->mBuffer->size() > 0) { 1043 int64_t mediaTimeUs; 1044 CHECK(entry->mBuffer->meta()->findInt64("timeUs", &mediaTimeUs)); 1045 ALOGV("onDrainAudioQueue: rendering audio at media time %.2f secs", 1046 mediaTimeUs / 1E6); 1047 onNewAudioMediaTime(mediaTimeUs); 1048 } 1049 1050 size_t copy = entry->mBuffer->size() - entry->mOffset; 1051 1052 ssize_t written = mAudioSink->write(entry->mBuffer->data() + entry->mOffset, 1053 copy, false /* blocking */); 1054 if (written < 0) { 1055 // An error in AudioSink write. Perhaps the AudioSink was not properly opened. 1056 if (written == WOULD_BLOCK) { 1057 ALOGV("AudioSink write would block when writing %zu bytes", copy); 1058 } else { 1059 ALOGE("AudioSink write error(%zd) when writing %zu bytes", written, copy); 1060 // This can only happen when AudioSink was opened with doNotReconnect flag set to 1061 // true, in which case the NuPlayer will handle the reconnect. 1062 notifyAudioTearDown(kDueToError); 1063 } 1064 break; 1065 } 1066 1067 entry->mOffset += written; 1068 size_t remainder = entry->mBuffer->size() - entry->mOffset; 1069 if ((ssize_t)remainder < mAudioSink->frameSize()) { 1070 if (remainder > 0) { 1071 ALOGW("Corrupted audio buffer has fractional frames, discarding %zu bytes.", 1072 remainder); 1073 entry->mOffset += remainder; 1074 copy -= remainder; 1075 } 1076 1077 entry->mNotifyConsumed->post(); 1078 mAudioQueue.erase(mAudioQueue.begin()); 1079 1080 entry = NULL; 1081 } 1082 1083 size_t copiedFrames = written / mAudioSink->frameSize(); 1084 mNumFramesWritten += copiedFrames; 1085 1086 { 1087 Mutex::Autolock autoLock(mLock); 1088 int64_t maxTimeMedia; 1089 maxTimeMedia = 1090 mAnchorTimeMediaUs + 1091 (int64_t)(max((long long)mNumFramesWritten - mAnchorNumFramesWritten, 0LL) 1092 * 1000LL * mAudioSink->msecsPerFrame()); 1093 mMediaClock->updateMaxTimeMedia(maxTimeMedia); 1094 1095 notifyIfMediaRenderingStarted_l(); 1096 } 1097 1098 if (written != (ssize_t)copy) { 1099 // A short count was received from AudioSink::write() 1100 // 1101 // AudioSink write is called in non-blocking mode. 1102 // It may return with a short count when: 1103 // 1104 // 1) Size to be copied is not a multiple of the frame size. Fractional frames are 1105 // discarded. 1106 // 2) The data to be copied exceeds the available buffer in AudioSink. 1107 // 3) An error occurs and data has been partially copied to the buffer in AudioSink. 1108 // 4) AudioSink is an AudioCache for data retrieval, and the AudioCache is exceeded. 1109 1110 // (Case 1) 1111 // Must be a multiple of the frame size. If it is not a multiple of a frame size, it 1112 // needs to fail, as we should not carry over fractional frames between calls. 1113 CHECK_EQ(copy % mAudioSink->frameSize(), 0u); 1114 1115 // (Case 2, 3, 4) 1116 // Return early to the caller. 1117 // Beware of calling immediately again as this may busy-loop if you are not careful. 1118 ALOGV("AudioSink write short frame count %zd < %zu", written, copy); 1119 break; 1120 } 1121 } 1122 1123 // calculate whether we need to reschedule another write. 1124 bool reschedule = !mAudioQueue.empty() 1125 && (!mPaused 1126 || prevFramesWritten != mNumFramesWritten); // permit pause to fill buffers 1127 //ALOGD("reschedule:%d empty:%d mPaused:%d prevFramesWritten:%u mNumFramesWritten:%u", 1128 // reschedule, mAudioQueue.empty(), mPaused, prevFramesWritten, mNumFramesWritten); 1129 return reschedule; 1130 } 1131 1132 int64_t NuPlayer::Renderer::getDurationUsIfPlayedAtSampleRate(uint32_t numFrames) { 1133 int32_t sampleRate = offloadingAudio() ? 1134 mCurrentOffloadInfo.sample_rate : mCurrentPcmInfo.mSampleRate; 1135 if (sampleRate == 0) { 1136 ALOGE("sampleRate is 0 in %s mode", offloadingAudio() ? "offload" : "non-offload"); 1137 return 0; 1138 } 1139 // TODO: remove the (int32_t) casting below as it may overflow at 12.4 hours. 1140 return (int64_t)((int32_t)numFrames * 1000000LL / sampleRate); 1141 } 1142 1143 // Calculate duration of pending samples if played at normal rate (i.e., 1.0). 1144 int64_t NuPlayer::Renderer::getPendingAudioPlayoutDurationUs(int64_t nowUs) { 1145 int64_t writtenAudioDurationUs = getDurationUsIfPlayedAtSampleRate(mNumFramesWritten); 1146 if (mUseVirtualAudioSink) { 1147 int64_t nowUs = ALooper::GetNowUs(); 1148 int64_t mediaUs; 1149 if (mMediaClock->getMediaTime(nowUs, &mediaUs) != OK) { 1150 return 0ll; 1151 } else { 1152 return writtenAudioDurationUs - (mediaUs - mAudioFirstAnchorTimeMediaUs); 1153 } 1154 } 1155 1156 const int64_t audioSinkPlayedUs = mAudioSink->getPlayedOutDurationUs(nowUs); 1157 int64_t pendingUs = writtenAudioDurationUs - audioSinkPlayedUs; 1158 if (pendingUs < 0) { 1159 // This shouldn't happen unless the timestamp is stale. 1160 ALOGW("%s: pendingUs %lld < 0, clamping to zero, potential resume after pause " 1161 "writtenAudioDurationUs: %lld, audioSinkPlayedUs: %lld", 1162 __func__, (long long)pendingUs, 1163 (long long)writtenAudioDurationUs, (long long)audioSinkPlayedUs); 1164 pendingUs = 0; 1165 } 1166 return pendingUs; 1167 } 1168 1169 int64_t NuPlayer::Renderer::getRealTimeUs(int64_t mediaTimeUs, int64_t nowUs) { 1170 int64_t realUs; 1171 if (mMediaClock->getRealTimeFor(mediaTimeUs, &realUs) != OK) { 1172 // If failed to get current position, e.g. due to audio clock is 1173 // not ready, then just play out video immediately without delay. 1174 return nowUs; 1175 } 1176 return realUs; 1177 } 1178 1179 void NuPlayer::Renderer::onNewAudioMediaTime(int64_t mediaTimeUs) { 1180 Mutex::Autolock autoLock(mLock); 1181 // TRICKY: vorbis decoder generates multiple frames with the same 1182 // timestamp, so only update on the first frame with a given timestamp 1183 if (mediaTimeUs == mAnchorTimeMediaUs) { 1184 return; 1185 } 1186 setAudioFirstAnchorTimeIfNeeded_l(mediaTimeUs); 1187 1188 // mNextAudioClockUpdateTimeUs is -1 if we're waiting for audio sink to start 1189 if (mNextAudioClockUpdateTimeUs == -1) { 1190 AudioTimestamp ts; 1191 if (mAudioSink->getTimestamp(ts) == OK && ts.mPosition > 0) { 1192 mNextAudioClockUpdateTimeUs = 0; // start our clock updates 1193 } 1194 } 1195 int64_t nowUs = ALooper::GetNowUs(); 1196 if (mNextAudioClockUpdateTimeUs >= 0) { 1197 if (nowUs >= mNextAudioClockUpdateTimeUs) { 1198 int64_t nowMediaUs = mediaTimeUs - getPendingAudioPlayoutDurationUs(nowUs); 1199 mMediaClock->updateAnchor(nowMediaUs, nowUs, mediaTimeUs); 1200 mUseVirtualAudioSink = false; 1201 mNextAudioClockUpdateTimeUs = nowUs + kMinimumAudioClockUpdatePeriodUs; 1202 } 1203 } else { 1204 int64_t unused; 1205 if ((mMediaClock->getMediaTime(nowUs, &unused) != OK) 1206 && (getDurationUsIfPlayedAtSampleRate(mNumFramesWritten) 1207 > kMaxAllowedAudioSinkDelayUs)) { 1208 // Enough data has been sent to AudioSink, but AudioSink has not rendered 1209 // any data yet. Something is wrong with AudioSink, e.g., the device is not 1210 // connected to audio out. 1211 // Switch to system clock. This essentially creates a virtual AudioSink with 1212 // initial latenty of getDurationUsIfPlayedAtSampleRate(mNumFramesWritten). 1213 // This virtual AudioSink renders audio data starting from the very first sample 1214 // and it's paced by system clock. 1215 ALOGW("AudioSink stuck. ARE YOU CONNECTED TO AUDIO OUT? Switching to system clock."); 1216 mMediaClock->updateAnchor(mAudioFirstAnchorTimeMediaUs, nowUs, mediaTimeUs); 1217 mUseVirtualAudioSink = true; 1218 } 1219 } 1220 mAnchorNumFramesWritten = mNumFramesWritten; 1221 mAnchorTimeMediaUs = mediaTimeUs; 1222 } 1223 1224 // Called without mLock acquired. 1225 void NuPlayer::Renderer::postDrainVideoQueue() { 1226 if (mDrainVideoQueuePending 1227 || getSyncQueues() 1228 || (mPaused && mVideoSampleReceived)) { 1229 return; 1230 } 1231 1232 if (mVideoQueue.empty()) { 1233 return; 1234 } 1235 1236 QueueEntry &entry = *mVideoQueue.begin(); 1237 1238 sp<AMessage> msg = new AMessage(kWhatDrainVideoQueue, this); 1239 msg->setInt32("drainGeneration", getDrainGeneration(false /* audio */)); 1240 1241 if (entry.mBuffer == NULL) { 1242 // EOS doesn't carry a timestamp. 1243 msg->post(); 1244 mDrainVideoQueuePending = true; 1245 return; 1246 } 1247 1248 bool needRepostDrainVideoQueue = false; 1249 int64_t delayUs; 1250 int64_t nowUs = ALooper::GetNowUs(); 1251 int64_t realTimeUs; 1252 if (mFlags & FLAG_REAL_TIME) { 1253 int64_t mediaTimeUs; 1254 CHECK(entry.mBuffer->meta()->findInt64("timeUs", &mediaTimeUs)); 1255 realTimeUs = mediaTimeUs; 1256 } else { 1257 int64_t mediaTimeUs; 1258 CHECK(entry.mBuffer->meta()->findInt64("timeUs", &mediaTimeUs)); 1259 1260 { 1261 Mutex::Autolock autoLock(mLock); 1262 if (mAnchorTimeMediaUs < 0) { 1263 mMediaClock->updateAnchor(mediaTimeUs, nowUs, mediaTimeUs); 1264 mAnchorTimeMediaUs = mediaTimeUs; 1265 realTimeUs = nowUs; 1266 } else if (!mVideoSampleReceived) { 1267 // Always render the first video frame. 1268 realTimeUs = nowUs; 1269 } else if (mAudioFirstAnchorTimeMediaUs < 0 1270 || mMediaClock->getRealTimeFor(mediaTimeUs, &realTimeUs) == OK) { 1271 realTimeUs = getRealTimeUs(mediaTimeUs, nowUs); 1272 } else if (mediaTimeUs - mAudioFirstAnchorTimeMediaUs >= 0) { 1273 needRepostDrainVideoQueue = true; 1274 realTimeUs = nowUs; 1275 } else { 1276 realTimeUs = nowUs; 1277 } 1278 } 1279 if (!mHasAudio) { 1280 // smooth out videos >= 10fps 1281 mMediaClock->updateMaxTimeMedia(mediaTimeUs + 100000); 1282 } 1283 1284 // Heuristics to handle situation when media time changed without a 1285 // discontinuity. If we have not drained an audio buffer that was 1286 // received after this buffer, repost in 10 msec. Otherwise repost 1287 // in 500 msec. 1288 delayUs = realTimeUs - nowUs; 1289 int64_t postDelayUs = -1; 1290 if (delayUs > 500000) { 1291 postDelayUs = 500000; 1292 if (mHasAudio && (mLastAudioBufferDrained - entry.mBufferOrdinal) <= 0) { 1293 postDelayUs = 10000; 1294 } 1295 } else if (needRepostDrainVideoQueue) { 1296 // CHECK(mPlaybackRate > 0); 1297 // CHECK(mAudioFirstAnchorTimeMediaUs >= 0); 1298 // CHECK(mediaTimeUs - mAudioFirstAnchorTimeMediaUs >= 0); 1299 postDelayUs = mediaTimeUs - mAudioFirstAnchorTimeMediaUs; 1300 postDelayUs /= mPlaybackRate; 1301 } 1302 1303 if (postDelayUs >= 0) { 1304 msg->setWhat(kWhatPostDrainVideoQueue); 1305 msg->post(postDelayUs); 1306 mVideoScheduler->restart(); 1307 ALOGI("possible video time jump of %dms (%lld : %lld) or uninitialized media clock," 1308 " retrying in %dms", 1309 (int)(delayUs / 1000), (long long)mediaTimeUs, 1310 (long long)mAudioFirstAnchorTimeMediaUs, (int)(postDelayUs / 1000)); 1311 mDrainVideoQueuePending = true; 1312 return; 1313 } 1314 } 1315 1316 realTimeUs = mVideoScheduler->schedule(realTimeUs * 1000) / 1000; 1317 int64_t twoVsyncsUs = 2 * (mVideoScheduler->getVsyncPeriod() / 1000); 1318 1319 delayUs = realTimeUs - nowUs; 1320 1321 ALOGW_IF(delayUs > 500000, "unusually high delayUs: %" PRId64, delayUs); 1322 // post 2 display refreshes before rendering is due 1323 msg->post(delayUs > twoVsyncsUs ? delayUs - twoVsyncsUs : 0); 1324 1325 mDrainVideoQueuePending = true; 1326 } 1327 1328 void NuPlayer::Renderer::onDrainVideoQueue() { 1329 if (mVideoQueue.empty()) { 1330 return; 1331 } 1332 1333 QueueEntry *entry = &*mVideoQueue.begin(); 1334 1335 if (entry->mBuffer == NULL) { 1336 // EOS 1337 1338 notifyEOS(false /* audio */, entry->mFinalResult); 1339 1340 mVideoQueue.erase(mVideoQueue.begin()); 1341 entry = NULL; 1342 1343 setVideoLateByUs(0); 1344 return; 1345 } 1346 1347 int64_t nowUs = ALooper::GetNowUs(); 1348 int64_t realTimeUs; 1349 int64_t mediaTimeUs = -1; 1350 if (mFlags & FLAG_REAL_TIME) { 1351 CHECK(entry->mBuffer->meta()->findInt64("timeUs", &realTimeUs)); 1352 } else { 1353 CHECK(entry->mBuffer->meta()->findInt64("timeUs", &mediaTimeUs)); 1354 1355 realTimeUs = getRealTimeUs(mediaTimeUs, nowUs); 1356 } 1357 1358 bool tooLate = false; 1359 1360 if (!mPaused) { 1361 setVideoLateByUs(nowUs - realTimeUs); 1362 tooLate = (mVideoLateByUs > 40000); 1363 1364 if (tooLate) { 1365 ALOGV("video late by %lld us (%.2f secs)", 1366 (long long)mVideoLateByUs, mVideoLateByUs / 1E6); 1367 } else { 1368 int64_t mediaUs = 0; 1369 mMediaClock->getMediaTime(realTimeUs, &mediaUs); 1370 ALOGV("rendering video at media time %.2f secs", 1371 (mFlags & FLAG_REAL_TIME ? realTimeUs : 1372 mediaUs) / 1E6); 1373 1374 if (!(mFlags & FLAG_REAL_TIME) 1375 && mLastAudioMediaTimeUs != -1 1376 && mediaTimeUs > mLastAudioMediaTimeUs) { 1377 // If audio ends before video, video continues to drive media clock. 1378 // Also smooth out videos >= 10fps. 1379 mMediaClock->updateMaxTimeMedia(mediaTimeUs + 100000); 1380 } 1381 } 1382 } else { 1383 setVideoLateByUs(0); 1384 if (!mVideoSampleReceived && !mHasAudio) { 1385 // This will ensure that the first frame after a flush won't be used as anchor 1386 // when renderer is in paused state, because resume can happen any time after seek. 1387 clearAnchorTime(); 1388 } 1389 } 1390 1391 // Always render the first video frame while keeping stats on A/V sync. 1392 if (!mVideoSampleReceived) { 1393 realTimeUs = nowUs; 1394 tooLate = false; 1395 } 1396 1397 entry->mNotifyConsumed->setInt64("timestampNs", realTimeUs * 1000ll); 1398 entry->mNotifyConsumed->setInt32("render", !tooLate); 1399 entry->mNotifyConsumed->post(); 1400 mVideoQueue.erase(mVideoQueue.begin()); 1401 entry = NULL; 1402 1403 mVideoSampleReceived = true; 1404 1405 if (!mPaused) { 1406 if (!mVideoRenderingStarted) { 1407 mVideoRenderingStarted = true; 1408 notifyVideoRenderingStart(); 1409 } 1410 Mutex::Autolock autoLock(mLock); 1411 notifyIfMediaRenderingStarted_l(); 1412 } 1413 } 1414 1415 void NuPlayer::Renderer::notifyVideoRenderingStart() { 1416 sp<AMessage> notify = mNotify->dup(); 1417 notify->setInt32("what", kWhatVideoRenderingStart); 1418 notify->post(); 1419 } 1420 1421 void NuPlayer::Renderer::notifyEOS(bool audio, status_t finalResult, int64_t delayUs) { 1422 Mutex::Autolock autoLock(mLock); 1423 notifyEOS_l(audio, finalResult, delayUs); 1424 } 1425 1426 void NuPlayer::Renderer::notifyEOS_l(bool audio, status_t finalResult, int64_t delayUs) { 1427 if (audio && delayUs > 0) { 1428 sp<AMessage> msg = new AMessage(kWhatEOS, this); 1429 msg->setInt32("audioEOSGeneration", mAudioEOSGeneration); 1430 msg->setInt32("finalResult", finalResult); 1431 msg->post(delayUs); 1432 return; 1433 } 1434 sp<AMessage> notify = mNotify->dup(); 1435 notify->setInt32("what", kWhatEOS); 1436 notify->setInt32("audio", static_cast<int32_t>(audio)); 1437 notify->setInt32("finalResult", finalResult); 1438 notify->post(delayUs); 1439 1440 if (audio) { 1441 // Video might outlive audio. Clear anchor to enable video only case. 1442 mAnchorTimeMediaUs = -1; 1443 } 1444 } 1445 1446 void NuPlayer::Renderer::notifyAudioTearDown(AudioTearDownReason reason) { 1447 sp<AMessage> msg = new AMessage(kWhatAudioTearDown, this); 1448 msg->setInt32("reason", reason); 1449 msg->post(); 1450 } 1451 1452 void NuPlayer::Renderer::onQueueBuffer(const sp<AMessage> &msg) { 1453 int32_t audio; 1454 CHECK(msg->findInt32("audio", &audio)); 1455 1456 if (dropBufferIfStale(audio, msg)) { 1457 return; 1458 } 1459 1460 if (audio) { 1461 mHasAudio = true; 1462 } else { 1463 mHasVideo = true; 1464 } 1465 1466 if (mHasVideo) { 1467 if (mVideoScheduler == NULL) { 1468 mVideoScheduler = new VideoFrameScheduler(); 1469 mVideoScheduler->init(); 1470 } 1471 } 1472 1473 sp<RefBase> obj; 1474 CHECK(msg->findObject("buffer", &obj)); 1475 sp<MediaCodecBuffer> buffer = static_cast<MediaCodecBuffer *>(obj.get()); 1476 1477 sp<AMessage> notifyConsumed; 1478 CHECK(msg->findMessage("notifyConsumed", ¬ifyConsumed)); 1479 1480 QueueEntry entry; 1481 entry.mBuffer = buffer; 1482 entry.mNotifyConsumed = notifyConsumed; 1483 entry.mOffset = 0; 1484 entry.mFinalResult = OK; 1485 entry.mBufferOrdinal = ++mTotalBuffersQueued; 1486 1487 if (audio) { 1488 Mutex::Autolock autoLock(mLock); 1489 mAudioQueue.push_back(entry); 1490 postDrainAudioQueue_l(); 1491 } else { 1492 mVideoQueue.push_back(entry); 1493 postDrainVideoQueue(); 1494 } 1495 1496 Mutex::Autolock autoLock(mLock); 1497 if (!mSyncQueues || mAudioQueue.empty() || mVideoQueue.empty()) { 1498 return; 1499 } 1500 1501 sp<MediaCodecBuffer> firstAudioBuffer = (*mAudioQueue.begin()).mBuffer; 1502 sp<MediaCodecBuffer> firstVideoBuffer = (*mVideoQueue.begin()).mBuffer; 1503 1504 if (firstAudioBuffer == NULL || firstVideoBuffer == NULL) { 1505 // EOS signalled on either queue. 1506 syncQueuesDone_l(); 1507 return; 1508 } 1509 1510 int64_t firstAudioTimeUs; 1511 int64_t firstVideoTimeUs; 1512 CHECK(firstAudioBuffer->meta() 1513 ->findInt64("timeUs", &firstAudioTimeUs)); 1514 CHECK(firstVideoBuffer->meta() 1515 ->findInt64("timeUs", &firstVideoTimeUs)); 1516 1517 int64_t diff = firstVideoTimeUs - firstAudioTimeUs; 1518 1519 ALOGV("queueDiff = %.2f secs", diff / 1E6); 1520 1521 if (diff > 100000ll) { 1522 // Audio data starts More than 0.1 secs before video. 1523 // Drop some audio. 1524 1525 (*mAudioQueue.begin()).mNotifyConsumed->post(); 1526 mAudioQueue.erase(mAudioQueue.begin()); 1527 return; 1528 } 1529 1530 syncQueuesDone_l(); 1531 } 1532 1533 void NuPlayer::Renderer::syncQueuesDone_l() { 1534 if (!mSyncQueues) { 1535 return; 1536 } 1537 1538 mSyncQueues = false; 1539 1540 if (!mAudioQueue.empty()) { 1541 postDrainAudioQueue_l(); 1542 } 1543 1544 if (!mVideoQueue.empty()) { 1545 mLock.unlock(); 1546 postDrainVideoQueue(); 1547 mLock.lock(); 1548 } 1549 } 1550 1551 void NuPlayer::Renderer::onQueueEOS(const sp<AMessage> &msg) { 1552 int32_t audio; 1553 CHECK(msg->findInt32("audio", &audio)); 1554 1555 if (dropBufferIfStale(audio, msg)) { 1556 return; 1557 } 1558 1559 int32_t finalResult; 1560 CHECK(msg->findInt32("finalResult", &finalResult)); 1561 1562 QueueEntry entry; 1563 entry.mOffset = 0; 1564 entry.mFinalResult = finalResult; 1565 1566 if (audio) { 1567 Mutex::Autolock autoLock(mLock); 1568 if (mAudioQueue.empty() && mSyncQueues) { 1569 syncQueuesDone_l(); 1570 } 1571 mAudioQueue.push_back(entry); 1572 postDrainAudioQueue_l(); 1573 } else { 1574 if (mVideoQueue.empty() && getSyncQueues()) { 1575 Mutex::Autolock autoLock(mLock); 1576 syncQueuesDone_l(); 1577 } 1578 mVideoQueue.push_back(entry); 1579 postDrainVideoQueue(); 1580 } 1581 } 1582 1583 void NuPlayer::Renderer::onFlush(const sp<AMessage> &msg) { 1584 int32_t audio, notifyComplete; 1585 CHECK(msg->findInt32("audio", &audio)); 1586 1587 { 1588 Mutex::Autolock autoLock(mLock); 1589 if (audio) { 1590 notifyComplete = mNotifyCompleteAudio; 1591 mNotifyCompleteAudio = false; 1592 mLastAudioMediaTimeUs = -1; 1593 } else { 1594 notifyComplete = mNotifyCompleteVideo; 1595 mNotifyCompleteVideo = false; 1596 } 1597 1598 // If we're currently syncing the queues, i.e. dropping audio while 1599 // aligning the first audio/video buffer times and only one of the 1600 // two queues has data, we may starve that queue by not requesting 1601 // more buffers from the decoder. If the other source then encounters 1602 // a discontinuity that leads to flushing, we'll never find the 1603 // corresponding discontinuity on the other queue. 1604 // Therefore we'll stop syncing the queues if at least one of them 1605 // is flushed. 1606 syncQueuesDone_l(); 1607 } 1608 clearAnchorTime(); 1609 1610 ALOGV("flushing %s", audio ? "audio" : "video"); 1611 if (audio) { 1612 { 1613 Mutex::Autolock autoLock(mLock); 1614 flushQueue(&mAudioQueue); 1615 1616 ++mAudioDrainGeneration; 1617 ++mAudioEOSGeneration; 1618 prepareForMediaRenderingStart_l(); 1619 1620 // the frame count will be reset after flush. 1621 clearAudioFirstAnchorTime_l(); 1622 } 1623 1624 mDrainAudioQueuePending = false; 1625 1626 if (offloadingAudio()) { 1627 mAudioSink->pause(); 1628 mAudioSink->flush(); 1629 if (!mPaused) { 1630 mAudioSink->start(); 1631 } 1632 } else { 1633 mAudioSink->pause(); 1634 mAudioSink->flush(); 1635 // Call stop() to signal to the AudioSink to completely fill the 1636 // internal buffer before resuming playback. 1637 // FIXME: this is ignored after flush(). 1638 mAudioSink->stop(); 1639 if (mPaused) { 1640 // Race condition: if renderer is paused and audio sink is stopped, 1641 // we need to make sure that the audio track buffer fully drains 1642 // before delivering data. 1643 // FIXME: remove this if we can detect if stop() is complete. 1644 const int delayUs = 2 * 50 * 1000; // (2 full mixer thread cycles at 50ms) 1645 mPauseDrainAudioAllowedUs = ALooper::GetNowUs() + delayUs; 1646 } else { 1647 mAudioSink->start(); 1648 } 1649 mNumFramesWritten = 0; 1650 } 1651 mNextAudioClockUpdateTimeUs = -1; 1652 } else { 1653 flushQueue(&mVideoQueue); 1654 1655 mDrainVideoQueuePending = false; 1656 1657 if (mVideoScheduler != NULL) { 1658 mVideoScheduler->restart(); 1659 } 1660 1661 Mutex::Autolock autoLock(mLock); 1662 ++mVideoDrainGeneration; 1663 prepareForMediaRenderingStart_l(); 1664 } 1665 1666 mVideoSampleReceived = false; 1667 1668 if (notifyComplete) { 1669 notifyFlushComplete(audio); 1670 } 1671 } 1672 1673 void NuPlayer::Renderer::flushQueue(List<QueueEntry> *queue) { 1674 while (!queue->empty()) { 1675 QueueEntry *entry = &*queue->begin(); 1676 1677 if (entry->mBuffer != NULL) { 1678 entry->mNotifyConsumed->post(); 1679 } else if (entry->mNotifyConsumed != nullptr) { 1680 // Is it needed to open audio sink now? 1681 onChangeAudioFormat(entry->mMeta, entry->mNotifyConsumed); 1682 } 1683 1684 queue->erase(queue->begin()); 1685 entry = NULL; 1686 } 1687 } 1688 1689 void NuPlayer::Renderer::notifyFlushComplete(bool audio) { 1690 sp<AMessage> notify = mNotify->dup(); 1691 notify->setInt32("what", kWhatFlushComplete); 1692 notify->setInt32("audio", static_cast<int32_t>(audio)); 1693 notify->post(); 1694 } 1695 1696 bool NuPlayer::Renderer::dropBufferIfStale( 1697 bool audio, const sp<AMessage> &msg) { 1698 int32_t queueGeneration; 1699 CHECK(msg->findInt32("queueGeneration", &queueGeneration)); 1700 1701 if (queueGeneration == getQueueGeneration(audio)) { 1702 return false; 1703 } 1704 1705 sp<AMessage> notifyConsumed; 1706 if (msg->findMessage("notifyConsumed", ¬ifyConsumed)) { 1707 notifyConsumed->post(); 1708 } 1709 1710 return true; 1711 } 1712 1713 void NuPlayer::Renderer::onAudioSinkChanged() { 1714 if (offloadingAudio()) { 1715 return; 1716 } 1717 CHECK(!mDrainAudioQueuePending); 1718 mNumFramesWritten = 0; 1719 mAnchorNumFramesWritten = -1; 1720 uint32_t written; 1721 if (mAudioSink->getFramesWritten(&written) == OK) { 1722 mNumFramesWritten = written; 1723 } 1724 } 1725 1726 void NuPlayer::Renderer::onDisableOffloadAudio() { 1727 Mutex::Autolock autoLock(mLock); 1728 mFlags &= ~FLAG_OFFLOAD_AUDIO; 1729 ++mAudioDrainGeneration; 1730 if (mAudioRenderingStartGeneration != -1) { 1731 prepareForMediaRenderingStart_l(); 1732 } 1733 } 1734 1735 void NuPlayer::Renderer::onEnableOffloadAudio() { 1736 Mutex::Autolock autoLock(mLock); 1737 mFlags |= FLAG_OFFLOAD_AUDIO; 1738 ++mAudioDrainGeneration; 1739 if (mAudioRenderingStartGeneration != -1) { 1740 prepareForMediaRenderingStart_l(); 1741 } 1742 } 1743 1744 void NuPlayer::Renderer::onPause() { 1745 if (mPaused) { 1746 return; 1747 } 1748 1749 { 1750 Mutex::Autolock autoLock(mLock); 1751 // we do not increment audio drain generation so that we fill audio buffer during pause. 1752 ++mVideoDrainGeneration; 1753 prepareForMediaRenderingStart_l(); 1754 mPaused = true; 1755 mMediaClock->setPlaybackRate(0.0); 1756 } 1757 1758 mDrainAudioQueuePending = false; 1759 mDrainVideoQueuePending = false; 1760 1761 // Note: audio data may not have been decoded, and the AudioSink may not be opened. 1762 mAudioSink->pause(); 1763 startAudioOffloadPauseTimeout(); 1764 1765 ALOGV("now paused audio queue has %zu entries, video has %zu entries", 1766 mAudioQueue.size(), mVideoQueue.size()); 1767 } 1768 1769 void NuPlayer::Renderer::onResume() { 1770 if (!mPaused) { 1771 return; 1772 } 1773 1774 // Note: audio data may not have been decoded, and the AudioSink may not be opened. 1775 cancelAudioOffloadPauseTimeout(); 1776 if (mAudioSink->ready()) { 1777 status_t err = mAudioSink->start(); 1778 if (err != OK) { 1779 ALOGE("cannot start AudioSink err %d", err); 1780 notifyAudioTearDown(kDueToError); 1781 } 1782 } 1783 1784 { 1785 Mutex::Autolock autoLock(mLock); 1786 mPaused = false; 1787 // rendering started message may have been delayed if we were paused. 1788 if (mRenderingDataDelivered) { 1789 notifyIfMediaRenderingStarted_l(); 1790 } 1791 // configure audiosink as we did not do it when pausing 1792 if (mAudioSink != NULL && mAudioSink->ready()) { 1793 mAudioSink->setPlaybackRate(mPlaybackSettings); 1794 } 1795 1796 mMediaClock->setPlaybackRate(mPlaybackRate); 1797 1798 if (!mAudioQueue.empty()) { 1799 postDrainAudioQueue_l(); 1800 } 1801 } 1802 1803 if (!mVideoQueue.empty()) { 1804 postDrainVideoQueue(); 1805 } 1806 } 1807 1808 void NuPlayer::Renderer::onSetVideoFrameRate(float fps) { 1809 if (mVideoScheduler == NULL) { 1810 mVideoScheduler = new VideoFrameScheduler(); 1811 } 1812 mVideoScheduler->init(fps); 1813 } 1814 1815 int32_t NuPlayer::Renderer::getQueueGeneration(bool audio) { 1816 Mutex::Autolock autoLock(mLock); 1817 return (audio ? mAudioQueueGeneration : mVideoQueueGeneration); 1818 } 1819 1820 int32_t NuPlayer::Renderer::getDrainGeneration(bool audio) { 1821 Mutex::Autolock autoLock(mLock); 1822 return (audio ? mAudioDrainGeneration : mVideoDrainGeneration); 1823 } 1824 1825 bool NuPlayer::Renderer::getSyncQueues() { 1826 Mutex::Autolock autoLock(mLock); 1827 return mSyncQueues; 1828 } 1829 1830 void NuPlayer::Renderer::onAudioTearDown(AudioTearDownReason reason) { 1831 if (mAudioTornDown) { 1832 return; 1833 } 1834 mAudioTornDown = true; 1835 1836 int64_t currentPositionUs; 1837 sp<AMessage> notify = mNotify->dup(); 1838 if (getCurrentPosition(¤tPositionUs) == OK) { 1839 notify->setInt64("positionUs", currentPositionUs); 1840 } 1841 1842 mAudioSink->stop(); 1843 mAudioSink->flush(); 1844 1845 notify->setInt32("what", kWhatAudioTearDown); 1846 notify->setInt32("reason", reason); 1847 notify->post(); 1848 } 1849 1850 void NuPlayer::Renderer::startAudioOffloadPauseTimeout() { 1851 if (offloadingAudio()) { 1852 mWakeLock->acquire(); 1853 sp<AMessage> msg = new AMessage(kWhatAudioOffloadPauseTimeout, this); 1854 msg->setInt32("drainGeneration", mAudioOffloadPauseTimeoutGeneration); 1855 msg->post(kOffloadPauseMaxUs); 1856 } 1857 } 1858 1859 void NuPlayer::Renderer::cancelAudioOffloadPauseTimeout() { 1860 // We may have called startAudioOffloadPauseTimeout() without 1861 // the AudioSink open and with offloadingAudio enabled. 1862 // 1863 // When we cancel, it may be that offloadingAudio is subsequently disabled, so regardless 1864 // we always release the wakelock and increment the pause timeout generation. 1865 // 1866 // Note: The acquired wakelock prevents the device from suspending 1867 // immediately after offload pause (in case a resume happens shortly thereafter). 1868 mWakeLock->release(true); 1869 ++mAudioOffloadPauseTimeoutGeneration; 1870 } 1871 1872 status_t NuPlayer::Renderer::onOpenAudioSink( 1873 const sp<AMessage> &format, 1874 bool offloadOnly, 1875 bool hasVideo, 1876 uint32_t flags, 1877 bool isStreaming) { 1878 ALOGV("openAudioSink: offloadOnly(%d) offloadingAudio(%d)", 1879 offloadOnly, offloadingAudio()); 1880 bool audioSinkChanged = false; 1881 1882 int32_t numChannels; 1883 CHECK(format->findInt32("channel-count", &numChannels)); 1884 1885 int32_t channelMask; 1886 if (!format->findInt32("channel-mask", &channelMask)) { 1887 // signal to the AudioSink to derive the mask from count. 1888 channelMask = CHANNEL_MASK_USE_CHANNEL_ORDER; 1889 } 1890 1891 int32_t sampleRate; 1892 CHECK(format->findInt32("sample-rate", &sampleRate)); 1893 1894 if (offloadingAudio()) { 1895 audio_format_t audioFormat = AUDIO_FORMAT_PCM_16_BIT; 1896 AString mime; 1897 CHECK(format->findString("mime", &mime)); 1898 status_t err = mapMimeToAudioFormat(audioFormat, mime.c_str()); 1899 1900 if (err != OK) { 1901 ALOGE("Couldn't map mime \"%s\" to a valid " 1902 "audio_format", mime.c_str()); 1903 onDisableOffloadAudio(); 1904 } else { 1905 ALOGV("Mime \"%s\" mapped to audio_format 0x%x", 1906 mime.c_str(), audioFormat); 1907 1908 int avgBitRate = -1; 1909 format->findInt32("bitrate", &avgBitRate); 1910 1911 int32_t aacProfile = -1; 1912 if (audioFormat == AUDIO_FORMAT_AAC 1913 && format->findInt32("aac-profile", &aacProfile)) { 1914 // Redefine AAC format as per aac profile 1915 mapAACProfileToAudioFormat( 1916 audioFormat, 1917 aacProfile); 1918 } 1919 1920 audio_offload_info_t offloadInfo = AUDIO_INFO_INITIALIZER; 1921 offloadInfo.duration_us = -1; 1922 format->findInt64( 1923 "durationUs", &offloadInfo.duration_us); 1924 offloadInfo.sample_rate = sampleRate; 1925 offloadInfo.channel_mask = channelMask; 1926 offloadInfo.format = audioFormat; 1927 offloadInfo.stream_type = AUDIO_STREAM_MUSIC; 1928 offloadInfo.bit_rate = avgBitRate; 1929 offloadInfo.has_video = hasVideo; 1930 offloadInfo.is_streaming = isStreaming; 1931 1932 if (memcmp(&mCurrentOffloadInfo, &offloadInfo, sizeof(offloadInfo)) == 0) { 1933 ALOGV("openAudioSink: no change in offload mode"); 1934 // no change from previous configuration, everything ok. 1935 return OK; 1936 } 1937 mCurrentPcmInfo = AUDIO_PCMINFO_INITIALIZER; 1938 1939 ALOGV("openAudioSink: try to open AudioSink in offload mode"); 1940 uint32_t offloadFlags = flags; 1941 offloadFlags |= AUDIO_OUTPUT_FLAG_COMPRESS_OFFLOAD; 1942 offloadFlags &= ~AUDIO_OUTPUT_FLAG_DEEP_BUFFER; 1943 audioSinkChanged = true; 1944 mAudioSink->close(); 1945 1946 err = mAudioSink->open( 1947 sampleRate, 1948 numChannels, 1949 (audio_channel_mask_t)channelMask, 1950 audioFormat, 1951 0 /* bufferCount - unused */, 1952 &NuPlayer::Renderer::AudioSinkCallback, 1953 this, 1954 (audio_output_flags_t)offloadFlags, 1955 &offloadInfo); 1956 1957 if (err == OK) { 1958 err = mAudioSink->setPlaybackRate(mPlaybackSettings); 1959 } 1960 1961 if (err == OK) { 1962 // If the playback is offloaded to h/w, we pass 1963 // the HAL some metadata information. 1964 // We don't want to do this for PCM because it 1965 // will be going through the AudioFlinger mixer 1966 // before reaching the hardware. 1967 // TODO 1968 mCurrentOffloadInfo = offloadInfo; 1969 if (!mPaused) { // for preview mode, don't start if paused 1970 err = mAudioSink->start(); 1971 } 1972 ALOGV_IF(err == OK, "openAudioSink: offload succeeded"); 1973 } 1974 if (err != OK) { 1975 // Clean up, fall back to non offload mode. 1976 mAudioSink->close(); 1977 onDisableOffloadAudio(); 1978 mCurrentOffloadInfo = AUDIO_INFO_INITIALIZER; 1979 ALOGV("openAudioSink: offload failed"); 1980 if (offloadOnly) { 1981 notifyAudioTearDown(kForceNonOffload); 1982 } 1983 } else { 1984 mUseAudioCallback = true; // offload mode transfers data through callback 1985 ++mAudioDrainGeneration; // discard pending kWhatDrainAudioQueue message. 1986 } 1987 } 1988 } 1989 if (!offloadOnly && !offloadingAudio()) { 1990 ALOGV("openAudioSink: open AudioSink in NON-offload mode"); 1991 uint32_t pcmFlags = flags; 1992 pcmFlags &= ~AUDIO_OUTPUT_FLAG_COMPRESS_OFFLOAD; 1993 1994 const PcmInfo info = { 1995 (audio_channel_mask_t)channelMask, 1996 (audio_output_flags_t)pcmFlags, 1997 AUDIO_FORMAT_PCM_16_BIT, // TODO: change to audioFormat 1998 numChannels, 1999 sampleRate 2000 }; 2001 if (memcmp(&mCurrentPcmInfo, &info, sizeof(info)) == 0) { 2002 ALOGV("openAudioSink: no change in pcm mode"); 2003 // no change from previous configuration, everything ok. 2004 return OK; 2005 } 2006 2007 audioSinkChanged = true; 2008 mAudioSink->close(); 2009 mCurrentOffloadInfo = AUDIO_INFO_INITIALIZER; 2010 // Note: It is possible to set up the callback, but not use it to send audio data. 2011 // This requires a fix in AudioSink to explicitly specify the transfer mode. 2012 mUseAudioCallback = getUseAudioCallbackSetting(); 2013 if (mUseAudioCallback) { 2014 ++mAudioDrainGeneration; // discard pending kWhatDrainAudioQueue message. 2015 } 2016 2017 // Compute the desired buffer size. 2018 // For callback mode, the amount of time before wakeup is about half the buffer size. 2019 const uint32_t frameCount = 2020 (unsigned long long)sampleRate * getAudioSinkPcmMsSetting() / 1000; 2021 2022 // The doNotReconnect means AudioSink will signal back and let NuPlayer to re-construct 2023 // AudioSink. We don't want this when there's video because it will cause a video seek to 2024 // the previous I frame. But we do want this when there's only audio because it will give 2025 // NuPlayer a chance to switch from non-offload mode to offload mode. 2026 // So we only set doNotReconnect when there's no video. 2027 const bool doNotReconnect = !hasVideo; 2028 2029 // We should always be able to set our playback settings if the sink is closed. 2030 LOG_ALWAYS_FATAL_IF(mAudioSink->setPlaybackRate(mPlaybackSettings) != OK, 2031 "onOpenAudioSink: can't set playback rate on closed sink"); 2032 status_t err = mAudioSink->open( 2033 sampleRate, 2034 numChannels, 2035 (audio_channel_mask_t)channelMask, 2036 AUDIO_FORMAT_PCM_16_BIT, 2037 0 /* bufferCount - unused */, 2038 mUseAudioCallback ? &NuPlayer::Renderer::AudioSinkCallback : NULL, 2039 mUseAudioCallback ? this : NULL, 2040 (audio_output_flags_t)pcmFlags, 2041 NULL, 2042 doNotReconnect, 2043 frameCount); 2044 if (err != OK) { 2045 ALOGW("openAudioSink: non offloaded open failed status: %d", err); 2046 mAudioSink->close(); 2047 mCurrentPcmInfo = AUDIO_PCMINFO_INITIALIZER; 2048 return err; 2049 } 2050 mCurrentPcmInfo = info; 2051 if (!mPaused) { // for preview mode, don't start if paused 2052 mAudioSink->start(); 2053 } 2054 } 2055 if (audioSinkChanged) { 2056 onAudioSinkChanged(); 2057 } 2058 mAudioTornDown = false; 2059 return OK; 2060 } 2061 2062 void NuPlayer::Renderer::onCloseAudioSink() { 2063 mAudioSink->close(); 2064 mCurrentOffloadInfo = AUDIO_INFO_INITIALIZER; 2065 mCurrentPcmInfo = AUDIO_PCMINFO_INITIALIZER; 2066 } 2067 2068 void NuPlayer::Renderer::onChangeAudioFormat( 2069 const sp<AMessage> &meta, const sp<AMessage> ¬ify) { 2070 sp<AMessage> format; 2071 CHECK(meta->findMessage("format", &format)); 2072 2073 int32_t offloadOnly; 2074 CHECK(meta->findInt32("offload-only", &offloadOnly)); 2075 2076 int32_t hasVideo; 2077 CHECK(meta->findInt32("has-video", &hasVideo)); 2078 2079 uint32_t flags; 2080 CHECK(meta->findInt32("flags", (int32_t *)&flags)); 2081 2082 uint32_t isStreaming; 2083 CHECK(meta->findInt32("isStreaming", (int32_t *)&isStreaming)); 2084 2085 status_t err = onOpenAudioSink(format, offloadOnly, hasVideo, flags, isStreaming); 2086 2087 if (err != OK) { 2088 notify->setInt32("err", err); 2089 } 2090 notify->post(); 2091 } 2092 2093 } // namespace android 2094 2095