1 /* 2 * Copyright 2015 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17 //#define LOG_NDEBUG 0 18 #define LOG_TAG "MediaSync" 19 #include <inttypes.h> 20 21 #include <gui/BufferQueue.h> 22 #include <gui/IGraphicBufferConsumer.h> 23 #include <gui/IGraphicBufferProducer.h> 24 25 #include <media/AudioTrack.h> 26 #include <media/stagefright/MediaClock.h> 27 #include <media/stagefright/MediaSync.h> 28 #include <media/stagefright/VideoFrameScheduler.h> 29 #include <media/stagefright/foundation/ADebug.h> 30 #include <media/stagefright/foundation/ALooper.h> 31 #include <media/stagefright/foundation/AMessage.h> 32 33 #include <ui/GraphicBuffer.h> 34 35 // Maximum late time allowed for a video frame to be rendered. When a video 36 // frame arrives later than this number, it will be discarded without rendering. 37 static const int64_t kMaxAllowedVideoLateTimeUs = 40000ll; 38 39 namespace android { 40 41 // static 42 sp<MediaSync> MediaSync::create() { 43 sp<MediaSync> sync = new MediaSync(); 44 sync->mLooper->registerHandler(sync); 45 return sync; 46 } 47 48 MediaSync::MediaSync() 49 : mIsAbandoned(false), 50 mMutex(), 51 mReleaseCondition(), 52 mNumOutstandingBuffers(0), 53 mUsageFlagsFromOutput(0), 54 mMaxAcquiredBufferCount(1), 55 mReturnPendingInputFrame(false), 56 mNativeSampleRateInHz(0), 57 mNumFramesWritten(0), 58 mHasAudio(false), 59 mNextBufferItemMediaUs(-1), 60 mPlaybackRate(0.0) { 61 mMediaClock = new MediaClock; 62 63 // initialize settings 64 mPlaybackSettings = AUDIO_PLAYBACK_RATE_DEFAULT; 65 mPlaybackSettings.mSpeed = mPlaybackRate; 66 67 mLooper = new ALooper; 68 mLooper->setName("MediaSync"); 69 mLooper->start(false, false, ANDROID_PRIORITY_AUDIO); 70 } 71 72 MediaSync::~MediaSync() { 73 if (mInput != NULL) { 74 mInput->consumerDisconnect(); 75 } 76 if (mOutput != NULL) { 77 mOutput->disconnect(NATIVE_WINDOW_API_MEDIA); 78 } 79 80 if (mLooper != NULL) { 81 mLooper->unregisterHandler(id()); 82 mLooper->stop(); 83 } 84 } 85 86 status_t MediaSync::setSurface(const sp<IGraphicBufferProducer> &output) { 87 Mutex::Autolock lock(mMutex); 88 89 if (output == mOutput) { 90 return NO_ERROR; // same output surface. 91 } 92 93 if (output == NULL && mSyncSettings.mSource == AVSYNC_SOURCE_VSYNC) { 94 ALOGE("setSurface: output surface is used as sync source and cannot be removed."); 95 return INVALID_OPERATION; 96 } 97 98 if (output != NULL) { 99 int newUsage = 0; 100 output->query(NATIVE_WINDOW_CONSUMER_USAGE_BITS, &newUsage); 101 102 // Check usage flags only when current output surface has been used to create input surface. 103 if (mOutput != NULL && mInput != NULL) { 104 int ignoredFlags = (GRALLOC_USAGE_HW_TEXTURE | GRALLOC_USAGE_HW_COMPOSER 105 | GRALLOC_USAGE_EXTERNAL_DISP); 106 // New output surface is not allowed to add new usage flag except ignored ones. 107 if ((newUsage & ~(mUsageFlagsFromOutput | ignoredFlags)) != 0) { 108 ALOGE("setSurface: new output surface has new usage flag not used by current one."); 109 return BAD_VALUE; 110 } 111 } 112 113 // Try to connect to new output surface. If failed, current output surface will not 114 // be changed. 115 IGraphicBufferProducer::QueueBufferOutput queueBufferOutput; 116 sp<OutputListener> listener(new OutputListener(this, output)); 117 IInterface::asBinder(output)->linkToDeath(listener); 118 status_t status = 119 output->connect(listener, 120 NATIVE_WINDOW_API_MEDIA, 121 true /* producerControlledByApp */, 122 &queueBufferOutput); 123 if (status != NO_ERROR) { 124 ALOGE("setSurface: failed to connect (%d)", status); 125 return status; 126 } 127 128 if (mFrameScheduler == NULL) { 129 mFrameScheduler = new VideoFrameScheduler(); 130 mFrameScheduler->init(); 131 } 132 } 133 134 if (mOutput != NULL) { 135 mOutput->disconnect(NATIVE_WINDOW_API_MEDIA); 136 while (!mBuffersSentToOutput.isEmpty()) { 137 returnBufferToInput_l(mBuffersSentToOutput.valueAt(0), Fence::NO_FENCE); 138 mBuffersSentToOutput.removeItemsAt(0); 139 } 140 } 141 142 mOutput = output; 143 144 return NO_ERROR; 145 } 146 147 // |audioTrack| is used only for querying information. 148 status_t MediaSync::setAudioTrack(const sp<AudioTrack> &audioTrack) { 149 Mutex::Autolock lock(mMutex); 150 151 // TODO: support audio track change. 152 if (mAudioTrack != NULL) { 153 ALOGE("setAudioTrack: audioTrack has already been configured."); 154 return INVALID_OPERATION; 155 } 156 157 if (audioTrack == NULL && mSyncSettings.mSource == AVSYNC_SOURCE_AUDIO) { 158 ALOGE("setAudioTrack: audioTrack is used as sync source and cannot be removed."); 159 return INVALID_OPERATION; 160 } 161 162 if (audioTrack != NULL) { 163 // check if audio track supports the playback settings 164 if (mPlaybackSettings.mSpeed != 0.f 165 && audioTrack->setPlaybackRate(mPlaybackSettings) != OK) { 166 ALOGE("playback settings are not supported by the audio track"); 167 return INVALID_OPERATION; 168 } 169 uint32_t nativeSampleRateInHz = audioTrack->getOriginalSampleRate(); 170 if (nativeSampleRateInHz <= 0) { 171 ALOGE("setAudioTrack: native sample rate should be positive."); 172 return BAD_VALUE; 173 } 174 mAudioTrack = audioTrack; 175 mNativeSampleRateInHz = nativeSampleRateInHz; 176 (void)setPlaybackSettings_l(mPlaybackSettings); 177 } 178 else { 179 mAudioTrack = NULL; 180 mNativeSampleRateInHz = 0; 181 } 182 183 // potentially resync to new source 184 resync_l(); 185 return OK; 186 } 187 188 status_t MediaSync::createInputSurface( 189 sp<IGraphicBufferProducer> *outBufferProducer) { 190 if (outBufferProducer == NULL) { 191 return BAD_VALUE; 192 } 193 194 Mutex::Autolock lock(mMutex); 195 196 if (mOutput == NULL) { 197 return NO_INIT; 198 } 199 200 if (mInput != NULL) { 201 return INVALID_OPERATION; 202 } 203 204 sp<IGraphicBufferProducer> bufferProducer; 205 sp<IGraphicBufferConsumer> bufferConsumer; 206 BufferQueue::createBufferQueue(&bufferProducer, &bufferConsumer); 207 208 sp<InputListener> listener(new InputListener(this)); 209 IInterface::asBinder(bufferConsumer)->linkToDeath(listener); 210 status_t status = 211 bufferConsumer->consumerConnect(listener, false /* controlledByApp */); 212 if (status == NO_ERROR) { 213 bufferConsumer->setConsumerName(String8("MediaSync")); 214 // propagate usage bits from output surface 215 mUsageFlagsFromOutput = 0; 216 mOutput->query(NATIVE_WINDOW_CONSUMER_USAGE_BITS, &mUsageFlagsFromOutput); 217 bufferConsumer->setConsumerUsageBits(mUsageFlagsFromOutput); 218 *outBufferProducer = bufferProducer; 219 mInput = bufferConsumer; 220 221 // set undequeued buffer count 222 int minUndequeuedBuffers; 223 mOutput->query(NATIVE_WINDOW_MIN_UNDEQUEUED_BUFFERS, &minUndequeuedBuffers); 224 mMaxAcquiredBufferCount = minUndequeuedBuffers; 225 bufferConsumer->setMaxAcquiredBufferCount(mMaxAcquiredBufferCount); 226 } 227 return status; 228 } 229 230 void MediaSync::resync_l() { 231 AVSyncSource src = mSyncSettings.mSource; 232 if (src == AVSYNC_SOURCE_DEFAULT) { 233 if (mAudioTrack != NULL) { 234 src = AVSYNC_SOURCE_AUDIO; 235 } else { 236 src = AVSYNC_SOURCE_SYSTEM_CLOCK; 237 } 238 } 239 240 // TODO: resync ourselves to the current clock (e.g. on sync source change) 241 updatePlaybackRate_l(mPlaybackRate); 242 } 243 244 void MediaSync::updatePlaybackRate_l(float rate) { 245 if (rate > mPlaybackRate) { 246 mNextBufferItemMediaUs = -1; 247 } 248 mPlaybackRate = rate; 249 // TODO: update frame scheduler with this info 250 mMediaClock->setPlaybackRate(rate); 251 onDrainVideo_l(); 252 } 253 254 sp<const MediaClock> MediaSync::getMediaClock() { 255 return mMediaClock; 256 } 257 258 status_t MediaSync::getPlayTimeForPendingAudioFrames(int64_t *outTimeUs) { 259 Mutex::Autolock lock(mMutex); 260 // User should check the playback rate if it doesn't want to receive a 261 // huge number for play time. 262 if (mPlaybackRate == 0.0f) { 263 *outTimeUs = INT64_MAX; 264 return OK; 265 } 266 267 uint32_t numFramesPlayed = 0; 268 if (mAudioTrack != NULL) { 269 status_t res = mAudioTrack->getPosition(&numFramesPlayed); 270 if (res != OK) { 271 return res; 272 } 273 } 274 275 int64_t numPendingFrames = mNumFramesWritten - numFramesPlayed; 276 if (numPendingFrames < 0) { 277 numPendingFrames = 0; 278 ALOGW("getPlayTimeForPendingAudioFrames: pending frame count is negative."); 279 } 280 double timeUs = numPendingFrames * 1000000.0 281 / (mNativeSampleRateInHz * (double)mPlaybackRate); 282 if (timeUs > (double)INT64_MAX) { 283 // Overflow. 284 *outTimeUs = INT64_MAX; 285 ALOGW("getPlayTimeForPendingAudioFrames: play time for pending audio frames " 286 "is too high, possibly due to super low playback rate(%f)", mPlaybackRate); 287 } else { 288 *outTimeUs = (int64_t)timeUs; 289 } 290 291 return OK; 292 } 293 294 status_t MediaSync::updateQueuedAudioData( 295 size_t sizeInBytes, int64_t presentationTimeUs) { 296 if (sizeInBytes == 0) { 297 return OK; 298 } 299 300 Mutex::Autolock lock(mMutex); 301 302 if (mAudioTrack == NULL) { 303 ALOGW("updateQueuedAudioData: audioTrack has NOT been configured."); 304 return INVALID_OPERATION; 305 } 306 307 int64_t numFrames = sizeInBytes / mAudioTrack->frameSize(); 308 int64_t maxMediaTimeUs = presentationTimeUs 309 + getDurationIfPlayedAtNativeSampleRate_l(numFrames); 310 311 int64_t nowUs = ALooper::GetNowUs(); 312 int64_t nowMediaUs = presentationTimeUs 313 - getDurationIfPlayedAtNativeSampleRate_l(mNumFramesWritten) 314 + getPlayedOutAudioDurationMedia_l(nowUs); 315 316 mNumFramesWritten += numFrames; 317 318 int64_t oldRealTime = -1; 319 if (mNextBufferItemMediaUs != -1) { 320 oldRealTime = getRealTime(mNextBufferItemMediaUs, nowUs); 321 } 322 323 mMediaClock->updateAnchor(nowMediaUs, nowUs, maxMediaTimeUs); 324 mHasAudio = true; 325 326 if (oldRealTime != -1) { 327 int64_t newRealTime = getRealTime(mNextBufferItemMediaUs, nowUs); 328 if (newRealTime >= oldRealTime) { 329 return OK; 330 } 331 } 332 333 mNextBufferItemMediaUs = -1; 334 onDrainVideo_l(); 335 return OK; 336 } 337 338 void MediaSync::setName(const AString &name) { 339 Mutex::Autolock lock(mMutex); 340 mInput->setConsumerName(String8(name.c_str())); 341 } 342 343 void MediaSync::flush() { 344 Mutex::Autolock lock(mMutex); 345 if (mFrameScheduler != NULL) { 346 mFrameScheduler->restart(); 347 } 348 while (!mBufferItems.empty()) { 349 BufferItem *bufferItem = &*mBufferItems.begin(); 350 returnBufferToInput_l(bufferItem->mGraphicBuffer, bufferItem->mFence); 351 mBufferItems.erase(mBufferItems.begin()); 352 } 353 mNextBufferItemMediaUs = -1; 354 mNumFramesWritten = 0; 355 mReturnPendingInputFrame = true; 356 mReleaseCondition.signal(); 357 mMediaClock->clearAnchor(); 358 } 359 360 status_t MediaSync::setVideoFrameRateHint(float rate) { 361 Mutex::Autolock lock(mMutex); 362 if (rate < 0.f) { 363 return BAD_VALUE; 364 } 365 if (mFrameScheduler != NULL) { 366 mFrameScheduler->init(rate); 367 } 368 return OK; 369 } 370 371 float MediaSync::getVideoFrameRate() { 372 Mutex::Autolock lock(mMutex); 373 if (mFrameScheduler != NULL) { 374 float fps = mFrameScheduler->getFrameRate(); 375 if (fps > 0.f) { 376 return fps; 377 } 378 } 379 380 // we don't have or know the frame rate 381 return -1.f; 382 } 383 384 status_t MediaSync::setSyncSettings(const AVSyncSettings &syncSettings) { 385 // validate settings 386 if (syncSettings.mSource >= AVSYNC_SOURCE_MAX 387 || syncSettings.mAudioAdjustMode >= AVSYNC_AUDIO_ADJUST_MODE_MAX 388 || syncSettings.mTolerance < 0.f 389 || syncSettings.mTolerance >= AVSYNC_TOLERANCE_MAX) { 390 return BAD_VALUE; 391 } 392 393 Mutex::Autolock lock(mMutex); 394 395 // verify that we have the sync source 396 switch (syncSettings.mSource) { 397 case AVSYNC_SOURCE_AUDIO: 398 if (mAudioTrack == NULL) { 399 ALOGE("setSyncSettings: audio sync source requires an audio track"); 400 return BAD_VALUE; 401 } 402 break; 403 case AVSYNC_SOURCE_VSYNC: 404 if (mOutput == NULL) { 405 ALOGE("setSyncSettings: vsync sync source requires an output surface"); 406 return BAD_VALUE; 407 } 408 break; 409 default: 410 break; 411 } 412 413 mSyncSettings = syncSettings; 414 resync_l(); 415 return OK; 416 } 417 418 void MediaSync::getSyncSettings(AVSyncSettings *syncSettings) { 419 Mutex::Autolock lock(mMutex); 420 *syncSettings = mSyncSettings; 421 } 422 423 status_t MediaSync::setPlaybackSettings(const AudioPlaybackRate &rate) { 424 Mutex::Autolock lock(mMutex); 425 426 status_t err = setPlaybackSettings_l(rate); 427 if (err == OK) { 428 // TODO: adjust rate if using VSYNC as source 429 updatePlaybackRate_l(rate.mSpeed); 430 } 431 return err; 432 } 433 434 status_t MediaSync::setPlaybackSettings_l(const AudioPlaybackRate &rate) { 435 if (rate.mSpeed < 0.f || rate.mPitch < 0.f) { 436 // We don't validate other audio settings. 437 // They will be validated when/if audiotrack is set. 438 return BAD_VALUE; 439 } 440 441 if (mAudioTrack != NULL) { 442 if (rate.mSpeed == 0.f) { 443 mAudioTrack->pause(); 444 } else { 445 status_t err = mAudioTrack->setPlaybackRate(rate); 446 if (err != OK) { 447 return BAD_VALUE; 448 } 449 450 // ignore errors 451 (void)mAudioTrack->start(); 452 } 453 } 454 mPlaybackSettings = rate; 455 return OK; 456 } 457 458 void MediaSync::getPlaybackSettings(AudioPlaybackRate *rate) { 459 Mutex::Autolock lock(mMutex); 460 *rate = mPlaybackSettings; 461 } 462 463 int64_t MediaSync::getRealTime(int64_t mediaTimeUs, int64_t nowUs) { 464 int64_t realUs; 465 if (mMediaClock->getRealTimeFor(mediaTimeUs, &realUs) != OK) { 466 // If failed to get current position, e.g. due to audio clock is 467 // not ready, then just play out video immediately without delay. 468 return nowUs; 469 } 470 return realUs; 471 } 472 473 int64_t MediaSync::getDurationIfPlayedAtNativeSampleRate_l(int64_t numFrames) { 474 return (numFrames * 1000000LL / mNativeSampleRateInHz); 475 } 476 477 int64_t MediaSync::getPlayedOutAudioDurationMedia_l(int64_t nowUs) { 478 CHECK(mAudioTrack != NULL); 479 480 uint32_t numFramesPlayed; 481 int64_t numFramesPlayedAtUs; 482 AudioTimestamp ts; 483 484 status_t res = mAudioTrack->getTimestamp(ts); 485 if (res == OK) { 486 // case 1: mixing audio tracks. 487 numFramesPlayed = ts.mPosition; 488 numFramesPlayedAtUs = ts.mTime.tv_sec * 1000000LL + ts.mTime.tv_nsec / 1000; 489 //ALOGD("getTimestamp: OK %d %lld", 490 // numFramesPlayed, (long long)numFramesPlayedAtUs); 491 } else if (res == WOULD_BLOCK) { 492 // case 2: transitory state on start of a new track 493 numFramesPlayed = 0; 494 numFramesPlayedAtUs = nowUs; 495 //ALOGD("getTimestamp: WOULD_BLOCK %d %lld", 496 // numFramesPlayed, (long long)numFramesPlayedAtUs); 497 } else { 498 // case 3: transitory at new track or audio fast tracks. 499 res = mAudioTrack->getPosition(&numFramesPlayed); 500 CHECK_EQ(res, (status_t)OK); 501 numFramesPlayedAtUs = nowUs; 502 numFramesPlayedAtUs += 1000LL * mAudioTrack->latency() / 2; /* XXX */ 503 //ALOGD("getPosition: %d %lld", numFramesPlayed, (long long)numFramesPlayedAtUs); 504 } 505 506 //can't be negative until 12.4 hrs, test. 507 //CHECK_EQ(numFramesPlayed & (1 << 31), 0); 508 int64_t durationUs = 509 getDurationIfPlayedAtNativeSampleRate_l(numFramesPlayed) 510 + nowUs - numFramesPlayedAtUs; 511 if (durationUs < 0) { 512 // Occurs when numFramesPlayed position is very small and the following: 513 // (1) In case 1, the time nowUs is computed before getTimestamp() is 514 // called and numFramesPlayedAtUs is greater than nowUs by time more 515 // than numFramesPlayed. 516 // (2) In case 3, using getPosition and adding mAudioTrack->latency() 517 // to numFramesPlayedAtUs, by a time amount greater than 518 // numFramesPlayed. 519 // 520 // Both of these are transitory conditions. 521 ALOGV("getPlayedOutAudioDurationMedia_l: negative duration %lld " 522 "set to zero", (long long)durationUs); 523 durationUs = 0; 524 } 525 ALOGV("getPlayedOutAudioDurationMedia_l(%lld) nowUs(%lld) frames(%u) " 526 "framesAt(%lld)", 527 (long long)durationUs, (long long)nowUs, numFramesPlayed, 528 (long long)numFramesPlayedAtUs); 529 return durationUs; 530 } 531 532 void MediaSync::onDrainVideo_l() { 533 if (!isPlaying()) { 534 return; 535 } 536 537 while (!mBufferItems.empty()) { 538 int64_t nowUs = ALooper::GetNowUs(); 539 BufferItem *bufferItem = &*mBufferItems.begin(); 540 int64_t itemMediaUs = bufferItem->mTimestamp / 1000; 541 int64_t itemRealUs = getRealTime(itemMediaUs, nowUs); 542 543 // adjust video frame PTS based on vsync 544 itemRealUs = mFrameScheduler->schedule(itemRealUs * 1000) / 1000; 545 int64_t twoVsyncsUs = 2 * (mFrameScheduler->getVsyncPeriod() / 1000); 546 547 // post 2 display refreshes before rendering is due 548 if (itemRealUs <= nowUs + twoVsyncsUs) { 549 ALOGV("adjusting PTS from %lld to %lld", 550 (long long)bufferItem->mTimestamp / 1000, (long long)itemRealUs); 551 bufferItem->mTimestamp = itemRealUs * 1000; 552 bufferItem->mIsAutoTimestamp = false; 553 554 if (mHasAudio) { 555 if (nowUs - itemRealUs <= kMaxAllowedVideoLateTimeUs) { 556 renderOneBufferItem_l(*bufferItem); 557 } else { 558 // too late. 559 returnBufferToInput_l( 560 bufferItem->mGraphicBuffer, bufferItem->mFence); 561 mFrameScheduler->restart(); 562 } 563 } else { 564 // always render video buffer in video-only mode. 565 renderOneBufferItem_l(*bufferItem); 566 567 // smooth out videos >= 10fps 568 mMediaClock->updateAnchor( 569 itemMediaUs, nowUs, itemMediaUs + 100000); 570 } 571 572 mBufferItems.erase(mBufferItems.begin()); 573 mNextBufferItemMediaUs = -1; 574 } else { 575 if (mNextBufferItemMediaUs == -1 576 || mNextBufferItemMediaUs > itemMediaUs) { 577 sp<AMessage> msg = new AMessage(kWhatDrainVideo, this); 578 msg->post(itemRealUs - nowUs - twoVsyncsUs); 579 mNextBufferItemMediaUs = itemMediaUs; 580 } 581 break; 582 } 583 } 584 } 585 586 void MediaSync::onFrameAvailableFromInput() { 587 Mutex::Autolock lock(mMutex); 588 589 const static nsecs_t kAcquireWaitTimeout = 2000000000; // 2 seconds 590 591 mReturnPendingInputFrame = false; 592 593 // If there are too many outstanding buffers, wait until a buffer is 594 // released back to the input in onBufferReleased. 595 // NOTE: BufferQueue allows dequeuing maxAcquiredBufferCount + 1 buffers 596 while (mNumOutstandingBuffers > mMaxAcquiredBufferCount 597 && !mIsAbandoned && !mReturnPendingInputFrame) { 598 if (mReleaseCondition.waitRelative(mMutex, kAcquireWaitTimeout) != OK) { 599 ALOGI_IF(mPlaybackRate != 0.f, "still waiting to release a buffer before acquire"); 600 } 601 602 // If the sync is abandoned while we are waiting, the release 603 // condition variable will be broadcast, and we should just return 604 // without attempting to do anything more (since the input queue will 605 // also be abandoned). 606 if (mIsAbandoned) { 607 return; 608 } 609 } 610 611 // Acquire and detach the buffer from the input. 612 BufferItem bufferItem; 613 status_t status = mInput->acquireBuffer(&bufferItem, 0 /* presentWhen */); 614 if (status != NO_ERROR) { 615 ALOGE("acquiring buffer from input failed (%d)", status); 616 return; 617 } 618 ++mNumOutstandingBuffers; 619 620 ALOGV("acquired buffer %#llx from input", (long long)bufferItem.mGraphicBuffer->getId()); 621 622 status = mInput->detachBuffer(bufferItem.mSlot); 623 if (status != NO_ERROR) { 624 ALOGE("detaching buffer from input failed (%d)", status); 625 if (status == NO_INIT) { 626 // If the input has been abandoned, move on. 627 onAbandoned_l(true /* isInput */); 628 } 629 return; 630 } 631 632 if (mBuffersFromInput.indexOfKey(bufferItem.mGraphicBuffer->getId()) >= 0) { 633 // Something is wrong since this buffer should be at our hands, bail. 634 ALOGE("received buffer multiple times from input"); 635 mInput->consumerDisconnect(); 636 onAbandoned_l(true /* isInput */); 637 return; 638 } 639 mBuffersFromInput.add(bufferItem.mGraphicBuffer->getId(), bufferItem.mGraphicBuffer); 640 641 // If flush happened while waiting for a buffer to be released, simply return it 642 // TRICKY: do it here after it is detached so that we don't have to cache mGraphicBuffer. 643 if (mReturnPendingInputFrame) { 644 mReturnPendingInputFrame = false; 645 returnBufferToInput_l(bufferItem.mGraphicBuffer, bufferItem.mFence); 646 return; 647 } 648 649 mBufferItems.push_back(bufferItem); 650 651 if (mBufferItems.size() == 1) { 652 onDrainVideo_l(); 653 } 654 } 655 656 void MediaSync::renderOneBufferItem_l(const BufferItem &bufferItem) { 657 IGraphicBufferProducer::QueueBufferInput queueInput( 658 bufferItem.mTimestamp, 659 bufferItem.mIsAutoTimestamp, 660 bufferItem.mDataSpace, 661 bufferItem.mCrop, 662 static_cast<int32_t>(bufferItem.mScalingMode), 663 bufferItem.mTransform, 664 bufferItem.mFence); 665 666 // Attach and queue the buffer to the output. 667 int slot; 668 mOutput->setGenerationNumber(bufferItem.mGraphicBuffer->getGenerationNumber()); 669 status_t status = mOutput->attachBuffer(&slot, bufferItem.mGraphicBuffer); 670 ALOGE_IF(status != NO_ERROR, "attaching buffer to output failed (%d)", status); 671 if (status == NO_ERROR) { 672 IGraphicBufferProducer::QueueBufferOutput queueOutput; 673 status = mOutput->queueBuffer(slot, queueInput, &queueOutput); 674 ALOGE_IF(status != NO_ERROR, "queueing buffer to output failed (%d)", status); 675 } 676 677 if (status != NO_ERROR) { 678 returnBufferToInput_l(bufferItem.mGraphicBuffer, bufferItem.mFence); 679 if (status == NO_INIT) { 680 // If the output has been abandoned, move on. 681 onAbandoned_l(false /* isInput */); 682 } 683 return; 684 } 685 686 if (mBuffersSentToOutput.indexOfKey(bufferItem.mGraphicBuffer->getId()) >= 0) { 687 // Something is wrong since this buffer should be held by output now, bail. 688 mInput->consumerDisconnect(); 689 onAbandoned_l(true /* isInput */); 690 return; 691 } 692 mBuffersSentToOutput.add(bufferItem.mGraphicBuffer->getId(), bufferItem.mGraphicBuffer); 693 694 ALOGV("queued buffer %#llx to output", (long long)bufferItem.mGraphicBuffer->getId()); 695 } 696 697 void MediaSync::onBufferReleasedByOutput(sp<IGraphicBufferProducer> &output) { 698 Mutex::Autolock lock(mMutex); 699 700 if (output != mOutput) { 701 return; // This is not the current output, ignore. 702 } 703 704 sp<GraphicBuffer> buffer; 705 sp<Fence> fence; 706 status_t status = mOutput->detachNextBuffer(&buffer, &fence); 707 ALOGE_IF(status != NO_ERROR, "detaching buffer from output failed (%d)", status); 708 709 if (status == NO_INIT) { 710 // If the output has been abandoned, we can't do anything else, 711 // since buffer is invalid. 712 onAbandoned_l(false /* isInput */); 713 return; 714 } 715 716 ALOGV("detached buffer %#llx from output", (long long)buffer->getId()); 717 718 // If we've been abandoned, we can't return the buffer to the input, so just 719 // move on. 720 if (mIsAbandoned) { 721 return; 722 } 723 724 ssize_t ix = mBuffersSentToOutput.indexOfKey(buffer->getId()); 725 if (ix < 0) { 726 // The buffer is unknown, maybe leftover, ignore. 727 return; 728 } 729 mBuffersSentToOutput.removeItemsAt(ix); 730 731 returnBufferToInput_l(buffer, fence); 732 } 733 734 void MediaSync::returnBufferToInput_l( 735 const sp<GraphicBuffer> &buffer, const sp<Fence> &fence) { 736 ssize_t ix = mBuffersFromInput.indexOfKey(buffer->getId()); 737 if (ix < 0) { 738 // The buffer is unknown, something is wrong, bail. 739 ALOGE("output returned unknown buffer"); 740 mOutput->disconnect(NATIVE_WINDOW_API_MEDIA); 741 onAbandoned_l(false /* isInput */); 742 return; 743 } 744 sp<GraphicBuffer> oldBuffer = mBuffersFromInput.valueAt(ix); 745 mBuffersFromInput.removeItemsAt(ix); 746 747 // Attach and release the buffer back to the input. 748 int consumerSlot; 749 status_t status = mInput->attachBuffer(&consumerSlot, oldBuffer); 750 ALOGE_IF(status != NO_ERROR, "attaching buffer to input failed (%d)", status); 751 if (status == NO_ERROR) { 752 status = mInput->releaseBuffer(consumerSlot, 0 /* frameNumber */, 753 EGL_NO_DISPLAY, EGL_NO_SYNC_KHR, fence); 754 ALOGE_IF(status != NO_ERROR, "releasing buffer to input failed (%d)", status); 755 } 756 757 // Notify any waiting onFrameAvailable calls. 758 --mNumOutstandingBuffers; 759 mReleaseCondition.signal(); 760 761 if (status == NO_ERROR) { 762 ALOGV("released buffer %#llx to input", (long long)oldBuffer->getId()); 763 } 764 } 765 766 void MediaSync::onAbandoned_l(bool isInput) { 767 ALOGE("the %s has abandoned me", (isInput ? "input" : "output")); 768 if (!mIsAbandoned) { 769 if (isInput) { 770 mOutput->disconnect(NATIVE_WINDOW_API_MEDIA); 771 } else { 772 mInput->consumerDisconnect(); 773 } 774 mIsAbandoned = true; 775 } 776 mReleaseCondition.broadcast(); 777 } 778 779 void MediaSync::onMessageReceived(const sp<AMessage> &msg) { 780 switch (msg->what()) { 781 case kWhatDrainVideo: 782 { 783 Mutex::Autolock lock(mMutex); 784 if (mNextBufferItemMediaUs != -1) { 785 int64_t nowUs = ALooper::GetNowUs(); 786 int64_t itemRealUs = getRealTime(mNextBufferItemMediaUs, nowUs); 787 788 // The message could arrive earlier than expected due to 789 // various reasons, e.g., media clock has been changed because 790 // of new anchor time or playback rate. In such cases, the 791 // message needs to be re-posted. 792 if (itemRealUs > nowUs) { 793 msg->post(itemRealUs - nowUs); 794 break; 795 } 796 } 797 798 onDrainVideo_l(); 799 break; 800 } 801 802 default: 803 TRESPASS(); 804 break; 805 } 806 } 807 808 MediaSync::InputListener::InputListener(const sp<MediaSync> &sync) 809 : mSync(sync) {} 810 811 MediaSync::InputListener::~InputListener() {} 812 813 void MediaSync::InputListener::onFrameAvailable(const BufferItem &/* item */) { 814 mSync->onFrameAvailableFromInput(); 815 } 816 817 // We don't care about sideband streams, since we won't relay them. 818 void MediaSync::InputListener::onSidebandStreamChanged() { 819 ALOGE("onSidebandStreamChanged: got sideband stream unexpectedly."); 820 } 821 822 823 void MediaSync::InputListener::binderDied(const wp<IBinder> &/* who */) { 824 Mutex::Autolock lock(mSync->mMutex); 825 mSync->onAbandoned_l(true /* isInput */); 826 } 827 828 MediaSync::OutputListener::OutputListener(const sp<MediaSync> &sync, 829 const sp<IGraphicBufferProducer> &output) 830 : mSync(sync), 831 mOutput(output) {} 832 833 MediaSync::OutputListener::~OutputListener() {} 834 835 void MediaSync::OutputListener::onBufferReleased() { 836 mSync->onBufferReleasedByOutput(mOutput); 837 } 838 839 void MediaSync::OutputListener::binderDied(const wp<IBinder> &/* who */) { 840 Mutex::Autolock lock(mSync->mMutex); 841 mSync->onAbandoned_l(false /* isInput */); 842 } 843 844 } // namespace android 845