1 /* 2 * Copyright 2015 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17 //#define LOG_NDEBUG 0 18 #define LOG_TAG "MediaSync" 19 #include <inttypes.h> 20 21 #include <gui/BufferQueue.h> 22 #include <gui/IGraphicBufferConsumer.h> 23 #include <gui/IGraphicBufferProducer.h> 24 25 #include <media/AudioTrack.h> 26 #include <media/stagefright/MediaClock.h> 27 #include <media/stagefright/MediaSync.h> 28 #include <media/stagefright/VideoFrameScheduler.h> 29 #include <media/stagefright/foundation/ADebug.h> 30 #include <media/stagefright/foundation/ALooper.h> 31 #include <media/stagefright/foundation/AMessage.h> 32 33 #include <ui/GraphicBuffer.h> 34 35 // Maximum late time allowed for a video frame to be rendered. When a video 36 // frame arrives later than this number, it will be discarded without rendering. 37 static const int64_t kMaxAllowedVideoLateTimeUs = 40000ll; 38 39 namespace android { 40 41 // static 42 sp<MediaSync> MediaSync::create() { 43 sp<MediaSync> sync = new MediaSync(); 44 sync->mLooper->registerHandler(sync); 45 return sync; 46 } 47 48 MediaSync::MediaSync() 49 : mIsAbandoned(false), 50 mMutex(), 51 mReleaseCondition(), 52 mNumOutstandingBuffers(0), 53 mUsageFlagsFromOutput(0), 54 mMaxAcquiredBufferCount(1), 55 mReturnPendingInputFrame(false), 56 mNativeSampleRateInHz(0), 57 mNumFramesWritten(0), 58 mHasAudio(false), 59 mNextBufferItemMediaUs(-1), 60 mPlaybackRate(0.0) { 61 mMediaClock = new MediaClock; 62 63 // initialize settings 64 mPlaybackSettings = AUDIO_PLAYBACK_RATE_DEFAULT; 65 mPlaybackSettings.mSpeed = mPlaybackRate; 66 67 mLooper = new ALooper; 68 mLooper->setName("MediaSync"); 69 mLooper->start(false, false, ANDROID_PRIORITY_AUDIO); 70 } 71 72 MediaSync::~MediaSync() { 73 if (mInput != NULL) { 74 mInput->consumerDisconnect(); 75 } 76 if (mOutput != NULL) { 77 mOutput->disconnect(NATIVE_WINDOW_API_MEDIA); 78 } 79 80 if (mLooper != NULL) { 81 mLooper->unregisterHandler(id()); 82 mLooper->stop(); 83 } 84 } 85 86 status_t MediaSync::setSurface(const sp<IGraphicBufferProducer> &output) { 87 Mutex::Autolock lock(mMutex); 88 89 if (output == mOutput) { 90 return NO_ERROR; // same output surface. 91 } 92 93 if (output == NULL && mSyncSettings.mSource == AVSYNC_SOURCE_VSYNC) { 94 ALOGE("setSurface: output surface is used as sync source and cannot be removed."); 95 return INVALID_OPERATION; 96 } 97 98 if (output != NULL) { 99 int newUsage = 0; 100 output->query(NATIVE_WINDOW_CONSUMER_USAGE_BITS, &newUsage); 101 102 // Check usage flags only when current output surface has been used to create input surface. 103 if (mOutput != NULL && mInput != NULL) { 104 int ignoredFlags = (GRALLOC_USAGE_HW_TEXTURE | GRALLOC_USAGE_HW_COMPOSER 105 | GRALLOC_USAGE_EXTERNAL_DISP); 106 // New output surface is not allowed to add new usage flag except ignored ones. 107 if ((newUsage & ~(mUsageFlagsFromOutput | ignoredFlags)) != 0) { 108 ALOGE("setSurface: new output surface has new usage flag not used by current one."); 109 return BAD_VALUE; 110 } 111 } 112 113 // Try to connect to new output surface. If failed, current output surface will not 114 // be changed. 115 IGraphicBufferProducer::QueueBufferOutput queueBufferOutput; 116 sp<OutputListener> listener(new OutputListener(this, output)); 117 IInterface::asBinder(output)->linkToDeath(listener); 118 status_t status = 119 output->connect(listener, 120 NATIVE_WINDOW_API_MEDIA, 121 true /* producerControlledByApp */, 122 &queueBufferOutput); 123 if (status != NO_ERROR) { 124 ALOGE("setSurface: failed to connect (%d)", status); 125 return status; 126 } 127 128 if (mFrameScheduler == NULL) { 129 mFrameScheduler = new VideoFrameScheduler(); 130 mFrameScheduler->init(); 131 } 132 } 133 134 if (mOutput != NULL) { 135 mOutput->disconnect(NATIVE_WINDOW_API_MEDIA); 136 while (!mBuffersSentToOutput.isEmpty()) { 137 returnBufferToInput_l(mBuffersSentToOutput.valueAt(0), Fence::NO_FENCE); 138 mBuffersSentToOutput.removeItemsAt(0); 139 } 140 } 141 142 mOutput = output; 143 144 return NO_ERROR; 145 } 146 147 // |audioTrack| is used only for querying information. 148 status_t MediaSync::setAudioTrack(const sp<AudioTrack> &audioTrack) { 149 Mutex::Autolock lock(mMutex); 150 151 // TODO: support audio track change. 152 if (mAudioTrack != NULL) { 153 ALOGE("setAudioTrack: audioTrack has already been configured."); 154 return INVALID_OPERATION; 155 } 156 157 if (audioTrack == NULL && mSyncSettings.mSource == AVSYNC_SOURCE_AUDIO) { 158 ALOGE("setAudioTrack: audioTrack is used as sync source and cannot be removed."); 159 return INVALID_OPERATION; 160 } 161 162 if (audioTrack != NULL) { 163 // check if audio track supports the playback settings 164 if (mPlaybackSettings.mSpeed != 0.f 165 && audioTrack->setPlaybackRate(mPlaybackSettings) != OK) { 166 ALOGE("playback settings are not supported by the audio track"); 167 return INVALID_OPERATION; 168 } 169 uint32_t nativeSampleRateInHz = audioTrack->getOriginalSampleRate(); 170 if (nativeSampleRateInHz <= 0) { 171 ALOGE("setAudioTrack: native sample rate should be positive."); 172 return BAD_VALUE; 173 } 174 mAudioTrack = audioTrack; 175 mNativeSampleRateInHz = nativeSampleRateInHz; 176 (void)setPlaybackSettings_l(mPlaybackSettings); 177 } 178 else { 179 mAudioTrack = NULL; 180 mNativeSampleRateInHz = 0; 181 } 182 183 // potentially resync to new source 184 resync_l(); 185 return OK; 186 } 187 188 status_t MediaSync::createInputSurface( 189 sp<IGraphicBufferProducer> *outBufferProducer) { 190 if (outBufferProducer == NULL) { 191 return BAD_VALUE; 192 } 193 194 Mutex::Autolock lock(mMutex); 195 196 if (mOutput == NULL) { 197 return NO_INIT; 198 } 199 200 if (mInput != NULL) { 201 return INVALID_OPERATION; 202 } 203 204 sp<IGraphicBufferProducer> bufferProducer; 205 sp<IGraphicBufferConsumer> bufferConsumer; 206 BufferQueue::createBufferQueue(&bufferProducer, &bufferConsumer); 207 208 sp<InputListener> listener(new InputListener(this)); 209 IInterface::asBinder(bufferConsumer)->linkToDeath(listener); 210 status_t status = 211 bufferConsumer->consumerConnect(listener, false /* controlledByApp */); 212 if (status == NO_ERROR) { 213 bufferConsumer->setConsumerName(String8("MediaSync")); 214 // propagate usage bits from output surface 215 mUsageFlagsFromOutput = 0; 216 mOutput->query(NATIVE_WINDOW_CONSUMER_USAGE_BITS, &mUsageFlagsFromOutput); 217 bufferConsumer->setConsumerUsageBits(mUsageFlagsFromOutput); 218 *outBufferProducer = bufferProducer; 219 mInput = bufferConsumer; 220 221 // set undequeued buffer count 222 int minUndequeuedBuffers; 223 mOutput->query(NATIVE_WINDOW_MIN_UNDEQUEUED_BUFFERS, &minUndequeuedBuffers); 224 mMaxAcquiredBufferCount = minUndequeuedBuffers; 225 bufferConsumer->setMaxAcquiredBufferCount(mMaxAcquiredBufferCount); 226 } 227 return status; 228 } 229 230 void MediaSync::resync_l() { 231 AVSyncSource src = mSyncSettings.mSource; 232 if (src == AVSYNC_SOURCE_DEFAULT) { 233 if (mAudioTrack != NULL) { 234 src = AVSYNC_SOURCE_AUDIO; 235 } else { 236 src = AVSYNC_SOURCE_SYSTEM_CLOCK; 237 } 238 } 239 240 // TODO: resync ourselves to the current clock (e.g. on sync source change) 241 updatePlaybackRate_l(mPlaybackRate); 242 } 243 244 void MediaSync::updatePlaybackRate_l(float rate) { 245 if (rate > mPlaybackRate) { 246 mNextBufferItemMediaUs = -1; 247 } 248 mPlaybackRate = rate; 249 // TODO: update frame scheduler with this info 250 mMediaClock->setPlaybackRate(rate); 251 onDrainVideo_l(); 252 } 253 254 sp<const MediaClock> MediaSync::getMediaClock() { 255 return mMediaClock; 256 } 257 258 status_t MediaSync::getPlayTimeForPendingAudioFrames(int64_t *outTimeUs) { 259 Mutex::Autolock lock(mMutex); 260 // User should check the playback rate if it doesn't want to receive a 261 // huge number for play time. 262 if (mPlaybackRate == 0.0f) { 263 *outTimeUs = INT64_MAX; 264 return OK; 265 } 266 267 uint32_t numFramesPlayed = 0; 268 if (mAudioTrack != NULL) { 269 status_t res = mAudioTrack->getPosition(&numFramesPlayed); 270 if (res != OK) { 271 return res; 272 } 273 } 274 275 int64_t numPendingFrames = mNumFramesWritten - numFramesPlayed; 276 if (numPendingFrames < 0) { 277 numPendingFrames = 0; 278 ALOGW("getPlayTimeForPendingAudioFrames: pending frame count is negative."); 279 } 280 double timeUs = numPendingFrames * 1000000.0 281 / (mNativeSampleRateInHz * (double)mPlaybackRate); 282 if (timeUs > (double)INT64_MAX) { 283 // Overflow. 284 *outTimeUs = INT64_MAX; 285 ALOGW("getPlayTimeForPendingAudioFrames: play time for pending audio frames " 286 "is too high, possibly due to super low playback rate(%f)", mPlaybackRate); 287 } else { 288 *outTimeUs = (int64_t)timeUs; 289 } 290 291 return OK; 292 } 293 294 status_t MediaSync::updateQueuedAudioData( 295 size_t sizeInBytes, int64_t presentationTimeUs) { 296 if (sizeInBytes == 0) { 297 return OK; 298 } 299 300 Mutex::Autolock lock(mMutex); 301 302 if (mAudioTrack == NULL) { 303 ALOGW("updateQueuedAudioData: audioTrack has NOT been configured."); 304 return INVALID_OPERATION; 305 } 306 307 int64_t numFrames = sizeInBytes / mAudioTrack->frameSize(); 308 int64_t maxMediaTimeUs = presentationTimeUs 309 + getDurationIfPlayedAtNativeSampleRate_l(numFrames); 310 311 int64_t nowUs = ALooper::GetNowUs(); 312 int64_t nowMediaUs = presentationTimeUs 313 - getDurationIfPlayedAtNativeSampleRate_l(mNumFramesWritten) 314 + getPlayedOutAudioDurationMedia_l(nowUs); 315 316 mNumFramesWritten += numFrames; 317 318 int64_t oldRealTime = -1; 319 if (mNextBufferItemMediaUs != -1) { 320 oldRealTime = getRealTime(mNextBufferItemMediaUs, nowUs); 321 } 322 323 mMediaClock->updateAnchor(nowMediaUs, nowUs, maxMediaTimeUs); 324 mHasAudio = true; 325 326 if (oldRealTime != -1) { 327 int64_t newRealTime = getRealTime(mNextBufferItemMediaUs, nowUs); 328 if (newRealTime >= oldRealTime) { 329 return OK; 330 } 331 } 332 333 mNextBufferItemMediaUs = -1; 334 onDrainVideo_l(); 335 return OK; 336 } 337 338 void MediaSync::setName(const AString &name) { 339 Mutex::Autolock lock(mMutex); 340 mInput->setConsumerName(String8(name.c_str())); 341 } 342 343 void MediaSync::flush() { 344 Mutex::Autolock lock(mMutex); 345 if (mFrameScheduler != NULL) { 346 mFrameScheduler->restart(); 347 } 348 while (!mBufferItems.empty()) { 349 BufferItem *bufferItem = &*mBufferItems.begin(); 350 returnBufferToInput_l(bufferItem->mGraphicBuffer, bufferItem->mFence); 351 mBufferItems.erase(mBufferItems.begin()); 352 } 353 mNextBufferItemMediaUs = -1; 354 mNumFramesWritten = 0; 355 mReturnPendingInputFrame = true; 356 mReleaseCondition.signal(); 357 mMediaClock->clearAnchor(); 358 } 359 360 status_t MediaSync::setVideoFrameRateHint(float rate) { 361 Mutex::Autolock lock(mMutex); 362 if (rate < 0.f) { 363 return BAD_VALUE; 364 } 365 if (mFrameScheduler != NULL) { 366 mFrameScheduler->init(rate); 367 } 368 return OK; 369 } 370 371 float MediaSync::getVideoFrameRate() { 372 Mutex::Autolock lock(mMutex); 373 if (mFrameScheduler != NULL) { 374 float fps = mFrameScheduler->getFrameRate(); 375 if (fps > 0.f) { 376 return fps; 377 } 378 } 379 380 // we don't have or know the frame rate 381 return -1.f; 382 } 383 384 status_t MediaSync::setSyncSettings(const AVSyncSettings &syncSettings) { 385 // validate settings 386 if (syncSettings.mSource >= AVSYNC_SOURCE_MAX 387 || syncSettings.mAudioAdjustMode >= AVSYNC_AUDIO_ADJUST_MODE_MAX 388 || syncSettings.mTolerance < 0.f 389 || syncSettings.mTolerance >= AVSYNC_TOLERANCE_MAX) { 390 return BAD_VALUE; 391 } 392 393 Mutex::Autolock lock(mMutex); 394 395 // verify that we have the sync source 396 switch (syncSettings.mSource) { 397 case AVSYNC_SOURCE_AUDIO: 398 if (mAudioTrack == NULL) { 399 ALOGE("setSyncSettings: audio sync source requires an audio track"); 400 return BAD_VALUE; 401 } 402 break; 403 case AVSYNC_SOURCE_VSYNC: 404 if (mOutput == NULL) { 405 ALOGE("setSyncSettings: vsync sync source requires an output surface"); 406 return BAD_VALUE; 407 } 408 break; 409 default: 410 break; 411 } 412 413 mSyncSettings = syncSettings; 414 resync_l(); 415 return OK; 416 } 417 418 void MediaSync::getSyncSettings(AVSyncSettings *syncSettings) { 419 Mutex::Autolock lock(mMutex); 420 *syncSettings = mSyncSettings; 421 } 422 423 status_t MediaSync::setPlaybackSettings(const AudioPlaybackRate &rate) { 424 Mutex::Autolock lock(mMutex); 425 426 status_t err = setPlaybackSettings_l(rate); 427 if (err == OK) { 428 // TODO: adjust rate if using VSYNC as source 429 updatePlaybackRate_l(rate.mSpeed); 430 } 431 return err; 432 } 433 434 status_t MediaSync::setPlaybackSettings_l(const AudioPlaybackRate &rate) { 435 if (rate.mSpeed < 0.f || rate.mPitch < 0.f) { 436 // We don't validate other audio settings. 437 // They will be validated when/if audiotrack is set. 438 return BAD_VALUE; 439 } 440 441 if (mAudioTrack != NULL) { 442 if (rate.mSpeed == 0.f) { 443 mAudioTrack->pause(); 444 } else { 445 status_t err = mAudioTrack->setPlaybackRate(rate); 446 if (err != OK) { 447 return BAD_VALUE; 448 } 449 450 // ignore errors 451 (void)mAudioTrack->start(); 452 } 453 } 454 mPlaybackSettings = rate; 455 return OK; 456 } 457 458 void MediaSync::getPlaybackSettings(AudioPlaybackRate *rate) { 459 Mutex::Autolock lock(mMutex); 460 *rate = mPlaybackSettings; 461 } 462 463 int64_t MediaSync::getRealTime(int64_t mediaTimeUs, int64_t nowUs) { 464 int64_t realUs; 465 if (mMediaClock->getRealTimeFor(mediaTimeUs, &realUs) != OK) { 466 // If failed to get current position, e.g. due to audio clock is 467 // not ready, then just play out video immediately without delay. 468 return nowUs; 469 } 470 return realUs; 471 } 472 473 int64_t MediaSync::getDurationIfPlayedAtNativeSampleRate_l(int64_t numFrames) { 474 return (numFrames * 1000000LL / mNativeSampleRateInHz); 475 } 476 477 int64_t MediaSync::getPlayedOutAudioDurationMedia_l(int64_t nowUs) { 478 CHECK(mAudioTrack != NULL); 479 480 uint32_t numFramesPlayed; 481 int64_t numFramesPlayedAt; 482 AudioTimestamp ts; 483 static const int64_t kStaleTimestamp100ms = 100000; 484 485 status_t res = mAudioTrack->getTimestamp(ts); 486 if (res == OK) { 487 // case 1: mixing audio tracks. 488 numFramesPlayed = ts.mPosition; 489 numFramesPlayedAt = 490 ts.mTime.tv_sec * 1000000LL + ts.mTime.tv_nsec / 1000; 491 const int64_t timestampAge = nowUs - numFramesPlayedAt; 492 if (timestampAge > kStaleTimestamp100ms) { 493 // This is an audio FIXME. 494 // getTimestamp returns a timestamp which may come from audio 495 // mixing threads. After pausing, the MixerThread may go idle, 496 // thus the mTime estimate may become stale. Assuming that the 497 // MixerThread runs 20ms, with FastMixer at 5ms, the max latency 498 // should be about 25ms with an average around 12ms (to be 499 // verified). For safety we use 100ms. 500 ALOGV("getTimestamp: returned stale timestamp nowUs(%lld) " 501 "numFramesPlayedAt(%lld)", 502 (long long)nowUs, (long long)numFramesPlayedAt); 503 numFramesPlayedAt = nowUs - kStaleTimestamp100ms; 504 } 505 //ALOGD("getTimestamp: OK %d %lld", 506 // numFramesPlayed, (long long)numFramesPlayedAt); 507 } else if (res == WOULD_BLOCK) { 508 // case 2: transitory state on start of a new track 509 numFramesPlayed = 0; 510 numFramesPlayedAt = nowUs; 511 //ALOGD("getTimestamp: WOULD_BLOCK %d %lld", 512 // numFramesPlayed, (long long)numFramesPlayedAt); 513 } else { 514 // case 3: transitory at new track or audio fast tracks. 515 res = mAudioTrack->getPosition(&numFramesPlayed); 516 CHECK_EQ(res, (status_t)OK); 517 numFramesPlayedAt = nowUs; 518 numFramesPlayedAt += 1000LL * mAudioTrack->latency() / 2; /* XXX */ 519 //ALOGD("getPosition: %d %lld", numFramesPlayed, (long long)numFramesPlayedAt); 520 } 521 522 //can't be negative until 12.4 hrs, test. 523 //CHECK_EQ(numFramesPlayed & (1 << 31), 0); 524 int64_t durationUs = 525 getDurationIfPlayedAtNativeSampleRate_l(numFramesPlayed) 526 + nowUs - numFramesPlayedAt; 527 if (durationUs < 0) { 528 // Occurs when numFramesPlayed position is very small and the following: 529 // (1) In case 1, the time nowUs is computed before getTimestamp() is 530 // called and numFramesPlayedAt is greater than nowUs by time more 531 // than numFramesPlayed. 532 // (2) In case 3, using getPosition and adding mAudioTrack->latency() 533 // to numFramesPlayedAt, by a time amount greater than 534 // numFramesPlayed. 535 // 536 // Both of these are transitory conditions. 537 ALOGV("getPlayedOutAudioDurationMedia_l: negative duration %lld " 538 "set to zero", (long long)durationUs); 539 durationUs = 0; 540 } 541 ALOGV("getPlayedOutAudioDurationMedia_l(%lld) nowUs(%lld) frames(%u) " 542 "framesAt(%lld)", 543 (long long)durationUs, (long long)nowUs, numFramesPlayed, 544 (long long)numFramesPlayedAt); 545 return durationUs; 546 } 547 548 void MediaSync::onDrainVideo_l() { 549 if (!isPlaying()) { 550 return; 551 } 552 553 while (!mBufferItems.empty()) { 554 int64_t nowUs = ALooper::GetNowUs(); 555 BufferItem *bufferItem = &*mBufferItems.begin(); 556 int64_t itemMediaUs = bufferItem->mTimestamp / 1000; 557 int64_t itemRealUs = getRealTime(itemMediaUs, nowUs); 558 559 // adjust video frame PTS based on vsync 560 itemRealUs = mFrameScheduler->schedule(itemRealUs * 1000) / 1000; 561 int64_t twoVsyncsUs = 2 * (mFrameScheduler->getVsyncPeriod() / 1000); 562 563 // post 2 display refreshes before rendering is due 564 if (itemRealUs <= nowUs + twoVsyncsUs) { 565 ALOGV("adjusting PTS from %lld to %lld", 566 (long long)bufferItem->mTimestamp / 1000, (long long)itemRealUs); 567 bufferItem->mTimestamp = itemRealUs * 1000; 568 bufferItem->mIsAutoTimestamp = false; 569 570 if (mHasAudio) { 571 if (nowUs - itemRealUs <= kMaxAllowedVideoLateTimeUs) { 572 renderOneBufferItem_l(*bufferItem); 573 } else { 574 // too late. 575 returnBufferToInput_l( 576 bufferItem->mGraphicBuffer, bufferItem->mFence); 577 mFrameScheduler->restart(); 578 } 579 } else { 580 // always render video buffer in video-only mode. 581 renderOneBufferItem_l(*bufferItem); 582 583 // smooth out videos >= 10fps 584 mMediaClock->updateAnchor( 585 itemMediaUs, nowUs, itemMediaUs + 100000); 586 } 587 588 mBufferItems.erase(mBufferItems.begin()); 589 mNextBufferItemMediaUs = -1; 590 } else { 591 if (mNextBufferItemMediaUs == -1 592 || mNextBufferItemMediaUs > itemMediaUs) { 593 sp<AMessage> msg = new AMessage(kWhatDrainVideo, this); 594 msg->post(itemRealUs - nowUs - twoVsyncsUs); 595 mNextBufferItemMediaUs = itemMediaUs; 596 } 597 break; 598 } 599 } 600 } 601 602 void MediaSync::onFrameAvailableFromInput() { 603 Mutex::Autolock lock(mMutex); 604 605 const static nsecs_t kAcquireWaitTimeout = 2000000000; // 2 seconds 606 607 mReturnPendingInputFrame = false; 608 609 // If there are too many outstanding buffers, wait until a buffer is 610 // released back to the input in onBufferReleased. 611 // NOTE: BufferQueue allows dequeuing maxAcquiredBufferCount + 1 buffers 612 while (mNumOutstandingBuffers > mMaxAcquiredBufferCount 613 && !mIsAbandoned && !mReturnPendingInputFrame) { 614 if (mReleaseCondition.waitRelative(mMutex, kAcquireWaitTimeout) != OK) { 615 ALOGI_IF(mPlaybackRate != 0.f, "still waiting to release a buffer before acquire"); 616 } 617 618 // If the sync is abandoned while we are waiting, the release 619 // condition variable will be broadcast, and we should just return 620 // without attempting to do anything more (since the input queue will 621 // also be abandoned). 622 if (mIsAbandoned) { 623 return; 624 } 625 } 626 627 // Acquire and detach the buffer from the input. 628 BufferItem bufferItem; 629 status_t status = mInput->acquireBuffer(&bufferItem, 0 /* presentWhen */); 630 if (status != NO_ERROR) { 631 ALOGE("acquiring buffer from input failed (%d)", status); 632 return; 633 } 634 ++mNumOutstandingBuffers; 635 636 ALOGV("acquired buffer %#llx from input", (long long)bufferItem.mGraphicBuffer->getId()); 637 638 status = mInput->detachBuffer(bufferItem.mSlot); 639 if (status != NO_ERROR) { 640 ALOGE("detaching buffer from input failed (%d)", status); 641 if (status == NO_INIT) { 642 // If the input has been abandoned, move on. 643 onAbandoned_l(true /* isInput */); 644 } 645 return; 646 } 647 648 if (mBuffersFromInput.indexOfKey(bufferItem.mGraphicBuffer->getId()) >= 0) { 649 // Something is wrong since this buffer should be at our hands, bail. 650 ALOGE("received buffer multiple times from input"); 651 mInput->consumerDisconnect(); 652 onAbandoned_l(true /* isInput */); 653 return; 654 } 655 mBuffersFromInput.add(bufferItem.mGraphicBuffer->getId(), bufferItem.mGraphicBuffer); 656 657 // If flush happened while waiting for a buffer to be released, simply return it 658 // TRICKY: do it here after it is detached so that we don't have to cache mGraphicBuffer. 659 if (mReturnPendingInputFrame) { 660 mReturnPendingInputFrame = false; 661 returnBufferToInput_l(bufferItem.mGraphicBuffer, bufferItem.mFence); 662 return; 663 } 664 665 mBufferItems.push_back(bufferItem); 666 667 if (mBufferItems.size() == 1) { 668 onDrainVideo_l(); 669 } 670 } 671 672 void MediaSync::renderOneBufferItem_l(const BufferItem &bufferItem) { 673 IGraphicBufferProducer::QueueBufferInput queueInput( 674 bufferItem.mTimestamp, 675 bufferItem.mIsAutoTimestamp, 676 bufferItem.mDataSpace, 677 bufferItem.mCrop, 678 static_cast<int32_t>(bufferItem.mScalingMode), 679 bufferItem.mTransform, 680 bufferItem.mFence); 681 682 // Attach and queue the buffer to the output. 683 int slot; 684 mOutput->setGenerationNumber(bufferItem.mGraphicBuffer->getGenerationNumber()); 685 status_t status = mOutput->attachBuffer(&slot, bufferItem.mGraphicBuffer); 686 ALOGE_IF(status != NO_ERROR, "attaching buffer to output failed (%d)", status); 687 if (status == NO_ERROR) { 688 IGraphicBufferProducer::QueueBufferOutput queueOutput; 689 status = mOutput->queueBuffer(slot, queueInput, &queueOutput); 690 ALOGE_IF(status != NO_ERROR, "queueing buffer to output failed (%d)", status); 691 } 692 693 if (status != NO_ERROR) { 694 returnBufferToInput_l(bufferItem.mGraphicBuffer, bufferItem.mFence); 695 if (status == NO_INIT) { 696 // If the output has been abandoned, move on. 697 onAbandoned_l(false /* isInput */); 698 } 699 return; 700 } 701 702 if (mBuffersSentToOutput.indexOfKey(bufferItem.mGraphicBuffer->getId()) >= 0) { 703 // Something is wrong since this buffer should be held by output now, bail. 704 mInput->consumerDisconnect(); 705 onAbandoned_l(true /* isInput */); 706 return; 707 } 708 mBuffersSentToOutput.add(bufferItem.mGraphicBuffer->getId(), bufferItem.mGraphicBuffer); 709 710 ALOGV("queued buffer %#llx to output", (long long)bufferItem.mGraphicBuffer->getId()); 711 } 712 713 void MediaSync::onBufferReleasedByOutput(sp<IGraphicBufferProducer> &output) { 714 Mutex::Autolock lock(mMutex); 715 716 if (output != mOutput) { 717 return; // This is not the current output, ignore. 718 } 719 720 sp<GraphicBuffer> buffer; 721 sp<Fence> fence; 722 status_t status = mOutput->detachNextBuffer(&buffer, &fence); 723 ALOGE_IF(status != NO_ERROR, "detaching buffer from output failed (%d)", status); 724 725 if (status == NO_INIT) { 726 // If the output has been abandoned, we can't do anything else, 727 // since buffer is invalid. 728 onAbandoned_l(false /* isInput */); 729 return; 730 } 731 732 ALOGV("detached buffer %#llx from output", (long long)buffer->getId()); 733 734 // If we've been abandoned, we can't return the buffer to the input, so just 735 // move on. 736 if (mIsAbandoned) { 737 return; 738 } 739 740 ssize_t ix = mBuffersSentToOutput.indexOfKey(buffer->getId()); 741 if (ix < 0) { 742 // The buffer is unknown, maybe leftover, ignore. 743 return; 744 } 745 mBuffersSentToOutput.removeItemsAt(ix); 746 747 returnBufferToInput_l(buffer, fence); 748 } 749 750 void MediaSync::returnBufferToInput_l( 751 const sp<GraphicBuffer> &buffer, const sp<Fence> &fence) { 752 ssize_t ix = mBuffersFromInput.indexOfKey(buffer->getId()); 753 if (ix < 0) { 754 // The buffer is unknown, something is wrong, bail. 755 ALOGE("output returned unknown buffer"); 756 mOutput->disconnect(NATIVE_WINDOW_API_MEDIA); 757 onAbandoned_l(false /* isInput */); 758 return; 759 } 760 sp<GraphicBuffer> oldBuffer = mBuffersFromInput.valueAt(ix); 761 mBuffersFromInput.removeItemsAt(ix); 762 763 // Attach and release the buffer back to the input. 764 int consumerSlot; 765 status_t status = mInput->attachBuffer(&consumerSlot, oldBuffer); 766 ALOGE_IF(status != NO_ERROR, "attaching buffer to input failed (%d)", status); 767 if (status == NO_ERROR) { 768 status = mInput->releaseBuffer(consumerSlot, 0 /* frameNumber */, 769 EGL_NO_DISPLAY, EGL_NO_SYNC_KHR, fence); 770 ALOGE_IF(status != NO_ERROR, "releasing buffer to input failed (%d)", status); 771 } 772 773 // Notify any waiting onFrameAvailable calls. 774 --mNumOutstandingBuffers; 775 mReleaseCondition.signal(); 776 777 if (status == NO_ERROR) { 778 ALOGV("released buffer %#llx to input", (long long)oldBuffer->getId()); 779 } 780 } 781 782 void MediaSync::onAbandoned_l(bool isInput) { 783 ALOGE("the %s has abandoned me", (isInput ? "input" : "output")); 784 if (!mIsAbandoned) { 785 if (isInput) { 786 mOutput->disconnect(NATIVE_WINDOW_API_MEDIA); 787 } else { 788 mInput->consumerDisconnect(); 789 } 790 mIsAbandoned = true; 791 } 792 mReleaseCondition.broadcast(); 793 } 794 795 void MediaSync::onMessageReceived(const sp<AMessage> &msg) { 796 switch (msg->what()) { 797 case kWhatDrainVideo: 798 { 799 Mutex::Autolock lock(mMutex); 800 if (mNextBufferItemMediaUs != -1) { 801 int64_t nowUs = ALooper::GetNowUs(); 802 int64_t itemRealUs = getRealTime(mNextBufferItemMediaUs, nowUs); 803 804 // The message could arrive earlier than expected due to 805 // various reasons, e.g., media clock has been changed because 806 // of new anchor time or playback rate. In such cases, the 807 // message needs to be re-posted. 808 if (itemRealUs > nowUs) { 809 msg->post(itemRealUs - nowUs); 810 break; 811 } 812 } 813 814 onDrainVideo_l(); 815 break; 816 } 817 818 default: 819 TRESPASS(); 820 break; 821 } 822 } 823 824 MediaSync::InputListener::InputListener(const sp<MediaSync> &sync) 825 : mSync(sync) {} 826 827 MediaSync::InputListener::~InputListener() {} 828 829 void MediaSync::InputListener::onFrameAvailable(const BufferItem &/* item */) { 830 mSync->onFrameAvailableFromInput(); 831 } 832 833 // We don't care about sideband streams, since we won't relay them. 834 void MediaSync::InputListener::onSidebandStreamChanged() { 835 ALOGE("onSidebandStreamChanged: got sideband stream unexpectedly."); 836 } 837 838 839 void MediaSync::InputListener::binderDied(const wp<IBinder> &/* who */) { 840 Mutex::Autolock lock(mSync->mMutex); 841 mSync->onAbandoned_l(true /* isInput */); 842 } 843 844 MediaSync::OutputListener::OutputListener(const sp<MediaSync> &sync, 845 const sp<IGraphicBufferProducer> &output) 846 : mSync(sync), 847 mOutput(output) {} 848 849 MediaSync::OutputListener::~OutputListener() {} 850 851 void MediaSync::OutputListener::onBufferReleased() { 852 mSync->onBufferReleasedByOutput(mOutput); 853 } 854 855 void MediaSync::OutputListener::binderDied(const wp<IBinder> &/* who */) { 856 Mutex::Autolock lock(mSync->mMutex); 857 mSync->onAbandoned_l(false /* isInput */); 858 } 859 860 } // namespace android 861