1 /* 2 * Copyright 2014, The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17 //#define LOG_NDEBUG 0 18 #define LOG_TAG "MediaCodecSource" 19 #define DEBUG_DRIFT_TIME 0 20 21 #include <inttypes.h> 22 23 #include <gui/IGraphicBufferProducer.h> 24 #include <gui/Surface.h> 25 #include <media/ICrypto.h> 26 #include <media/MediaBufferHolder.h> 27 #include <media/MediaCodecBuffer.h> 28 #include <media/MediaSource.h> 29 #include <media/stagefright/foundation/ABuffer.h> 30 #include <media/stagefright/foundation/ADebug.h> 31 #include <media/stagefright/foundation/ALooper.h> 32 #include <media/stagefright/foundation/AMessage.h> 33 #include <media/stagefright/MediaBuffer.h> 34 #include <media/stagefright/MediaCodec.h> 35 #include <media/stagefright/MediaCodecList.h> 36 #include <media/stagefright/MediaCodecSource.h> 37 #include <media/stagefright/MediaErrors.h> 38 #include <media/stagefright/MetaData.h> 39 #include <media/stagefright/Utils.h> 40 41 namespace android { 42 43 const int32_t kDefaultSwVideoEncoderFormat = HAL_PIXEL_FORMAT_YCbCr_420_888; 44 const int32_t kDefaultHwVideoEncoderFormat = HAL_PIXEL_FORMAT_IMPLEMENTATION_DEFINED; 45 const int32_t kDefaultVideoEncoderDataSpace = HAL_DATASPACE_V0_BT709; 46 47 const int kStopTimeoutUs = 300000; // allow 1 sec for shutting down encoder 48 // allow maximum 1 sec for stop time offset. This limits the the delay in the 49 // input source. 50 const int kMaxStopTimeOffsetUs = 1000000; 51 52 struct MediaCodecSource::Puller : public AHandler { 53 explicit Puller(const sp<MediaSource> &source); 54 55 void interruptSource(); 56 status_t start(const sp<MetaData> &meta, const sp<AMessage> ¬ify); 57 void stop(); 58 void stopSource(); 59 void pause(); 60 void resume(); 61 status_t setStopTimeUs(int64_t stopTimeUs); 62 bool readBuffer(MediaBufferBase **buffer); 63 64 protected: 65 virtual void onMessageReceived(const sp<AMessage> &msg); 66 virtual ~Puller(); 67 68 private: 69 enum { 70 kWhatStart = 'msta', 71 kWhatStop, 72 kWhatPull, 73 kWhatSetStopTimeUs, 74 }; 75 76 sp<MediaSource> mSource; 77 sp<AMessage> mNotify; 78 sp<ALooper> mLooper; 79 bool mIsAudio; 80 81 struct Queue { 82 Queue() 83 : mReadPendingSince(0), 84 mPaused(false), 85 mPulling(false) { } 86 int64_t mReadPendingSince; 87 bool mPaused; 88 bool mPulling; 89 Vector<MediaBufferBase *> mReadBuffers; 90 91 void flush(); 92 // if queue is empty, return false and set *|buffer| to NULL . Otherwise, pop 93 // buffer from front of the queue, place it into *|buffer| and return true. 94 bool readBuffer(MediaBufferBase **buffer); 95 // add a buffer to the back of the queue 96 void pushBuffer(MediaBufferBase *mbuf); 97 }; 98 Mutexed<Queue> mQueue; 99 100 status_t postSynchronouslyAndReturnError(const sp<AMessage> &msg); 101 void schedulePull(); 102 void handleEOS(); 103 104 DISALLOW_EVIL_CONSTRUCTORS(Puller); 105 }; 106 107 MediaCodecSource::Puller::Puller(const sp<MediaSource> &source) 108 : mSource(source), 109 mLooper(new ALooper()), 110 mIsAudio(false) 111 { 112 sp<MetaData> meta = source->getFormat(); 113 const char *mime; 114 CHECK(meta->findCString(kKeyMIMEType, &mime)); 115 116 mIsAudio = !strncasecmp(mime, "audio/", 6); 117 118 mLooper->setName("pull_looper"); 119 } 120 121 MediaCodecSource::Puller::~Puller() { 122 mLooper->unregisterHandler(id()); 123 mLooper->stop(); 124 } 125 126 void MediaCodecSource::Puller::Queue::pushBuffer(MediaBufferBase *mbuf) { 127 mReadBuffers.push_back(mbuf); 128 } 129 130 bool MediaCodecSource::Puller::Queue::readBuffer(MediaBufferBase **mbuf) { 131 if (mReadBuffers.empty()) { 132 *mbuf = NULL; 133 return false; 134 } 135 *mbuf = *mReadBuffers.begin(); 136 mReadBuffers.erase(mReadBuffers.begin()); 137 return true; 138 } 139 140 void MediaCodecSource::Puller::Queue::flush() { 141 MediaBufferBase *mbuf; 142 while (readBuffer(&mbuf)) { 143 // there are no null buffers in the queue 144 mbuf->release(); 145 } 146 } 147 148 bool MediaCodecSource::Puller::readBuffer(MediaBufferBase **mbuf) { 149 Mutexed<Queue>::Locked queue(mQueue); 150 return queue->readBuffer(mbuf); 151 } 152 153 status_t MediaCodecSource::Puller::postSynchronouslyAndReturnError( 154 const sp<AMessage> &msg) { 155 sp<AMessage> response; 156 status_t err = msg->postAndAwaitResponse(&response); 157 158 if (err != OK) { 159 return err; 160 } 161 162 if (!response->findInt32("err", &err)) { 163 err = OK; 164 } 165 166 return err; 167 } 168 169 status_t MediaCodecSource::Puller::setStopTimeUs(int64_t stopTimeUs) { 170 sp<AMessage> msg = new AMessage(kWhatSetStopTimeUs, this); 171 msg->setInt64("stop-time-us", stopTimeUs); 172 return postSynchronouslyAndReturnError(msg); 173 } 174 175 status_t MediaCodecSource::Puller::start(const sp<MetaData> &meta, const sp<AMessage> ¬ify) { 176 ALOGV("puller (%s) start", mIsAudio ? "audio" : "video"); 177 mLooper->start( 178 false /* runOnCallingThread */, 179 false /* canCallJava */, 180 PRIORITY_AUDIO); 181 mLooper->registerHandler(this); 182 mNotify = notify; 183 184 sp<AMessage> msg = new AMessage(kWhatStart, this); 185 msg->setObject("meta", meta); 186 return postSynchronouslyAndReturnError(msg); 187 } 188 189 void MediaCodecSource::Puller::stop() { 190 bool interrupt = false; 191 { 192 // mark stopping before actually reaching kWhatStop on the looper, so the pulling will 193 // stop. 194 Mutexed<Queue>::Locked queue(mQueue); 195 queue->mPulling = false; 196 interrupt = queue->mReadPendingSince && (queue->mReadPendingSince < ALooper::GetNowUs() - 1000000); 197 queue->flush(); // flush any unprocessed pulled buffers 198 } 199 200 if (interrupt) { 201 interruptSource(); 202 } 203 } 204 205 void MediaCodecSource::Puller::interruptSource() { 206 // call source->stop if read has been pending for over a second 207 // We have to call this outside the looper as looper is pending on the read. 208 mSource->stop(); 209 } 210 211 void MediaCodecSource::Puller::stopSource() { 212 sp<AMessage> msg = new AMessage(kWhatStop, this); 213 (void)postSynchronouslyAndReturnError(msg); 214 } 215 216 void MediaCodecSource::Puller::pause() { 217 Mutexed<Queue>::Locked queue(mQueue); 218 queue->mPaused = true; 219 } 220 221 void MediaCodecSource::Puller::resume() { 222 Mutexed<Queue>::Locked queue(mQueue); 223 queue->mPaused = false; 224 } 225 226 void MediaCodecSource::Puller::schedulePull() { 227 (new AMessage(kWhatPull, this))->post(); 228 } 229 230 void MediaCodecSource::Puller::handleEOS() { 231 ALOGV("puller (%s) posting EOS", mIsAudio ? "audio" : "video"); 232 sp<AMessage> msg = mNotify->dup(); 233 msg->setInt32("eos", 1); 234 msg->post(); 235 } 236 237 void MediaCodecSource::Puller::onMessageReceived(const sp<AMessage> &msg) { 238 switch (msg->what()) { 239 case kWhatStart: 240 { 241 sp<RefBase> obj; 242 CHECK(msg->findObject("meta", &obj)); 243 244 { 245 Mutexed<Queue>::Locked queue(mQueue); 246 queue->mPulling = true; 247 } 248 249 status_t err = mSource->start(static_cast<MetaData *>(obj.get())); 250 251 if (err == OK) { 252 schedulePull(); 253 } 254 255 sp<AMessage> response = new AMessage; 256 response->setInt32("err", err); 257 258 sp<AReplyToken> replyID; 259 CHECK(msg->senderAwaitsResponse(&replyID)); 260 response->postReply(replyID); 261 break; 262 } 263 264 case kWhatSetStopTimeUs: 265 { 266 sp<AReplyToken> replyID; 267 CHECK(msg->senderAwaitsResponse(&replyID)); 268 int64_t stopTimeUs; 269 CHECK(msg->findInt64("stop-time-us", &stopTimeUs)); 270 status_t err = mSource->setStopTimeUs(stopTimeUs); 271 272 sp<AMessage> response = new AMessage; 273 response->setInt32("err", err); 274 response->postReply(replyID); 275 break; 276 } 277 278 case kWhatStop: 279 { 280 mSource->stop(); 281 282 sp<AMessage> response = new AMessage; 283 response->setInt32("err", OK); 284 285 sp<AReplyToken> replyID; 286 CHECK(msg->senderAwaitsResponse(&replyID)); 287 response->postReply(replyID); 288 break; 289 } 290 291 case kWhatPull: 292 { 293 Mutexed<Queue>::Locked queue(mQueue); 294 queue->mReadPendingSince = ALooper::GetNowUs(); 295 if (!queue->mPulling) { 296 handleEOS(); 297 break; 298 } 299 300 queue.unlock(); 301 MediaBufferBase *mbuf = NULL; 302 status_t err = mSource->read(&mbuf); 303 queue.lock(); 304 305 queue->mReadPendingSince = 0; 306 // if we need to discard buffer 307 if (!queue->mPulling || queue->mPaused || err != OK) { 308 if (mbuf != NULL) { 309 mbuf->release(); 310 mbuf = NULL; 311 } 312 if (queue->mPulling && err == OK) { 313 msg->post(); // if simply paused, keep pulling source 314 break; 315 } else if (err == ERROR_END_OF_STREAM) { 316 ALOGV("stream ended, mbuf %p", mbuf); 317 } else if (err != OK) { 318 ALOGE("error %d reading stream.", err); 319 } 320 } 321 322 if (mbuf != NULL) { 323 queue->pushBuffer(mbuf); 324 } 325 326 queue.unlock(); 327 328 if (mbuf != NULL) { 329 mNotify->post(); 330 msg->post(); 331 } else { 332 handleEOS(); 333 } 334 break; 335 } 336 337 default: 338 TRESPASS(); 339 } 340 } 341 342 MediaCodecSource::Output::Output() 343 : mEncoderReachedEOS(false), 344 mErrorCode(OK) { 345 } 346 347 // static 348 sp<MediaCodecSource> MediaCodecSource::Create( 349 const sp<ALooper> &looper, 350 const sp<AMessage> &format, 351 const sp<MediaSource> &source, 352 const sp<PersistentSurface> &persistentSurface, 353 uint32_t flags) { 354 sp<MediaCodecSource> mediaSource = new MediaCodecSource( 355 looper, format, source, persistentSurface, flags); 356 357 if (mediaSource->init() == OK) { 358 return mediaSource; 359 } 360 return NULL; 361 } 362 363 status_t MediaCodecSource::setInputBufferTimeOffset(int64_t timeOffsetUs) { 364 sp<AMessage> msg = new AMessage(kWhatSetInputBufferTimeOffset, mReflector); 365 msg->setInt64("time-offset-us", timeOffsetUs); 366 return postSynchronouslyAndReturnError(msg); 367 } 368 369 int64_t MediaCodecSource::getFirstSampleSystemTimeUs() { 370 sp<AMessage> msg = new AMessage(kWhatGetFirstSampleSystemTimeUs, mReflector); 371 sp<AMessage> response; 372 msg->postAndAwaitResponse(&response); 373 int64_t timeUs; 374 if (!response->findInt64("time-us", &timeUs)) { 375 timeUs = -1ll; 376 } 377 return timeUs; 378 } 379 380 status_t MediaCodecSource::start(MetaData* params) { 381 sp<AMessage> msg = new AMessage(kWhatStart, mReflector); 382 msg->setObject("meta", params); 383 return postSynchronouslyAndReturnError(msg); 384 } 385 386 status_t MediaCodecSource::stop() { 387 sp<AMessage> msg = new AMessage(kWhatStop, mReflector); 388 return postSynchronouslyAndReturnError(msg); 389 } 390 391 392 status_t MediaCodecSource::setStopTimeUs(int64_t stopTimeUs) { 393 sp<AMessage> msg = new AMessage(kWhatSetStopTimeUs, mReflector); 394 msg->setInt64("stop-time-us", stopTimeUs); 395 return postSynchronouslyAndReturnError(msg); 396 } 397 398 status_t MediaCodecSource::pause(MetaData* params) { 399 sp<AMessage> msg = new AMessage(kWhatPause, mReflector); 400 msg->setObject("meta", params); 401 msg->post(); 402 return OK; 403 } 404 405 sp<MetaData> MediaCodecSource::getFormat() { 406 Mutexed<sp<MetaData>>::Locked meta(mMeta); 407 return *meta; 408 } 409 410 sp<IGraphicBufferProducer> MediaCodecSource::getGraphicBufferProducer() { 411 CHECK(mFlags & FLAG_USE_SURFACE_INPUT); 412 return mGraphicBufferProducer; 413 } 414 415 status_t MediaCodecSource::read( 416 MediaBufferBase** buffer, const ReadOptions* /* options */) { 417 Mutexed<Output>::Locked output(mOutput); 418 419 *buffer = NULL; 420 while (output->mBufferQueue.size() == 0 && !output->mEncoderReachedEOS) { 421 output.waitForCondition(output->mCond); 422 } 423 if (!output->mEncoderReachedEOS) { 424 *buffer = *output->mBufferQueue.begin(); 425 output->mBufferQueue.erase(output->mBufferQueue.begin()); 426 return OK; 427 } 428 return output->mErrorCode; 429 } 430 431 void MediaCodecSource::signalBufferReturned(MediaBufferBase *buffer) { 432 buffer->setObserver(0); 433 buffer->release(); 434 } 435 436 MediaCodecSource::MediaCodecSource( 437 const sp<ALooper> &looper, 438 const sp<AMessage> &outputFormat, 439 const sp<MediaSource> &source, 440 const sp<PersistentSurface> &persistentSurface, 441 uint32_t flags) 442 : mLooper(looper), 443 mOutputFormat(outputFormat), 444 mMeta(new MetaData), 445 mFlags(flags), 446 mIsVideo(false), 447 mStarted(false), 448 mStopping(false), 449 mDoMoreWorkPending(false), 450 mSetEncoderFormat(false), 451 mEncoderFormat(0), 452 mEncoderDataSpace(0), 453 mPersistentSurface(persistentSurface), 454 mInputBufferTimeOffsetUs(0), 455 mFirstSampleSystemTimeUs(-1ll), 456 mPausePending(false), 457 mFirstSampleTimeUs(-1ll), 458 mGeneration(0) { 459 CHECK(mLooper != NULL); 460 461 if (!(mFlags & FLAG_USE_SURFACE_INPUT)) { 462 mPuller = new Puller(source); 463 } 464 } 465 466 MediaCodecSource::~MediaCodecSource() { 467 releaseEncoder(); 468 469 mCodecLooper->stop(); 470 mLooper->unregisterHandler(mReflector->id()); 471 } 472 473 status_t MediaCodecSource::init() { 474 status_t err = initEncoder(); 475 476 if (err != OK) { 477 releaseEncoder(); 478 } 479 480 return err; 481 } 482 483 status_t MediaCodecSource::initEncoder() { 484 485 mReflector = new AHandlerReflector<MediaCodecSource>(this); 486 mLooper->registerHandler(mReflector); 487 488 mCodecLooper = new ALooper; 489 mCodecLooper->setName("codec_looper"); 490 mCodecLooper->start(); 491 492 if (mFlags & FLAG_USE_SURFACE_INPUT) { 493 mOutputFormat->setInt32("create-input-buffers-suspended", 1); 494 } 495 496 AString outputMIME; 497 CHECK(mOutputFormat->findString("mime", &outputMIME)); 498 mIsVideo = outputMIME.startsWithIgnoreCase("video/"); 499 500 AString name; 501 status_t err = NO_INIT; 502 if (mOutputFormat->findString("testing-name", &name)) { 503 mEncoder = MediaCodec::CreateByComponentName(mCodecLooper, name); 504 505 mEncoderActivityNotify = new AMessage(kWhatEncoderActivity, mReflector); 506 mEncoder->setCallback(mEncoderActivityNotify); 507 508 err = mEncoder->configure( 509 mOutputFormat, 510 NULL /* nativeWindow */, 511 NULL /* crypto */, 512 MediaCodec::CONFIGURE_FLAG_ENCODE); 513 } else { 514 Vector<AString> matchingCodecs; 515 MediaCodecList::findMatchingCodecs( 516 outputMIME.c_str(), true /* encoder */, 517 ((mFlags & FLAG_PREFER_SOFTWARE_CODEC) ? MediaCodecList::kPreferSoftwareCodecs : 0), 518 &matchingCodecs); 519 520 for (size_t ix = 0; ix < matchingCodecs.size(); ++ix) { 521 mEncoder = MediaCodec::CreateByComponentName( 522 mCodecLooper, matchingCodecs[ix]); 523 524 if (mEncoder == NULL) { 525 continue; 526 } 527 528 ALOGV("output format is '%s'", mOutputFormat->debugString(0).c_str()); 529 530 mEncoderActivityNotify = new AMessage(kWhatEncoderActivity, mReflector); 531 mEncoder->setCallback(mEncoderActivityNotify); 532 533 err = mEncoder->configure( 534 mOutputFormat, 535 NULL /* nativeWindow */, 536 NULL /* crypto */, 537 MediaCodec::CONFIGURE_FLAG_ENCODE); 538 539 if (err == OK) { 540 break; 541 } 542 mEncoder->release(); 543 mEncoder = NULL; 544 } 545 } 546 547 if (err != OK) { 548 return err; 549 } 550 551 mEncoder->getOutputFormat(&mOutputFormat); 552 sp<MetaData> meta = new MetaData; 553 convertMessageToMetaData(mOutputFormat, meta); 554 mMeta.lock().set(meta); 555 556 if (mFlags & FLAG_USE_SURFACE_INPUT) { 557 CHECK(mIsVideo); 558 559 if (mPersistentSurface != NULL) { 560 // When using persistent surface, we are only interested in the 561 // consumer, but have to use PersistentSurface as a wrapper to 562 // pass consumer over messages (similar to BufferProducerWrapper) 563 err = mEncoder->setInputSurface(mPersistentSurface); 564 } else { 565 err = mEncoder->createInputSurface(&mGraphicBufferProducer); 566 } 567 568 if (err != OK) { 569 return err; 570 } 571 } 572 573 sp<AMessage> inputFormat; 574 int32_t usingSwReadOften; 575 mSetEncoderFormat = false; 576 if (mEncoder->getInputFormat(&inputFormat) == OK) { 577 mSetEncoderFormat = true; 578 if (inputFormat->findInt32("using-sw-read-often", &usingSwReadOften) 579 && usingSwReadOften) { 580 // this is a SW encoder; signal source to allocate SW readable buffers 581 mEncoderFormat = kDefaultSwVideoEncoderFormat; 582 } else { 583 mEncoderFormat = kDefaultHwVideoEncoderFormat; 584 } 585 if (!inputFormat->findInt32("android._dataspace", &mEncoderDataSpace)) { 586 mEncoderDataSpace = kDefaultVideoEncoderDataSpace; 587 } 588 ALOGV("setting dataspace %#x, format %#x", mEncoderDataSpace, mEncoderFormat); 589 } 590 591 err = mEncoder->start(); 592 593 if (err != OK) { 594 return err; 595 } 596 597 { 598 Mutexed<Output>::Locked output(mOutput); 599 output->mEncoderReachedEOS = false; 600 output->mErrorCode = OK; 601 } 602 603 return OK; 604 } 605 606 void MediaCodecSource::releaseEncoder() { 607 if (mEncoder == NULL) { 608 return; 609 } 610 611 mEncoder->release(); 612 mEncoder.clear(); 613 } 614 615 status_t MediaCodecSource::postSynchronouslyAndReturnError( 616 const sp<AMessage> &msg) { 617 sp<AMessage> response; 618 status_t err = msg->postAndAwaitResponse(&response); 619 620 if (err != OK) { 621 return err; 622 } 623 624 if (!response->findInt32("err", &err)) { 625 err = OK; 626 } 627 628 return err; 629 } 630 631 void MediaCodecSource::signalEOS(status_t err) { 632 bool reachedEOS = false; 633 { 634 Mutexed<Output>::Locked output(mOutput); 635 reachedEOS = output->mEncoderReachedEOS; 636 if (!reachedEOS) { 637 ALOGV("encoder (%s) reached EOS", mIsVideo ? "video" : "audio"); 638 // release all unread media buffers 639 for (List<MediaBufferBase*>::iterator it = output->mBufferQueue.begin(); 640 it != output->mBufferQueue.end(); it++) { 641 (*it)->release(); 642 } 643 output->mBufferQueue.clear(); 644 output->mEncoderReachedEOS = true; 645 output->mErrorCode = err; 646 output->mCond.signal(); 647 648 reachedEOS = true; 649 output.unlock(); 650 releaseEncoder(); 651 } 652 } 653 654 if (mStopping && reachedEOS) { 655 ALOGI("encoder (%s) stopped", mIsVideo ? "video" : "audio"); 656 if (mPuller != NULL) { 657 mPuller->stopSource(); 658 } 659 ALOGV("source (%s) stopped", mIsVideo ? "video" : "audio"); 660 // posting reply to everyone that's waiting 661 List<sp<AReplyToken>>::iterator it; 662 for (it = mStopReplyIDQueue.begin(); 663 it != mStopReplyIDQueue.end(); it++) { 664 (new AMessage)->postReply(*it); 665 } 666 mStopReplyIDQueue.clear(); 667 mStopping = false; 668 ++mGeneration; 669 } 670 } 671 672 void MediaCodecSource::resume(int64_t resumeStartTimeUs) { 673 CHECK(mFlags & FLAG_USE_SURFACE_INPUT); 674 if (mEncoder != NULL) { 675 sp<AMessage> params = new AMessage; 676 params->setInt32("drop-input-frames", false); 677 if (resumeStartTimeUs > 0) { 678 params->setInt64("drop-start-time-us", resumeStartTimeUs); 679 } 680 mEncoder->setParameters(params); 681 } 682 } 683 684 status_t MediaCodecSource::feedEncoderInputBuffers() { 685 MediaBufferBase* mbuf = NULL; 686 while (!mAvailEncoderInputIndices.empty() && mPuller->readBuffer(&mbuf)) { 687 size_t bufferIndex = *mAvailEncoderInputIndices.begin(); 688 mAvailEncoderInputIndices.erase(mAvailEncoderInputIndices.begin()); 689 690 int64_t timeUs = 0ll; 691 uint32_t flags = 0; 692 size_t size = 0; 693 694 if (mbuf != NULL) { 695 CHECK(mbuf->meta_data().findInt64(kKeyTime, &timeUs)); 696 if (mFirstSampleSystemTimeUs < 0ll) { 697 mFirstSampleSystemTimeUs = systemTime() / 1000; 698 if (mPausePending) { 699 mPausePending = false; 700 onPause(mFirstSampleSystemTimeUs); 701 mbuf->release(); 702 mAvailEncoderInputIndices.push_back(bufferIndex); 703 return OK; 704 } 705 } 706 707 timeUs += mInputBufferTimeOffsetUs; 708 709 // push decoding time for video, or drift time for audio 710 if (mIsVideo) { 711 mDecodingTimeQueue.push_back(timeUs); 712 } else { 713 #if DEBUG_DRIFT_TIME 714 if (mFirstSampleTimeUs < 0ll) { 715 mFirstSampleTimeUs = timeUs; 716 } 717 int64_t driftTimeUs = 0; 718 if (mbuf->meta_data().findInt64(kKeyDriftTime, &driftTimeUs) 719 && driftTimeUs) { 720 driftTimeUs = timeUs - mFirstSampleTimeUs - driftTimeUs; 721 } 722 mDriftTimeQueue.push_back(driftTimeUs); 723 #endif // DEBUG_DRIFT_TIME 724 } 725 726 sp<MediaCodecBuffer> inbuf; 727 status_t err = mEncoder->getInputBuffer(bufferIndex, &inbuf); 728 729 if (err != OK || inbuf == NULL || inbuf->data() == NULL 730 || mbuf->data() == NULL || mbuf->size() == 0) { 731 mbuf->release(); 732 signalEOS(); 733 break; 734 } 735 736 size = mbuf->size(); 737 738 memcpy(inbuf->data(), mbuf->data(), size); 739 740 if (mIsVideo) { 741 // video encoder will release MediaBuffer when done 742 // with underlying data. 743 inbuf->meta()->setObject("mediaBufferHolder", new MediaBufferHolder(mbuf)); 744 mbuf->release(); 745 } else { 746 mbuf->release(); 747 } 748 } else { 749 flags = MediaCodec::BUFFER_FLAG_EOS; 750 } 751 752 status_t err = mEncoder->queueInputBuffer( 753 bufferIndex, 0, size, timeUs, flags); 754 755 if (err != OK) { 756 return err; 757 } 758 } 759 760 return OK; 761 } 762 763 status_t MediaCodecSource::onStart(MetaData *params) { 764 if (mStopping) { 765 ALOGE("Failed to start while we're stopping"); 766 return INVALID_OPERATION; 767 } 768 int64_t startTimeUs; 769 if (params == NULL || !params->findInt64(kKeyTime, &startTimeUs)) { 770 startTimeUs = -1ll; 771 } 772 773 if (mStarted) { 774 ALOGI("MediaCodecSource (%s) resuming", mIsVideo ? "video" : "audio"); 775 if (mPausePending) { 776 mPausePending = false; 777 return OK; 778 } 779 if (mIsVideo) { 780 mEncoder->requestIDRFrame(); 781 } 782 if (mFlags & FLAG_USE_SURFACE_INPUT) { 783 resume(startTimeUs); 784 } else { 785 CHECK(mPuller != NULL); 786 mPuller->resume(); 787 } 788 return OK; 789 } 790 791 ALOGI("MediaCodecSource (%s) starting", mIsVideo ? "video" : "audio"); 792 793 status_t err = OK; 794 795 if (mFlags & FLAG_USE_SURFACE_INPUT) { 796 if (mEncoder != NULL) { 797 sp<AMessage> params = new AMessage; 798 params->setInt32("drop-input-frames", false); 799 if (startTimeUs >= 0) { 800 params->setInt64("skip-frames-before", startTimeUs); 801 } 802 mEncoder->setParameters(params); 803 } 804 } else { 805 CHECK(mPuller != NULL); 806 sp<MetaData> meta = params; 807 if (mSetEncoderFormat) { 808 if (meta == NULL) { 809 meta = new MetaData; 810 } 811 meta->setInt32(kKeyPixelFormat, mEncoderFormat); 812 meta->setInt32(kKeyColorSpace, mEncoderDataSpace); 813 } 814 815 sp<AMessage> notify = new AMessage(kWhatPullerNotify, mReflector); 816 err = mPuller->start(meta.get(), notify); 817 if (err != OK) { 818 return err; 819 } 820 } 821 822 ALOGI("MediaCodecSource (%s) started", mIsVideo ? "video" : "audio"); 823 824 mStarted = true; 825 return OK; 826 } 827 828 void MediaCodecSource::onPause(int64_t pauseStartTimeUs) { 829 if ((mFlags & FLAG_USE_SURFACE_INPUT) && (mEncoder != NULL)) { 830 sp<AMessage> params = new AMessage; 831 params->setInt32("drop-input-frames", true); 832 params->setInt64("drop-start-time-us", pauseStartTimeUs); 833 mEncoder->setParameters(params); 834 } else { 835 CHECK(mPuller != NULL); 836 mPuller->pause(); 837 } 838 } 839 840 void MediaCodecSource::onMessageReceived(const sp<AMessage> &msg) { 841 switch (msg->what()) { 842 case kWhatPullerNotify: 843 { 844 int32_t eos = 0; 845 if (msg->findInt32("eos", &eos) && eos) { 846 ALOGV("puller (%s) reached EOS", mIsVideo ? "video" : "audio"); 847 signalEOS(); 848 break; 849 } 850 851 if (mEncoder == NULL) { 852 ALOGV("got msg '%s' after encoder shutdown.", msg->debugString().c_str()); 853 break; 854 } 855 856 feedEncoderInputBuffers(); 857 break; 858 } 859 case kWhatEncoderActivity: 860 { 861 if (mEncoder == NULL) { 862 break; 863 } 864 865 int32_t cbID; 866 CHECK(msg->findInt32("callbackID", &cbID)); 867 if (cbID == MediaCodec::CB_INPUT_AVAILABLE) { 868 int32_t index; 869 CHECK(msg->findInt32("index", &index)); 870 871 mAvailEncoderInputIndices.push_back(index); 872 feedEncoderInputBuffers(); 873 } else if (cbID == MediaCodec::CB_OUTPUT_FORMAT_CHANGED) { 874 status_t err = mEncoder->getOutputFormat(&mOutputFormat); 875 if (err != OK) { 876 signalEOS(err); 877 break; 878 } 879 sp<MetaData> meta = new MetaData; 880 convertMessageToMetaData(mOutputFormat, meta); 881 mMeta.lock().set(meta); 882 } else if (cbID == MediaCodec::CB_OUTPUT_AVAILABLE) { 883 int32_t index; 884 size_t offset; 885 size_t size; 886 int64_t timeUs; 887 int32_t flags; 888 889 CHECK(msg->findInt32("index", &index)); 890 CHECK(msg->findSize("offset", &offset)); 891 CHECK(msg->findSize("size", &size)); 892 CHECK(msg->findInt64("timeUs", &timeUs)); 893 CHECK(msg->findInt32("flags", &flags)); 894 895 if (flags & MediaCodec::BUFFER_FLAG_EOS) { 896 mEncoder->releaseOutputBuffer(index); 897 signalEOS(); 898 break; 899 } 900 901 sp<MediaCodecBuffer> outbuf; 902 status_t err = mEncoder->getOutputBuffer(index, &outbuf); 903 if (err != OK || outbuf == NULL || outbuf->data() == NULL 904 || outbuf->size() == 0) { 905 signalEOS(); 906 break; 907 } 908 909 MediaBufferBase *mbuf = new MediaBuffer(outbuf->size()); 910 mbuf->setObserver(this); 911 mbuf->add_ref(); 912 913 if (!(flags & MediaCodec::BUFFER_FLAG_CODECCONFIG)) { 914 if (mIsVideo) { 915 int64_t decodingTimeUs; 916 if (mFlags & FLAG_USE_SURFACE_INPUT) { 917 if (mFirstSampleSystemTimeUs < 0ll) { 918 mFirstSampleSystemTimeUs = systemTime() / 1000; 919 if (mPausePending) { 920 mPausePending = false; 921 onPause(mFirstSampleSystemTimeUs); 922 mbuf->release(); 923 break; 924 } 925 } 926 // Timestamp offset is already adjusted in GraphicBufferSource. 927 // GraphicBufferSource is supposed to discard samples 928 // queued before start, and offset timeUs by start time 929 CHECK_GE(timeUs, 0ll); 930 // TODO: 931 // Decoding time for surface source is unavailable, 932 // use presentation time for now. May need to move 933 // this logic into MediaCodec. 934 decodingTimeUs = timeUs; 935 } else { 936 CHECK(!mDecodingTimeQueue.empty()); 937 decodingTimeUs = *(mDecodingTimeQueue.begin()); 938 mDecodingTimeQueue.erase(mDecodingTimeQueue.begin()); 939 } 940 mbuf->meta_data().setInt64(kKeyDecodingTime, decodingTimeUs); 941 942 ALOGV("[video] time %" PRId64 " us (%.2f secs), dts/pts diff %" PRId64, 943 timeUs, timeUs / 1E6, decodingTimeUs - timeUs); 944 } else { 945 int64_t driftTimeUs = 0; 946 #if DEBUG_DRIFT_TIME 947 CHECK(!mDriftTimeQueue.empty()); 948 driftTimeUs = *(mDriftTimeQueue.begin()); 949 mDriftTimeQueue.erase(mDriftTimeQueue.begin()); 950 mbuf->meta_data().setInt64(kKeyDriftTime, driftTimeUs); 951 #endif // DEBUG_DRIFT_TIME 952 ALOGV("[audio] time %" PRId64 " us (%.2f secs), drift %" PRId64, 953 timeUs, timeUs / 1E6, driftTimeUs); 954 } 955 mbuf->meta_data().setInt64(kKeyTime, timeUs); 956 } else { 957 mbuf->meta_data().setInt64(kKeyTime, 0ll); 958 mbuf->meta_data().setInt32(kKeyIsCodecConfig, true); 959 } 960 if (flags & MediaCodec::BUFFER_FLAG_SYNCFRAME) { 961 mbuf->meta_data().setInt32(kKeyIsSyncFrame, true); 962 } 963 memcpy(mbuf->data(), outbuf->data(), outbuf->size()); 964 965 { 966 Mutexed<Output>::Locked output(mOutput); 967 output->mBufferQueue.push_back(mbuf); 968 output->mCond.signal(); 969 } 970 971 mEncoder->releaseOutputBuffer(index); 972 } else if (cbID == MediaCodec::CB_ERROR) { 973 status_t err; 974 CHECK(msg->findInt32("err", &err)); 975 ALOGE("Encoder (%s) reported error : 0x%x", 976 mIsVideo ? "video" : "audio", err); 977 if (!(mFlags & FLAG_USE_SURFACE_INPUT)) { 978 mStopping = true; 979 mPuller->stop(); 980 } 981 signalEOS(); 982 } 983 break; 984 } 985 case kWhatStart: 986 { 987 sp<AReplyToken> replyID; 988 CHECK(msg->senderAwaitsResponse(&replyID)); 989 990 sp<RefBase> obj; 991 CHECK(msg->findObject("meta", &obj)); 992 MetaData *params = static_cast<MetaData *>(obj.get()); 993 994 sp<AMessage> response = new AMessage; 995 response->setInt32("err", onStart(params)); 996 response->postReply(replyID); 997 break; 998 } 999 case kWhatStop: 1000 { 1001 ALOGI("encoder (%s) stopping", mIsVideo ? "video" : "audio"); 1002 1003 sp<AReplyToken> replyID; 1004 CHECK(msg->senderAwaitsResponse(&replyID)); 1005 1006 if (mOutput.lock()->mEncoderReachedEOS) { 1007 // if we already reached EOS, reply and return now 1008 ALOGI("encoder (%s) already stopped", 1009 mIsVideo ? "video" : "audio"); 1010 (new AMessage)->postReply(replyID); 1011 break; 1012 } 1013 1014 mStopReplyIDQueue.push_back(replyID); 1015 if (mStopping) { 1016 // nothing to do if we're already stopping, reply will be posted 1017 // to all when we're stopped. 1018 break; 1019 } 1020 1021 mStopping = true; 1022 1023 int64_t timeoutUs = kStopTimeoutUs; 1024 // if using surface, signal source EOS and wait for EOS to come back. 1025 // otherwise, stop puller (which also clears the input buffer queue) 1026 // and wait for the EOS message. We cannot call source->stop() because 1027 // the encoder may still be processing input buffers. 1028 if (mFlags & FLAG_USE_SURFACE_INPUT) { 1029 mEncoder->signalEndOfInputStream(); 1030 // Increase the timeout if there is delay in the GraphicBufferSource 1031 sp<AMessage> inputFormat; 1032 int64_t stopTimeOffsetUs; 1033 if (mEncoder->getInputFormat(&inputFormat) == OK && 1034 inputFormat->findInt64("android._stop-time-offset-us", &stopTimeOffsetUs) && 1035 stopTimeOffsetUs > 0) { 1036 if (stopTimeOffsetUs > kMaxStopTimeOffsetUs) { 1037 ALOGW("Source stopTimeOffsetUs %lld too large, limit at %lld us", 1038 (long long)stopTimeOffsetUs, (long long)kMaxStopTimeOffsetUs); 1039 stopTimeOffsetUs = kMaxStopTimeOffsetUs; 1040 } 1041 timeoutUs += stopTimeOffsetUs; 1042 } else { 1043 // Use kMaxStopTimeOffsetUs if stop time offset is not provided by input source 1044 timeoutUs = kMaxStopTimeOffsetUs; 1045 } 1046 } else { 1047 mPuller->stop(); 1048 } 1049 1050 // complete stop even if encoder/puller stalled 1051 sp<AMessage> timeoutMsg = new AMessage(kWhatStopStalled, mReflector); 1052 timeoutMsg->setInt32("generation", mGeneration); 1053 timeoutMsg->post(timeoutUs); 1054 break; 1055 } 1056 1057 case kWhatStopStalled: 1058 { 1059 int32_t generation; 1060 CHECK(msg->findInt32("generation", &generation)); 1061 if (generation != mGeneration) { 1062 break; 1063 } 1064 1065 if (!(mFlags & FLAG_USE_SURFACE_INPUT)) { 1066 ALOGV("source (%s) stopping", mIsVideo ? "video" : "audio"); 1067 mPuller->interruptSource(); 1068 ALOGV("source (%s) stopped", mIsVideo ? "video" : "audio"); 1069 } 1070 signalEOS(); 1071 break; 1072 } 1073 1074 case kWhatPause: 1075 { 1076 if (mFirstSampleSystemTimeUs < 0) { 1077 mPausePending = true; 1078 } else { 1079 sp<RefBase> obj; 1080 CHECK(msg->findObject("meta", &obj)); 1081 MetaData *params = static_cast<MetaData *>(obj.get()); 1082 int64_t pauseStartTimeUs = -1; 1083 if (params == NULL || !params->findInt64(kKeyTime, &pauseStartTimeUs)) { 1084 pauseStartTimeUs = -1ll; 1085 } 1086 onPause(pauseStartTimeUs); 1087 } 1088 break; 1089 } 1090 case kWhatSetInputBufferTimeOffset: 1091 { 1092 sp<AReplyToken> replyID; 1093 CHECK(msg->senderAwaitsResponse(&replyID)); 1094 status_t err = OK; 1095 CHECK(msg->findInt64("time-offset-us", &mInputBufferTimeOffsetUs)); 1096 1097 // Propagate the timestamp offset to GraphicBufferSource. 1098 if (mFlags & FLAG_USE_SURFACE_INPUT) { 1099 sp<AMessage> params = new AMessage; 1100 params->setInt64("time-offset-us", mInputBufferTimeOffsetUs); 1101 err = mEncoder->setParameters(params); 1102 } 1103 1104 sp<AMessage> response = new AMessage; 1105 response->setInt32("err", err); 1106 response->postReply(replyID); 1107 break; 1108 } 1109 case kWhatSetStopTimeUs: 1110 { 1111 sp<AReplyToken> replyID; 1112 CHECK(msg->senderAwaitsResponse(&replyID)); 1113 status_t err = OK; 1114 int64_t stopTimeUs; 1115 CHECK(msg->findInt64("stop-time-us", &stopTimeUs)); 1116 1117 // Propagate the stop time to GraphicBufferSource. 1118 if (mFlags & FLAG_USE_SURFACE_INPUT) { 1119 sp<AMessage> params = new AMessage; 1120 params->setInt64("stop-time-us", stopTimeUs); 1121 err = mEncoder->setParameters(params); 1122 } else { 1123 err = mPuller->setStopTimeUs(stopTimeUs); 1124 } 1125 1126 sp<AMessage> response = new AMessage; 1127 response->setInt32("err", err); 1128 response->postReply(replyID); 1129 break; 1130 } 1131 case kWhatGetFirstSampleSystemTimeUs: 1132 { 1133 sp<AReplyToken> replyID; 1134 CHECK(msg->senderAwaitsResponse(&replyID)); 1135 1136 sp<AMessage> response = new AMessage; 1137 response->setInt64("time-us", mFirstSampleSystemTimeUs); 1138 response->postReply(replyID); 1139 break; 1140 } 1141 default: 1142 TRESPASS(); 1143 } 1144 } 1145 1146 } // namespace android 1147