1 /* 2 * Copyright 2014, The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17 //#define LOG_NDEBUG 0 18 #define LOG_TAG "MediaCodecSource" 19 #define DEBUG_DRIFT_TIME 0 20 21 #include <inttypes.h> 22 23 #include <gui/IGraphicBufferProducer.h> 24 #include <gui/Surface.h> 25 #include <media/ICrypto.h> 26 #include <media/MediaCodecBuffer.h> 27 #include <media/stagefright/foundation/ABuffer.h> 28 #include <media/stagefright/foundation/ADebug.h> 29 #include <media/stagefright/foundation/ALooper.h> 30 #include <media/stagefright/foundation/AMessage.h> 31 #include <media/stagefright/MediaBuffer.h> 32 #include <media/stagefright/MediaCodec.h> 33 #include <media/stagefright/MediaCodecList.h> 34 #include <media/stagefright/MediaCodecSource.h> 35 #include <media/stagefright/MediaErrors.h> 36 #include <media/stagefright/MediaSource.h> 37 #include <media/stagefright/MetaData.h> 38 #include <media/stagefright/Utils.h> 39 40 namespace android { 41 42 const int32_t kDefaultSwVideoEncoderFormat = HAL_PIXEL_FORMAT_YCbCr_420_888; 43 const int32_t kDefaultHwVideoEncoderFormat = HAL_PIXEL_FORMAT_IMPLEMENTATION_DEFINED; 44 const int32_t kDefaultVideoEncoderDataSpace = HAL_DATASPACE_V0_BT709; 45 46 const int kStopTimeoutUs = 300000; // allow 1 sec for shutting down encoder 47 48 struct MediaCodecSource::Puller : public AHandler { 49 explicit Puller(const sp<MediaSource> &source); 50 51 void interruptSource(); 52 status_t start(const sp<MetaData> &meta, const sp<AMessage> ¬ify); 53 void stop(); 54 void stopSource(); 55 void pause(); 56 void resume(); 57 58 bool readBuffer(MediaBuffer **buffer); 59 60 protected: 61 virtual void onMessageReceived(const sp<AMessage> &msg); 62 virtual ~Puller(); 63 64 private: 65 enum { 66 kWhatStart = 'msta', 67 kWhatStop, 68 kWhatPull, 69 }; 70 71 sp<MediaSource> mSource; 72 sp<AMessage> mNotify; 73 sp<ALooper> mLooper; 74 bool mIsAudio; 75 76 struct Queue { 77 Queue() 78 : mReadPendingSince(0), 79 mPaused(false), 80 mPulling(false) { } 81 int64_t mReadPendingSince; 82 bool mPaused; 83 bool mPulling; 84 Vector<MediaBuffer *> mReadBuffers; 85 86 void flush(); 87 // if queue is empty, return false and set *|buffer| to NULL . Otherwise, pop 88 // buffer from front of the queue, place it into *|buffer| and return true. 89 bool readBuffer(MediaBuffer **buffer); 90 // add a buffer to the back of the queue 91 void pushBuffer(MediaBuffer *mbuf); 92 }; 93 Mutexed<Queue> mQueue; 94 95 status_t postSynchronouslyAndReturnError(const sp<AMessage> &msg); 96 void schedulePull(); 97 void handleEOS(); 98 99 DISALLOW_EVIL_CONSTRUCTORS(Puller); 100 }; 101 102 MediaCodecSource::Puller::Puller(const sp<MediaSource> &source) 103 : mSource(source), 104 mLooper(new ALooper()), 105 mIsAudio(false) 106 { 107 sp<MetaData> meta = source->getFormat(); 108 const char *mime; 109 CHECK(meta->findCString(kKeyMIMEType, &mime)); 110 111 mIsAudio = !strncasecmp(mime, "audio/", 6); 112 113 mLooper->setName("pull_looper"); 114 } 115 116 MediaCodecSource::Puller::~Puller() { 117 mLooper->unregisterHandler(id()); 118 mLooper->stop(); 119 } 120 121 void MediaCodecSource::Puller::Queue::pushBuffer(MediaBuffer *mbuf) { 122 mReadBuffers.push_back(mbuf); 123 } 124 125 bool MediaCodecSource::Puller::Queue::readBuffer(MediaBuffer **mbuf) { 126 if (mReadBuffers.empty()) { 127 *mbuf = NULL; 128 return false; 129 } 130 *mbuf = *mReadBuffers.begin(); 131 mReadBuffers.erase(mReadBuffers.begin()); 132 return true; 133 } 134 135 void MediaCodecSource::Puller::Queue::flush() { 136 MediaBuffer *mbuf; 137 while (readBuffer(&mbuf)) { 138 // there are no null buffers in the queue 139 mbuf->release(); 140 } 141 } 142 143 bool MediaCodecSource::Puller::readBuffer(MediaBuffer **mbuf) { 144 Mutexed<Queue>::Locked queue(mQueue); 145 return queue->readBuffer(mbuf); 146 } 147 148 status_t MediaCodecSource::Puller::postSynchronouslyAndReturnError( 149 const sp<AMessage> &msg) { 150 sp<AMessage> response; 151 status_t err = msg->postAndAwaitResponse(&response); 152 153 if (err != OK) { 154 return err; 155 } 156 157 if (!response->findInt32("err", &err)) { 158 err = OK; 159 } 160 161 return err; 162 } 163 164 status_t MediaCodecSource::Puller::start(const sp<MetaData> &meta, const sp<AMessage> ¬ify) { 165 ALOGV("puller (%s) start", mIsAudio ? "audio" : "video"); 166 mLooper->start( 167 false /* runOnCallingThread */, 168 false /* canCallJava */, 169 PRIORITY_AUDIO); 170 mLooper->registerHandler(this); 171 mNotify = notify; 172 173 sp<AMessage> msg = new AMessage(kWhatStart, this); 174 msg->setObject("meta", meta); 175 return postSynchronouslyAndReturnError(msg); 176 } 177 178 void MediaCodecSource::Puller::stop() { 179 bool interrupt = false; 180 { 181 // mark stopping before actually reaching kWhatStop on the looper, so the pulling will 182 // stop. 183 Mutexed<Queue>::Locked queue(mQueue); 184 queue->mPulling = false; 185 interrupt = queue->mReadPendingSince && (queue->mReadPendingSince < ALooper::GetNowUs() - 1000000); 186 queue->flush(); // flush any unprocessed pulled buffers 187 } 188 189 if (interrupt) { 190 interruptSource(); 191 } 192 } 193 194 void MediaCodecSource::Puller::interruptSource() { 195 // call source->stop if read has been pending for over a second 196 // We have to call this outside the looper as looper is pending on the read. 197 mSource->stop(); 198 } 199 200 void MediaCodecSource::Puller::stopSource() { 201 sp<AMessage> msg = new AMessage(kWhatStop, this); 202 (void)postSynchronouslyAndReturnError(msg); 203 } 204 205 void MediaCodecSource::Puller::pause() { 206 Mutexed<Queue>::Locked queue(mQueue); 207 queue->mPaused = true; 208 } 209 210 void MediaCodecSource::Puller::resume() { 211 Mutexed<Queue>::Locked queue(mQueue); 212 queue->mPaused = false; 213 } 214 215 void MediaCodecSource::Puller::schedulePull() { 216 (new AMessage(kWhatPull, this))->post(); 217 } 218 219 void MediaCodecSource::Puller::handleEOS() { 220 ALOGV("puller (%s) posting EOS", mIsAudio ? "audio" : "video"); 221 sp<AMessage> msg = mNotify->dup(); 222 msg->setInt32("eos", 1); 223 msg->post(); 224 } 225 226 void MediaCodecSource::Puller::onMessageReceived(const sp<AMessage> &msg) { 227 switch (msg->what()) { 228 case kWhatStart: 229 { 230 sp<RefBase> obj; 231 CHECK(msg->findObject("meta", &obj)); 232 233 { 234 Mutexed<Queue>::Locked queue(mQueue); 235 queue->mPulling = true; 236 } 237 238 status_t err = mSource->start(static_cast<MetaData *>(obj.get())); 239 240 if (err == OK) { 241 schedulePull(); 242 } 243 244 sp<AMessage> response = new AMessage; 245 response->setInt32("err", err); 246 247 sp<AReplyToken> replyID; 248 CHECK(msg->senderAwaitsResponse(&replyID)); 249 response->postReply(replyID); 250 break; 251 } 252 253 case kWhatStop: 254 { 255 mSource->stop(); 256 257 sp<AMessage> response = new AMessage; 258 response->setInt32("err", OK); 259 260 sp<AReplyToken> replyID; 261 CHECK(msg->senderAwaitsResponse(&replyID)); 262 response->postReply(replyID); 263 break; 264 } 265 266 case kWhatPull: 267 { 268 Mutexed<Queue>::Locked queue(mQueue); 269 queue->mReadPendingSince = ALooper::GetNowUs(); 270 if (!queue->mPulling) { 271 handleEOS(); 272 break; 273 } 274 275 queue.unlock(); 276 MediaBuffer *mbuf = NULL; 277 status_t err = mSource->read(&mbuf); 278 queue.lock(); 279 280 queue->mReadPendingSince = 0; 281 // if we need to discard buffer 282 if (!queue->mPulling || queue->mPaused || err != OK) { 283 if (mbuf != NULL) { 284 mbuf->release(); 285 mbuf = NULL; 286 } 287 if (queue->mPulling && err == OK) { 288 msg->post(); // if simply paused, keep pulling source 289 break; 290 } else if (err == ERROR_END_OF_STREAM) { 291 ALOGV("stream ended, mbuf %p", mbuf); 292 } else if (err != OK) { 293 ALOGE("error %d reading stream.", err); 294 } 295 } 296 297 if (mbuf != NULL) { 298 queue->pushBuffer(mbuf); 299 } 300 301 queue.unlock(); 302 303 if (mbuf != NULL) { 304 mNotify->post(); 305 msg->post(); 306 } else { 307 handleEOS(); 308 } 309 break; 310 } 311 312 default: 313 TRESPASS(); 314 } 315 } 316 317 MediaCodecSource::Output::Output() 318 : mEncoderReachedEOS(false), 319 mErrorCode(OK) { 320 } 321 322 // static 323 sp<MediaCodecSource> MediaCodecSource::Create( 324 const sp<ALooper> &looper, 325 const sp<AMessage> &format, 326 const sp<MediaSource> &source, 327 const sp<PersistentSurface> &persistentSurface, 328 uint32_t flags) { 329 sp<MediaCodecSource> mediaSource = new MediaCodecSource( 330 looper, format, source, persistentSurface, flags); 331 332 if (mediaSource->init() == OK) { 333 return mediaSource; 334 } 335 return NULL; 336 } 337 338 status_t MediaCodecSource::setInputBufferTimeOffset(int64_t timeOffsetUs) { 339 sp<AMessage> msg = new AMessage(kWhatSetInputBufferTimeOffset, mReflector); 340 msg->setInt64("time-offset-us", timeOffsetUs); 341 return postSynchronouslyAndReturnError(msg); 342 } 343 344 int64_t MediaCodecSource::getFirstSampleSystemTimeUs() { 345 sp<AMessage> msg = new AMessage(kWhatGetFirstSampleSystemTimeUs, mReflector); 346 sp<AMessage> response; 347 msg->postAndAwaitResponse(&response); 348 int64_t timeUs; 349 if (!response->findInt64("time-us", &timeUs)) { 350 timeUs = -1ll; 351 } 352 return timeUs; 353 } 354 355 status_t MediaCodecSource::start(MetaData* params) { 356 sp<AMessage> msg = new AMessage(kWhatStart, mReflector); 357 msg->setObject("meta", params); 358 return postSynchronouslyAndReturnError(msg); 359 } 360 361 status_t MediaCodecSource::stop() { 362 sp<AMessage> msg = new AMessage(kWhatStop, mReflector); 363 return postSynchronouslyAndReturnError(msg); 364 } 365 366 367 status_t MediaCodecSource::setStopStimeUs(int64_t stopTimeUs) { 368 if (!(mFlags & FLAG_USE_SURFACE_INPUT)) { 369 return OK; 370 } 371 sp<AMessage> msg = new AMessage(kWhatSetStopTimeOffset, mReflector); 372 msg->setInt64("stop-time-us", stopTimeUs); 373 return postSynchronouslyAndReturnError(msg); 374 } 375 376 status_t MediaCodecSource::pause(MetaData* params) { 377 sp<AMessage> msg = new AMessage(kWhatPause, mReflector); 378 msg->setObject("meta", params); 379 msg->post(); 380 return OK; 381 } 382 383 sp<MetaData> MediaCodecSource::getFormat() { 384 Mutexed<sp<MetaData>>::Locked meta(mMeta); 385 return *meta; 386 } 387 388 sp<IGraphicBufferProducer> MediaCodecSource::getGraphicBufferProducer() { 389 CHECK(mFlags & FLAG_USE_SURFACE_INPUT); 390 return mGraphicBufferProducer; 391 } 392 393 status_t MediaCodecSource::read( 394 MediaBuffer** buffer, const ReadOptions* /* options */) { 395 Mutexed<Output>::Locked output(mOutput); 396 397 *buffer = NULL; 398 while (output->mBufferQueue.size() == 0 && !output->mEncoderReachedEOS) { 399 output.waitForCondition(output->mCond); 400 } 401 if (!output->mEncoderReachedEOS) { 402 *buffer = *output->mBufferQueue.begin(); 403 output->mBufferQueue.erase(output->mBufferQueue.begin()); 404 return OK; 405 } 406 return output->mErrorCode; 407 } 408 409 void MediaCodecSource::signalBufferReturned(MediaBuffer *buffer) { 410 buffer->setObserver(0); 411 buffer->release(); 412 } 413 414 MediaCodecSource::MediaCodecSource( 415 const sp<ALooper> &looper, 416 const sp<AMessage> &outputFormat, 417 const sp<MediaSource> &source, 418 const sp<PersistentSurface> &persistentSurface, 419 uint32_t flags) 420 : mLooper(looper), 421 mOutputFormat(outputFormat), 422 mMeta(new MetaData), 423 mFlags(flags), 424 mIsVideo(false), 425 mStarted(false), 426 mStopping(false), 427 mDoMoreWorkPending(false), 428 mSetEncoderFormat(false), 429 mEncoderFormat(0), 430 mEncoderDataSpace(0), 431 mPersistentSurface(persistentSurface), 432 mInputBufferTimeOffsetUs(0), 433 mFirstSampleSystemTimeUs(-1ll), 434 mPausePending(false), 435 mFirstSampleTimeUs(-1ll), 436 mGeneration(0) { 437 CHECK(mLooper != NULL); 438 439 AString mime; 440 CHECK(mOutputFormat->findString("mime", &mime)); 441 442 if (!strncasecmp("video/", mime.c_str(), 6)) { 443 mIsVideo = true; 444 } 445 446 if (!(mFlags & FLAG_USE_SURFACE_INPUT)) { 447 mPuller = new Puller(source); 448 } 449 } 450 451 MediaCodecSource::~MediaCodecSource() { 452 releaseEncoder(); 453 454 mCodecLooper->stop(); 455 mLooper->unregisterHandler(mReflector->id()); 456 } 457 458 status_t MediaCodecSource::init() { 459 status_t err = initEncoder(); 460 461 if (err != OK) { 462 releaseEncoder(); 463 } 464 465 return err; 466 } 467 468 status_t MediaCodecSource::initEncoder() { 469 mReflector = new AHandlerReflector<MediaCodecSource>(this); 470 mLooper->registerHandler(mReflector); 471 472 mCodecLooper = new ALooper; 473 mCodecLooper->setName("codec_looper"); 474 mCodecLooper->start(); 475 476 if (mFlags & FLAG_USE_SURFACE_INPUT) { 477 mOutputFormat->setInt32("create-input-buffers-suspended", 1); 478 } 479 480 AString outputMIME; 481 CHECK(mOutputFormat->findString("mime", &outputMIME)); 482 483 Vector<AString> matchingCodecs; 484 MediaCodecList::findMatchingCodecs( 485 outputMIME.c_str(), true /* encoder */, 486 ((mFlags & FLAG_PREFER_SOFTWARE_CODEC) ? MediaCodecList::kPreferSoftwareCodecs : 0), 487 &matchingCodecs); 488 489 status_t err = NO_INIT; 490 for (size_t ix = 0; ix < matchingCodecs.size(); ++ix) { 491 mEncoder = MediaCodec::CreateByComponentName( 492 mCodecLooper, matchingCodecs[ix]); 493 494 if (mEncoder == NULL) { 495 continue; 496 } 497 498 ALOGV("output format is '%s'", mOutputFormat->debugString(0).c_str()); 499 500 mEncoderActivityNotify = new AMessage(kWhatEncoderActivity, mReflector); 501 mEncoder->setCallback(mEncoderActivityNotify); 502 503 err = mEncoder->configure( 504 mOutputFormat, 505 NULL /* nativeWindow */, 506 NULL /* crypto */, 507 MediaCodec::CONFIGURE_FLAG_ENCODE); 508 509 if (err == OK) { 510 break; 511 } 512 mEncoder->release(); 513 mEncoder = NULL; 514 } 515 516 if (err != OK) { 517 return err; 518 } 519 520 mEncoder->getOutputFormat(&mOutputFormat); 521 sp<MetaData> meta = new MetaData; 522 convertMessageToMetaData(mOutputFormat, meta); 523 mMeta.lock().set(meta); 524 525 if (mFlags & FLAG_USE_SURFACE_INPUT) { 526 CHECK(mIsVideo); 527 528 if (mPersistentSurface != NULL) { 529 // When using persistent surface, we are only interested in the 530 // consumer, but have to use PersistentSurface as a wrapper to 531 // pass consumer over messages (similar to BufferProducerWrapper) 532 err = mEncoder->setInputSurface(mPersistentSurface); 533 } else { 534 err = mEncoder->createInputSurface(&mGraphicBufferProducer); 535 } 536 537 if (err != OK) { 538 return err; 539 } 540 } 541 542 sp<AMessage> inputFormat; 543 int32_t usingSwReadOften; 544 mSetEncoderFormat = false; 545 if (mEncoder->getInputFormat(&inputFormat) == OK) { 546 mSetEncoderFormat = true; 547 if (inputFormat->findInt32("using-sw-read-often", &usingSwReadOften) 548 && usingSwReadOften) { 549 // this is a SW encoder; signal source to allocate SW readable buffers 550 mEncoderFormat = kDefaultSwVideoEncoderFormat; 551 } else { 552 mEncoderFormat = kDefaultHwVideoEncoderFormat; 553 } 554 if (!inputFormat->findInt32("android._dataspace", &mEncoderDataSpace)) { 555 mEncoderDataSpace = kDefaultVideoEncoderDataSpace; 556 } 557 ALOGV("setting dataspace %#x, format %#x", mEncoderDataSpace, mEncoderFormat); 558 } 559 560 err = mEncoder->start(); 561 562 if (err != OK) { 563 return err; 564 } 565 566 { 567 Mutexed<Output>::Locked output(mOutput); 568 output->mEncoderReachedEOS = false; 569 output->mErrorCode = OK; 570 } 571 572 return OK; 573 } 574 575 void MediaCodecSource::releaseEncoder() { 576 if (mEncoder == NULL) { 577 return; 578 } 579 580 mEncoder->release(); 581 mEncoder.clear(); 582 } 583 584 status_t MediaCodecSource::postSynchronouslyAndReturnError( 585 const sp<AMessage> &msg) { 586 sp<AMessage> response; 587 status_t err = msg->postAndAwaitResponse(&response); 588 589 if (err != OK) { 590 return err; 591 } 592 593 if (!response->findInt32("err", &err)) { 594 err = OK; 595 } 596 597 return err; 598 } 599 600 void MediaCodecSource::signalEOS(status_t err) { 601 bool reachedEOS = false; 602 { 603 Mutexed<Output>::Locked output(mOutput); 604 reachedEOS = output->mEncoderReachedEOS; 605 if (!reachedEOS) { 606 ALOGV("encoder (%s) reached EOS", mIsVideo ? "video" : "audio"); 607 // release all unread media buffers 608 for (List<MediaBuffer*>::iterator it = output->mBufferQueue.begin(); 609 it != output->mBufferQueue.end(); it++) { 610 (*it)->release(); 611 } 612 output->mBufferQueue.clear(); 613 output->mEncoderReachedEOS = true; 614 output->mErrorCode = err; 615 output->mCond.signal(); 616 617 reachedEOS = true; 618 output.unlock(); 619 releaseEncoder(); 620 } 621 } 622 623 if (mStopping && reachedEOS) { 624 ALOGI("encoder (%s) stopped", mIsVideo ? "video" : "audio"); 625 mPuller->stopSource(); 626 ALOGV("source (%s) stopped", mIsVideo ? "video" : "audio"); 627 // posting reply to everyone that's waiting 628 List<sp<AReplyToken>>::iterator it; 629 for (it = mStopReplyIDQueue.begin(); 630 it != mStopReplyIDQueue.end(); it++) { 631 (new AMessage)->postReply(*it); 632 } 633 mStopReplyIDQueue.clear(); 634 mStopping = false; 635 ++mGeneration; 636 } 637 } 638 639 void MediaCodecSource::resume(int64_t resumeStartTimeUs) { 640 CHECK(mFlags & FLAG_USE_SURFACE_INPUT); 641 if (mEncoder != NULL) { 642 sp<AMessage> params = new AMessage; 643 params->setInt32("drop-input-frames", false); 644 if (resumeStartTimeUs > 0) { 645 params->setInt64("drop-start-time-us", resumeStartTimeUs); 646 } 647 mEncoder->setParameters(params); 648 } 649 } 650 651 status_t MediaCodecSource::feedEncoderInputBuffers() { 652 MediaBuffer* mbuf = NULL; 653 while (!mAvailEncoderInputIndices.empty() && mPuller->readBuffer(&mbuf)) { 654 size_t bufferIndex = *mAvailEncoderInputIndices.begin(); 655 mAvailEncoderInputIndices.erase(mAvailEncoderInputIndices.begin()); 656 657 int64_t timeUs = 0ll; 658 uint32_t flags = 0; 659 size_t size = 0; 660 661 if (mbuf != NULL) { 662 CHECK(mbuf->meta_data()->findInt64(kKeyTime, &timeUs)); 663 if (mFirstSampleSystemTimeUs < 0ll) { 664 mFirstSampleSystemTimeUs = systemTime() / 1000; 665 if (mPausePending) { 666 mPausePending = false; 667 onPause(mFirstSampleSystemTimeUs); 668 mbuf->release(); 669 mAvailEncoderInputIndices.push_back(bufferIndex); 670 return OK; 671 } 672 } 673 674 timeUs += mInputBufferTimeOffsetUs; 675 676 // push decoding time for video, or drift time for audio 677 if (mIsVideo) { 678 mDecodingTimeQueue.push_back(timeUs); 679 } else { 680 #if DEBUG_DRIFT_TIME 681 if (mFirstSampleTimeUs < 0ll) { 682 mFirstSampleTimeUs = timeUs; 683 } 684 int64_t driftTimeUs = 0; 685 if (mbuf->meta_data()->findInt64(kKeyDriftTime, &driftTimeUs) 686 && driftTimeUs) { 687 driftTimeUs = timeUs - mFirstSampleTimeUs - driftTimeUs; 688 } 689 mDriftTimeQueue.push_back(driftTimeUs); 690 #endif // DEBUG_DRIFT_TIME 691 } 692 693 sp<MediaCodecBuffer> inbuf; 694 status_t err = mEncoder->getInputBuffer(bufferIndex, &inbuf); 695 696 if (err != OK || inbuf == NULL || inbuf->data() == NULL 697 || mbuf->data() == NULL || mbuf->size() == 0) { 698 mbuf->release(); 699 signalEOS(); 700 break; 701 } 702 703 size = mbuf->size(); 704 705 memcpy(inbuf->data(), mbuf->data(), size); 706 707 if (mIsVideo) { 708 // video encoder will release MediaBuffer when done 709 // with underlying data. 710 inbuf->setMediaBufferBase(mbuf); 711 } else { 712 mbuf->release(); 713 } 714 } else { 715 flags = MediaCodec::BUFFER_FLAG_EOS; 716 } 717 718 status_t err = mEncoder->queueInputBuffer( 719 bufferIndex, 0, size, timeUs, flags); 720 721 if (err != OK) { 722 return err; 723 } 724 } 725 726 return OK; 727 } 728 729 status_t MediaCodecSource::onStart(MetaData *params) { 730 if (mStopping) { 731 ALOGE("Failed to start while we're stopping"); 732 return INVALID_OPERATION; 733 } 734 int64_t startTimeUs; 735 if (params == NULL || !params->findInt64(kKeyTime, &startTimeUs)) { 736 startTimeUs = -1ll; 737 } 738 739 if (mStarted) { 740 ALOGI("MediaCodecSource (%s) resuming", mIsVideo ? "video" : "audio"); 741 if (mPausePending) { 742 mPausePending = false; 743 return OK; 744 } 745 if (mIsVideo) { 746 mEncoder->requestIDRFrame(); 747 } 748 if (mFlags & FLAG_USE_SURFACE_INPUT) { 749 resume(startTimeUs); 750 } else { 751 CHECK(mPuller != NULL); 752 mPuller->resume(); 753 } 754 return OK; 755 } 756 757 ALOGI("MediaCodecSource (%s) starting", mIsVideo ? "video" : "audio"); 758 759 status_t err = OK; 760 761 if (mFlags & FLAG_USE_SURFACE_INPUT) { 762 if (mEncoder != NULL) { 763 sp<AMessage> params = new AMessage; 764 params->setInt32("drop-input-frames", false); 765 if (startTimeUs >= 0) { 766 params->setInt64("skip-frames-before", startTimeUs); 767 } 768 mEncoder->setParameters(params); 769 } 770 } else { 771 CHECK(mPuller != NULL); 772 sp<MetaData> meta = params; 773 if (mSetEncoderFormat) { 774 if (meta == NULL) { 775 meta = new MetaData; 776 } 777 meta->setInt32(kKeyPixelFormat, mEncoderFormat); 778 meta->setInt32(kKeyColorSpace, mEncoderDataSpace); 779 } 780 781 sp<AMessage> notify = new AMessage(kWhatPullerNotify, mReflector); 782 err = mPuller->start(meta.get(), notify); 783 if (err != OK) { 784 return err; 785 } 786 } 787 788 ALOGI("MediaCodecSource (%s) started", mIsVideo ? "video" : "audio"); 789 790 mStarted = true; 791 return OK; 792 } 793 794 void MediaCodecSource::onPause(int64_t pauseStartTimeUs) { 795 if ((mFlags & FLAG_USE_SURFACE_INPUT) && (mEncoder != NULL)) { 796 sp<AMessage> params = new AMessage; 797 params->setInt32("drop-input-frames", true); 798 params->setInt64("drop-start-time-us", pauseStartTimeUs); 799 mEncoder->setParameters(params); 800 } else { 801 CHECK(mPuller != NULL); 802 mPuller->pause(); 803 } 804 } 805 806 void MediaCodecSource::onMessageReceived(const sp<AMessage> &msg) { 807 switch (msg->what()) { 808 case kWhatPullerNotify: 809 { 810 int32_t eos = 0; 811 if (msg->findInt32("eos", &eos) && eos) { 812 ALOGV("puller (%s) reached EOS", mIsVideo ? "video" : "audio"); 813 signalEOS(); 814 break; 815 } 816 817 if (mEncoder == NULL) { 818 ALOGV("got msg '%s' after encoder shutdown.", msg->debugString().c_str()); 819 break; 820 } 821 822 feedEncoderInputBuffers(); 823 break; 824 } 825 case kWhatEncoderActivity: 826 { 827 if (mEncoder == NULL) { 828 break; 829 } 830 831 int32_t cbID; 832 CHECK(msg->findInt32("callbackID", &cbID)); 833 if (cbID == MediaCodec::CB_INPUT_AVAILABLE) { 834 int32_t index; 835 CHECK(msg->findInt32("index", &index)); 836 837 mAvailEncoderInputIndices.push_back(index); 838 feedEncoderInputBuffers(); 839 } else if (cbID == MediaCodec::CB_OUTPUT_FORMAT_CHANGED) { 840 status_t err = mEncoder->getOutputFormat(&mOutputFormat); 841 if (err != OK) { 842 signalEOS(err); 843 break; 844 } 845 sp<MetaData> meta = new MetaData; 846 convertMessageToMetaData(mOutputFormat, meta); 847 mMeta.lock().set(meta); 848 } else if (cbID == MediaCodec::CB_OUTPUT_AVAILABLE) { 849 int32_t index; 850 size_t offset; 851 size_t size; 852 int64_t timeUs; 853 int32_t flags; 854 855 CHECK(msg->findInt32("index", &index)); 856 CHECK(msg->findSize("offset", &offset)); 857 CHECK(msg->findSize("size", &size)); 858 CHECK(msg->findInt64("timeUs", &timeUs)); 859 CHECK(msg->findInt32("flags", &flags)); 860 861 if (flags & MediaCodec::BUFFER_FLAG_EOS) { 862 mEncoder->releaseOutputBuffer(index); 863 signalEOS(); 864 break; 865 } 866 867 sp<MediaCodecBuffer> outbuf; 868 status_t err = mEncoder->getOutputBuffer(index, &outbuf); 869 if (err != OK || outbuf == NULL || outbuf->data() == NULL 870 || outbuf->size() == 0) { 871 signalEOS(); 872 break; 873 } 874 875 MediaBuffer *mbuf = new MediaBuffer(outbuf->size()); 876 mbuf->setObserver(this); 877 mbuf->add_ref(); 878 879 if (!(flags & MediaCodec::BUFFER_FLAG_CODECCONFIG)) { 880 if (mIsVideo) { 881 int64_t decodingTimeUs; 882 if (mFlags & FLAG_USE_SURFACE_INPUT) { 883 if (mFirstSampleSystemTimeUs < 0ll) { 884 mFirstSampleSystemTimeUs = systemTime() / 1000; 885 if (mPausePending) { 886 mPausePending = false; 887 onPause(mFirstSampleSystemTimeUs); 888 mbuf->release(); 889 break; 890 } 891 } 892 // Timestamp offset is already adjusted in GraphicBufferSource. 893 // GraphicBufferSource is supposed to discard samples 894 // queued before start, and offset timeUs by start time 895 CHECK_GE(timeUs, 0ll); 896 // TODO: 897 // Decoding time for surface source is unavailable, 898 // use presentation time for now. May need to move 899 // this logic into MediaCodec. 900 decodingTimeUs = timeUs; 901 } else { 902 CHECK(!mDecodingTimeQueue.empty()); 903 decodingTimeUs = *(mDecodingTimeQueue.begin()); 904 mDecodingTimeQueue.erase(mDecodingTimeQueue.begin()); 905 } 906 mbuf->meta_data()->setInt64(kKeyDecodingTime, decodingTimeUs); 907 908 ALOGV("[video] time %" PRId64 " us (%.2f secs), dts/pts diff %" PRId64, 909 timeUs, timeUs / 1E6, decodingTimeUs - timeUs); 910 } else { 911 int64_t driftTimeUs = 0; 912 #if DEBUG_DRIFT_TIME 913 CHECK(!mDriftTimeQueue.empty()); 914 driftTimeUs = *(mDriftTimeQueue.begin()); 915 mDriftTimeQueue.erase(mDriftTimeQueue.begin()); 916 mbuf->meta_data()->setInt64(kKeyDriftTime, driftTimeUs); 917 #endif // DEBUG_DRIFT_TIME 918 ALOGV("[audio] time %" PRId64 " us (%.2f secs), drift %" PRId64, 919 timeUs, timeUs / 1E6, driftTimeUs); 920 } 921 mbuf->meta_data()->setInt64(kKeyTime, timeUs); 922 } else { 923 mbuf->meta_data()->setInt64(kKeyTime, 0ll); 924 mbuf->meta_data()->setInt32(kKeyIsCodecConfig, true); 925 } 926 if (flags & MediaCodec::BUFFER_FLAG_SYNCFRAME) { 927 mbuf->meta_data()->setInt32(kKeyIsSyncFrame, true); 928 } 929 memcpy(mbuf->data(), outbuf->data(), outbuf->size()); 930 931 { 932 Mutexed<Output>::Locked output(mOutput); 933 output->mBufferQueue.push_back(mbuf); 934 output->mCond.signal(); 935 } 936 937 mEncoder->releaseOutputBuffer(index); 938 } else if (cbID == MediaCodec::CB_ERROR) { 939 status_t err; 940 CHECK(msg->findInt32("err", &err)); 941 ALOGE("Encoder (%s) reported error : 0x%x", 942 mIsVideo ? "video" : "audio", err); 943 if (!(mFlags & FLAG_USE_SURFACE_INPUT)) { 944 mStopping = true; 945 mPuller->stop(); 946 } 947 signalEOS(); 948 } 949 break; 950 } 951 case kWhatStart: 952 { 953 sp<AReplyToken> replyID; 954 CHECK(msg->senderAwaitsResponse(&replyID)); 955 956 sp<RefBase> obj; 957 CHECK(msg->findObject("meta", &obj)); 958 MetaData *params = static_cast<MetaData *>(obj.get()); 959 960 sp<AMessage> response = new AMessage; 961 response->setInt32("err", onStart(params)); 962 response->postReply(replyID); 963 break; 964 } 965 case kWhatStop: 966 { 967 ALOGI("encoder (%s) stopping", mIsVideo ? "video" : "audio"); 968 969 sp<AReplyToken> replyID; 970 CHECK(msg->senderAwaitsResponse(&replyID)); 971 972 if (mOutput.lock()->mEncoderReachedEOS) { 973 // if we already reached EOS, reply and return now 974 ALOGI("encoder (%s) already stopped", 975 mIsVideo ? "video" : "audio"); 976 (new AMessage)->postReply(replyID); 977 break; 978 } 979 980 mStopReplyIDQueue.push_back(replyID); 981 if (mStopping) { 982 // nothing to do if we're already stopping, reply will be posted 983 // to all when we're stopped. 984 break; 985 } 986 987 mStopping = true; 988 989 // if using surface, signal source EOS and wait for EOS to come back. 990 // otherwise, stop puller (which also clears the input buffer queue) 991 // and wait for the EOS message. We cannot call source->stop() because 992 // the encoder may still be processing input buffers. 993 if (mFlags & FLAG_USE_SURFACE_INPUT) { 994 mEncoder->signalEndOfInputStream(); 995 } else { 996 mPuller->stop(); 997 } 998 999 // complete stop even if encoder/puller stalled 1000 sp<AMessage> timeoutMsg = new AMessage(kWhatStopStalled, mReflector); 1001 timeoutMsg->setInt32("generation", mGeneration); 1002 timeoutMsg->post(kStopTimeoutUs); 1003 break; 1004 } 1005 1006 case kWhatStopStalled: 1007 { 1008 int32_t generation; 1009 CHECK(msg->findInt32("generation", &generation)); 1010 if (generation != mGeneration) { 1011 break; 1012 } 1013 1014 if (!(mFlags & FLAG_USE_SURFACE_INPUT)) { 1015 ALOGV("source (%s) stopping", mIsVideo ? "video" : "audio"); 1016 mPuller->interruptSource(); 1017 ALOGV("source (%s) stopped", mIsVideo ? "video" : "audio"); 1018 } 1019 signalEOS(); 1020 break; 1021 } 1022 1023 case kWhatPause: 1024 { 1025 if (mFirstSampleSystemTimeUs < 0) { 1026 mPausePending = true; 1027 } else { 1028 sp<RefBase> obj; 1029 CHECK(msg->findObject("meta", &obj)); 1030 MetaData *params = static_cast<MetaData *>(obj.get()); 1031 int64_t pauseStartTimeUs = -1; 1032 if (params == NULL || !params->findInt64(kKeyTime, &pauseStartTimeUs)) { 1033 pauseStartTimeUs = -1ll; 1034 } 1035 onPause(pauseStartTimeUs); 1036 } 1037 break; 1038 } 1039 case kWhatSetInputBufferTimeOffset: 1040 { 1041 sp<AReplyToken> replyID; 1042 CHECK(msg->senderAwaitsResponse(&replyID)); 1043 status_t err = OK; 1044 CHECK(msg->findInt64("time-offset-us", &mInputBufferTimeOffsetUs)); 1045 1046 // Propagate the timestamp offset to GraphicBufferSource. 1047 if (mFlags & FLAG_USE_SURFACE_INPUT) { 1048 sp<AMessage> params = new AMessage; 1049 params->setInt64("time-offset-us", mInputBufferTimeOffsetUs); 1050 err = mEncoder->setParameters(params); 1051 } 1052 1053 sp<AMessage> response = new AMessage; 1054 response->setInt32("err", err); 1055 response->postReply(replyID); 1056 break; 1057 } 1058 case kWhatSetStopTimeOffset: 1059 { 1060 sp<AReplyToken> replyID; 1061 CHECK(msg->senderAwaitsResponse(&replyID)); 1062 status_t err = OK; 1063 int64_t stopTimeUs; 1064 CHECK(msg->findInt64("stop-time-us", &stopTimeUs)); 1065 1066 // Propagate the timestamp offset to GraphicBufferSource. 1067 if (mFlags & FLAG_USE_SURFACE_INPUT) { 1068 sp<AMessage> params = new AMessage; 1069 params->setInt64("stop-time-us", stopTimeUs); 1070 err = mEncoder->setParameters(params); 1071 } 1072 1073 sp<AMessage> response = new AMessage; 1074 response->setInt32("err", err); 1075 response->postReply(replyID); 1076 break; 1077 } 1078 case kWhatGetFirstSampleSystemTimeUs: 1079 { 1080 sp<AReplyToken> replyID; 1081 CHECK(msg->senderAwaitsResponse(&replyID)); 1082 1083 sp<AMessage> response = new AMessage; 1084 response->setInt64("time-us", mFirstSampleSystemTimeUs); 1085 response->postReply(replyID); 1086 break; 1087 } 1088 default: 1089 TRESPASS(); 1090 } 1091 } 1092 1093 } // namespace android 1094