1 /* 2 * Copyright 2012, The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17 //#define LOG_NDEBUG 0 18 #define LOG_TAG "PlaybackSession" 19 #include <utils/Log.h> 20 21 #include "PlaybackSession.h" 22 23 #include "Converter.h" 24 #include "MediaPuller.h" 25 #include "RepeaterSource.h" 26 #include "include/avc_utils.h" 27 #include "WifiDisplaySource.h" 28 29 #include <binder/IServiceManager.h> 30 #include <cutils/properties.h> 31 #include <media/IHDCP.h> 32 #include <media/stagefright/foundation/ABitReader.h> 33 #include <media/stagefright/foundation/ABuffer.h> 34 #include <media/stagefright/foundation/ADebug.h> 35 #include <media/stagefright/foundation/AMessage.h> 36 #include <media/stagefright/foundation/hexdump.h> 37 #include <media/stagefright/AudioSource.h> 38 #include <media/stagefright/DataSource.h> 39 #include <media/stagefright/MediaDefs.h> 40 #include <media/stagefright/MediaErrors.h> 41 #include <media/stagefright/MediaSource.h> 42 #include <media/stagefright/MetaData.h> 43 #include <media/stagefright/NuMediaExtractor.h> 44 #include <media/stagefright/SurfaceMediaSource.h> 45 #include <media/stagefright/Utils.h> 46 47 #include <OMX_IVCommon.h> 48 49 namespace android { 50 51 struct WifiDisplaySource::PlaybackSession::Track : public AHandler { 52 enum { 53 kWhatStopped, 54 }; 55 56 Track(const sp<AMessage> ¬ify, 57 const sp<ALooper> &pullLooper, 58 const sp<ALooper> &codecLooper, 59 const sp<MediaPuller> &mediaPuller, 60 const sp<Converter> &converter); 61 62 Track(const sp<AMessage> ¬ify, const sp<AMessage> &format); 63 64 void setRepeaterSource(const sp<RepeaterSource> &source); 65 66 sp<AMessage> getFormat(); 67 bool isAudio() const; 68 69 const sp<Converter> &converter() const; 70 const sp<RepeaterSource> &repeaterSource() const; 71 72 ssize_t mediaSenderTrackIndex() const; 73 void setMediaSenderTrackIndex(size_t index); 74 75 status_t start(); 76 void stopAsync(); 77 78 void pause(); 79 void resume(); 80 81 void queueAccessUnit(const sp<ABuffer> &accessUnit); 82 sp<ABuffer> dequeueAccessUnit(); 83 84 bool hasOutputBuffer(int64_t *timeUs) const; 85 void queueOutputBuffer(const sp<ABuffer> &accessUnit); 86 sp<ABuffer> dequeueOutputBuffer(); 87 88 #if SUSPEND_VIDEO_IF_IDLE 89 bool isSuspended() const; 90 #endif 91 92 size_t countQueuedOutputBuffers() const { 93 return mQueuedOutputBuffers.size(); 94 } 95 96 void requestIDRFrame(); 97 98 protected: 99 virtual void onMessageReceived(const sp<AMessage> &msg); 100 virtual ~Track(); 101 102 private: 103 enum { 104 kWhatMediaPullerStopped, 105 }; 106 107 sp<AMessage> mNotify; 108 sp<ALooper> mPullLooper; 109 sp<ALooper> mCodecLooper; 110 sp<MediaPuller> mMediaPuller; 111 sp<Converter> mConverter; 112 sp<AMessage> mFormat; 113 bool mStarted; 114 ssize_t mMediaSenderTrackIndex; 115 bool mIsAudio; 116 List<sp<ABuffer> > mQueuedAccessUnits; 117 sp<RepeaterSource> mRepeaterSource; 118 List<sp<ABuffer> > mQueuedOutputBuffers; 119 int64_t mLastOutputBufferQueuedTimeUs; 120 121 static bool IsAudioFormat(const sp<AMessage> &format); 122 123 DISALLOW_EVIL_CONSTRUCTORS(Track); 124 }; 125 126 WifiDisplaySource::PlaybackSession::Track::Track( 127 const sp<AMessage> ¬ify, 128 const sp<ALooper> &pullLooper, 129 const sp<ALooper> &codecLooper, 130 const sp<MediaPuller> &mediaPuller, 131 const sp<Converter> &converter) 132 : mNotify(notify), 133 mPullLooper(pullLooper), 134 mCodecLooper(codecLooper), 135 mMediaPuller(mediaPuller), 136 mConverter(converter), 137 mStarted(false), 138 mIsAudio(IsAudioFormat(mConverter->getOutputFormat())), 139 mLastOutputBufferQueuedTimeUs(-1ll) { 140 } 141 142 WifiDisplaySource::PlaybackSession::Track::Track( 143 const sp<AMessage> ¬ify, const sp<AMessage> &format) 144 : mNotify(notify), 145 mFormat(format), 146 mStarted(false), 147 mIsAudio(IsAudioFormat(format)), 148 mLastOutputBufferQueuedTimeUs(-1ll) { 149 } 150 151 WifiDisplaySource::PlaybackSession::Track::~Track() { 152 CHECK(!mStarted); 153 } 154 155 // static 156 bool WifiDisplaySource::PlaybackSession::Track::IsAudioFormat( 157 const sp<AMessage> &format) { 158 AString mime; 159 CHECK(format->findString("mime", &mime)); 160 161 return !strncasecmp(mime.c_str(), "audio/", 6); 162 } 163 164 sp<AMessage> WifiDisplaySource::PlaybackSession::Track::getFormat() { 165 return mFormat != NULL ? mFormat : mConverter->getOutputFormat(); 166 } 167 168 bool WifiDisplaySource::PlaybackSession::Track::isAudio() const { 169 return mIsAudio; 170 } 171 172 const sp<Converter> &WifiDisplaySource::PlaybackSession::Track::converter() const { 173 return mConverter; 174 } 175 176 const sp<RepeaterSource> & 177 WifiDisplaySource::PlaybackSession::Track::repeaterSource() const { 178 return mRepeaterSource; 179 } 180 181 ssize_t WifiDisplaySource::PlaybackSession::Track::mediaSenderTrackIndex() const { 182 CHECK_GE(mMediaSenderTrackIndex, 0); 183 return mMediaSenderTrackIndex; 184 } 185 186 void WifiDisplaySource::PlaybackSession::Track::setMediaSenderTrackIndex( 187 size_t index) { 188 mMediaSenderTrackIndex = index; 189 } 190 191 status_t WifiDisplaySource::PlaybackSession::Track::start() { 192 ALOGV("Track::start isAudio=%d", mIsAudio); 193 194 CHECK(!mStarted); 195 196 status_t err = OK; 197 198 if (mMediaPuller != NULL) { 199 err = mMediaPuller->start(); 200 } 201 202 if (err == OK) { 203 mStarted = true; 204 } 205 206 return err; 207 } 208 209 void WifiDisplaySource::PlaybackSession::Track::stopAsync() { 210 ALOGV("Track::stopAsync isAudio=%d", mIsAudio); 211 212 if (mConverter != NULL) { 213 mConverter->shutdownAsync(); 214 } 215 216 sp<AMessage> msg = new AMessage(kWhatMediaPullerStopped, id()); 217 218 if (mStarted && mMediaPuller != NULL) { 219 if (mRepeaterSource != NULL) { 220 // Let's unblock MediaPuller's MediaSource::read(). 221 mRepeaterSource->wakeUp(); 222 } 223 224 mMediaPuller->stopAsync(msg); 225 } else { 226 mStarted = false; 227 msg->post(); 228 } 229 } 230 231 void WifiDisplaySource::PlaybackSession::Track::pause() { 232 mMediaPuller->pause(); 233 } 234 235 void WifiDisplaySource::PlaybackSession::Track::resume() { 236 mMediaPuller->resume(); 237 } 238 239 void WifiDisplaySource::PlaybackSession::Track::onMessageReceived( 240 const sp<AMessage> &msg) { 241 switch (msg->what()) { 242 case kWhatMediaPullerStopped: 243 { 244 mConverter.clear(); 245 246 mStarted = false; 247 248 sp<AMessage> notify = mNotify->dup(); 249 notify->setInt32("what", kWhatStopped); 250 notify->post(); 251 252 ALOGI("kWhatStopped %s posted", mIsAudio ? "audio" : "video"); 253 break; 254 } 255 256 default: 257 TRESPASS(); 258 } 259 } 260 261 void WifiDisplaySource::PlaybackSession::Track::queueAccessUnit( 262 const sp<ABuffer> &accessUnit) { 263 mQueuedAccessUnits.push_back(accessUnit); 264 } 265 266 sp<ABuffer> WifiDisplaySource::PlaybackSession::Track::dequeueAccessUnit() { 267 if (mQueuedAccessUnits.empty()) { 268 return NULL; 269 } 270 271 sp<ABuffer> accessUnit = *mQueuedAccessUnits.begin(); 272 CHECK(accessUnit != NULL); 273 274 mQueuedAccessUnits.erase(mQueuedAccessUnits.begin()); 275 276 return accessUnit; 277 } 278 279 void WifiDisplaySource::PlaybackSession::Track::setRepeaterSource( 280 const sp<RepeaterSource> &source) { 281 mRepeaterSource = source; 282 } 283 284 void WifiDisplaySource::PlaybackSession::Track::requestIDRFrame() { 285 if (mIsAudio) { 286 return; 287 } 288 289 if (mRepeaterSource != NULL) { 290 mRepeaterSource->wakeUp(); 291 } 292 293 mConverter->requestIDRFrame(); 294 } 295 296 bool WifiDisplaySource::PlaybackSession::Track::hasOutputBuffer( 297 int64_t *timeUs) const { 298 *timeUs = 0ll; 299 300 if (mQueuedOutputBuffers.empty()) { 301 return false; 302 } 303 304 const sp<ABuffer> &outputBuffer = *mQueuedOutputBuffers.begin(); 305 306 CHECK(outputBuffer->meta()->findInt64("timeUs", timeUs)); 307 308 return true; 309 } 310 311 void WifiDisplaySource::PlaybackSession::Track::queueOutputBuffer( 312 const sp<ABuffer> &accessUnit) { 313 mQueuedOutputBuffers.push_back(accessUnit); 314 mLastOutputBufferQueuedTimeUs = ALooper::GetNowUs(); 315 } 316 317 sp<ABuffer> WifiDisplaySource::PlaybackSession::Track::dequeueOutputBuffer() { 318 CHECK(!mQueuedOutputBuffers.empty()); 319 320 sp<ABuffer> outputBuffer = *mQueuedOutputBuffers.begin(); 321 mQueuedOutputBuffers.erase(mQueuedOutputBuffers.begin()); 322 323 return outputBuffer; 324 } 325 326 #if SUSPEND_VIDEO_IF_IDLE 327 bool WifiDisplaySource::PlaybackSession::Track::isSuspended() const { 328 if (!mQueuedOutputBuffers.empty()) { 329 return false; 330 } 331 332 if (mLastOutputBufferQueuedTimeUs < 0ll) { 333 // We've never seen an output buffer queued, but tracks start 334 // out live, not suspended. 335 return false; 336 } 337 338 // If we've not seen new output data for 60ms or more, we consider 339 // this track suspended for the time being. 340 return (ALooper::GetNowUs() - mLastOutputBufferQueuedTimeUs) > 60000ll; 341 } 342 #endif 343 344 //////////////////////////////////////////////////////////////////////////////// 345 346 WifiDisplaySource::PlaybackSession::PlaybackSession( 347 const sp<ANetworkSession> &netSession, 348 const sp<AMessage> ¬ify, 349 const in_addr &interfaceAddr, 350 const sp<IHDCP> &hdcp, 351 const char *path) 352 : mNetSession(netSession), 353 mNotify(notify), 354 mInterfaceAddr(interfaceAddr), 355 mHDCP(hdcp), 356 mLocalRTPPort(-1), 357 mWeAreDead(false), 358 mPaused(false), 359 mLastLifesignUs(), 360 mVideoTrackIndex(-1), 361 mPrevTimeUs(-1ll), 362 mPullExtractorPending(false), 363 mPullExtractorGeneration(0), 364 mFirstSampleTimeRealUs(-1ll), 365 mFirstSampleTimeUs(-1ll) { 366 if (path != NULL) { 367 mMediaPath.setTo(path); 368 } 369 } 370 371 status_t WifiDisplaySource::PlaybackSession::init( 372 const char *clientIP, 373 int32_t clientRtp, 374 RTPSender::TransportMode rtpMode, 375 int32_t clientRtcp, 376 RTPSender::TransportMode rtcpMode, 377 bool enableAudio, 378 bool usePCMAudio, 379 bool enableVideo, 380 VideoFormats::ResolutionType videoResolutionType, 381 size_t videoResolutionIndex, 382 VideoFormats::ProfileType videoProfileType, 383 VideoFormats::LevelType videoLevelType) { 384 sp<AMessage> notify = new AMessage(kWhatMediaSenderNotify, id()); 385 mMediaSender = new MediaSender(mNetSession, notify); 386 looper()->registerHandler(mMediaSender); 387 388 mMediaSender->setHDCP(mHDCP); 389 390 status_t err = setupPacketizer( 391 enableAudio, 392 usePCMAudio, 393 enableVideo, 394 videoResolutionType, 395 videoResolutionIndex, 396 videoProfileType, 397 videoLevelType); 398 399 if (err == OK) { 400 err = mMediaSender->initAsync( 401 -1 /* trackIndex */, 402 clientIP, 403 clientRtp, 404 rtpMode, 405 clientRtcp, 406 rtcpMode, 407 &mLocalRTPPort); 408 } 409 410 if (err != OK) { 411 mLocalRTPPort = -1; 412 413 looper()->unregisterHandler(mMediaSender->id()); 414 mMediaSender.clear(); 415 416 return err; 417 } 418 419 updateLiveness(); 420 421 return OK; 422 } 423 424 WifiDisplaySource::PlaybackSession::~PlaybackSession() { 425 } 426 427 int32_t WifiDisplaySource::PlaybackSession::getRTPPort() const { 428 return mLocalRTPPort; 429 } 430 431 int64_t WifiDisplaySource::PlaybackSession::getLastLifesignUs() const { 432 return mLastLifesignUs; 433 } 434 435 void WifiDisplaySource::PlaybackSession::updateLiveness() { 436 mLastLifesignUs = ALooper::GetNowUs(); 437 } 438 439 status_t WifiDisplaySource::PlaybackSession::play() { 440 updateLiveness(); 441 442 (new AMessage(kWhatResume, id()))->post(); 443 444 return OK; 445 } 446 447 status_t WifiDisplaySource::PlaybackSession::onMediaSenderInitialized() { 448 for (size_t i = 0; i < mTracks.size(); ++i) { 449 CHECK_EQ((status_t)OK, mTracks.editValueAt(i)->start()); 450 } 451 452 sp<AMessage> notify = mNotify->dup(); 453 notify->setInt32("what", kWhatSessionEstablished); 454 notify->post(); 455 456 return OK; 457 } 458 459 status_t WifiDisplaySource::PlaybackSession::pause() { 460 updateLiveness(); 461 462 (new AMessage(kWhatPause, id()))->post(); 463 464 return OK; 465 } 466 467 void WifiDisplaySource::PlaybackSession::destroyAsync() { 468 ALOGI("destroyAsync"); 469 470 for (size_t i = 0; i < mTracks.size(); ++i) { 471 mTracks.valueAt(i)->stopAsync(); 472 } 473 } 474 475 void WifiDisplaySource::PlaybackSession::onMessageReceived( 476 const sp<AMessage> &msg) { 477 switch (msg->what()) { 478 case kWhatConverterNotify: 479 { 480 if (mWeAreDead) { 481 ALOGV("dropping msg '%s' because we're dead", 482 msg->debugString().c_str()); 483 484 break; 485 } 486 487 int32_t what; 488 CHECK(msg->findInt32("what", &what)); 489 490 size_t trackIndex; 491 CHECK(msg->findSize("trackIndex", &trackIndex)); 492 493 if (what == Converter::kWhatAccessUnit) { 494 sp<ABuffer> accessUnit; 495 CHECK(msg->findBuffer("accessUnit", &accessUnit)); 496 497 const sp<Track> &track = mTracks.valueFor(trackIndex); 498 499 status_t err = mMediaSender->queueAccessUnit( 500 track->mediaSenderTrackIndex(), 501 accessUnit); 502 503 if (err != OK) { 504 notifySessionDead(); 505 } 506 break; 507 } else if (what == Converter::kWhatEOS) { 508 CHECK_EQ(what, Converter::kWhatEOS); 509 510 ALOGI("output EOS on track %d", trackIndex); 511 512 ssize_t index = mTracks.indexOfKey(trackIndex); 513 CHECK_GE(index, 0); 514 515 const sp<Converter> &converter = 516 mTracks.valueAt(index)->converter(); 517 looper()->unregisterHandler(converter->id()); 518 519 mTracks.removeItemsAt(index); 520 521 if (mTracks.isEmpty()) { 522 ALOGI("Reached EOS"); 523 } 524 } else if (what != Converter::kWhatShutdownCompleted) { 525 CHECK_EQ(what, Converter::kWhatError); 526 527 status_t err; 528 CHECK(msg->findInt32("err", &err)); 529 530 ALOGE("converter signaled error %d", err); 531 532 notifySessionDead(); 533 } 534 break; 535 } 536 537 case kWhatMediaSenderNotify: 538 { 539 int32_t what; 540 CHECK(msg->findInt32("what", &what)); 541 542 if (what == MediaSender::kWhatInitDone) { 543 status_t err; 544 CHECK(msg->findInt32("err", &err)); 545 546 if (err == OK) { 547 onMediaSenderInitialized(); 548 } else { 549 notifySessionDead(); 550 } 551 } else if (what == MediaSender::kWhatError) { 552 notifySessionDead(); 553 } else if (what == MediaSender::kWhatNetworkStall) { 554 size_t numBytesQueued; 555 CHECK(msg->findSize("numBytesQueued", &numBytesQueued)); 556 557 if (mVideoTrackIndex >= 0) { 558 const sp<Track> &videoTrack = 559 mTracks.valueFor(mVideoTrackIndex); 560 561 sp<Converter> converter = videoTrack->converter(); 562 if (converter != NULL) { 563 converter->dropAFrame(); 564 } 565 } 566 } else if (what == MediaSender::kWhatInformSender) { 567 onSinkFeedback(msg); 568 } else { 569 TRESPASS(); 570 } 571 break; 572 } 573 574 case kWhatTrackNotify: 575 { 576 int32_t what; 577 CHECK(msg->findInt32("what", &what)); 578 579 size_t trackIndex; 580 CHECK(msg->findSize("trackIndex", &trackIndex)); 581 582 if (what == Track::kWhatStopped) { 583 ALOGI("Track %d stopped", trackIndex); 584 585 sp<Track> track = mTracks.valueFor(trackIndex); 586 looper()->unregisterHandler(track->id()); 587 mTracks.removeItem(trackIndex); 588 track.clear(); 589 590 if (!mTracks.isEmpty()) { 591 ALOGI("not all tracks are stopped yet"); 592 break; 593 } 594 595 looper()->unregisterHandler(mMediaSender->id()); 596 mMediaSender.clear(); 597 598 sp<AMessage> notify = mNotify->dup(); 599 notify->setInt32("what", kWhatSessionDestroyed); 600 notify->post(); 601 } 602 break; 603 } 604 605 case kWhatPause: 606 { 607 if (mExtractor != NULL) { 608 ++mPullExtractorGeneration; 609 mFirstSampleTimeRealUs = -1ll; 610 mFirstSampleTimeUs = -1ll; 611 } 612 613 if (mPaused) { 614 break; 615 } 616 617 for (size_t i = 0; i < mTracks.size(); ++i) { 618 mTracks.editValueAt(i)->pause(); 619 } 620 621 mPaused = true; 622 break; 623 } 624 625 case kWhatResume: 626 { 627 if (mExtractor != NULL) { 628 schedulePullExtractor(); 629 } 630 631 if (!mPaused) { 632 break; 633 } 634 635 for (size_t i = 0; i < mTracks.size(); ++i) { 636 mTracks.editValueAt(i)->resume(); 637 } 638 639 mPaused = false; 640 break; 641 } 642 643 case kWhatPullExtractorSample: 644 { 645 int32_t generation; 646 CHECK(msg->findInt32("generation", &generation)); 647 648 if (generation != mPullExtractorGeneration) { 649 break; 650 } 651 652 mPullExtractorPending = false; 653 654 onPullExtractor(); 655 break; 656 } 657 658 default: 659 TRESPASS(); 660 } 661 } 662 663 void WifiDisplaySource::PlaybackSession::onSinkFeedback(const sp<AMessage> &msg) { 664 int64_t avgLatencyUs; 665 CHECK(msg->findInt64("avgLatencyUs", &avgLatencyUs)); 666 667 int64_t maxLatencyUs; 668 CHECK(msg->findInt64("maxLatencyUs", &maxLatencyUs)); 669 670 ALOGI("sink reports avg. latency of %lld ms (max %lld ms)", 671 avgLatencyUs / 1000ll, 672 maxLatencyUs / 1000ll); 673 674 if (mVideoTrackIndex >= 0) { 675 const sp<Track> &videoTrack = mTracks.valueFor(mVideoTrackIndex); 676 sp<Converter> converter = videoTrack->converter(); 677 678 if (converter != NULL) { 679 int32_t videoBitrate = 680 Converter::GetInt32Property("media.wfd.video-bitrate", -1); 681 682 char val[PROPERTY_VALUE_MAX]; 683 if (videoBitrate < 0 684 && property_get("media.wfd.video-bitrate", val, NULL) 685 && !strcasecmp("adaptive", val)) { 686 videoBitrate = converter->getVideoBitrate(); 687 688 if (avgLatencyUs > 300000ll) { 689 videoBitrate *= 0.6; 690 } else if (avgLatencyUs < 100000ll) { 691 videoBitrate *= 1.1; 692 } 693 } 694 695 if (videoBitrate > 0) { 696 if (videoBitrate < 500000) { 697 videoBitrate = 500000; 698 } else if (videoBitrate > 10000000) { 699 videoBitrate = 10000000; 700 } 701 702 if (videoBitrate != converter->getVideoBitrate()) { 703 ALOGI("setting video bitrate to %d bps", videoBitrate); 704 705 converter->setVideoBitrate(videoBitrate); 706 } 707 } 708 } 709 710 sp<RepeaterSource> repeaterSource = videoTrack->repeaterSource(); 711 if (repeaterSource != NULL) { 712 double rateHz = 713 Converter::GetInt32Property( 714 "media.wfd.video-framerate", -1); 715 716 char val[PROPERTY_VALUE_MAX]; 717 if (rateHz < 0.0 718 && property_get("media.wfd.video-framerate", val, NULL) 719 && !strcasecmp("adaptive", val)) { 720 rateHz = repeaterSource->getFrameRate(); 721 722 if (avgLatencyUs > 300000ll) { 723 rateHz *= 0.9; 724 } else if (avgLatencyUs < 200000ll) { 725 rateHz *= 1.1; 726 } 727 } 728 729 if (rateHz > 0) { 730 if (rateHz < 5.0) { 731 rateHz = 5.0; 732 } else if (rateHz > 30.0) { 733 rateHz = 30.0; 734 } 735 736 if (rateHz != repeaterSource->getFrameRate()) { 737 ALOGI("setting frame rate to %.2f Hz", rateHz); 738 739 repeaterSource->setFrameRate(rateHz); 740 } 741 } 742 } 743 } 744 } 745 746 status_t WifiDisplaySource::PlaybackSession::setupMediaPacketizer( 747 bool enableAudio, bool enableVideo) { 748 DataSource::RegisterDefaultSniffers(); 749 750 mExtractor = new NuMediaExtractor; 751 752 status_t err = mExtractor->setDataSource(mMediaPath.c_str()); 753 754 if (err != OK) { 755 return err; 756 } 757 758 size_t n = mExtractor->countTracks(); 759 bool haveAudio = false; 760 bool haveVideo = false; 761 for (size_t i = 0; i < n; ++i) { 762 sp<AMessage> format; 763 err = mExtractor->getTrackFormat(i, &format); 764 765 if (err != OK) { 766 continue; 767 } 768 769 AString mime; 770 CHECK(format->findString("mime", &mime)); 771 772 bool isAudio = !strncasecmp(mime.c_str(), "audio/", 6); 773 bool isVideo = !strncasecmp(mime.c_str(), "video/", 6); 774 775 if (isAudio && enableAudio && !haveAudio) { 776 haveAudio = true; 777 } else if (isVideo && enableVideo && !haveVideo) { 778 haveVideo = true; 779 } else { 780 continue; 781 } 782 783 err = mExtractor->selectTrack(i); 784 785 size_t trackIndex = mTracks.size(); 786 787 sp<AMessage> notify = new AMessage(kWhatTrackNotify, id()); 788 notify->setSize("trackIndex", trackIndex); 789 790 sp<Track> track = new Track(notify, format); 791 looper()->registerHandler(track); 792 793 mTracks.add(trackIndex, track); 794 795 mExtractorTrackToInternalTrack.add(i, trackIndex); 796 797 if (isVideo) { 798 mVideoTrackIndex = trackIndex; 799 } 800 801 uint32_t flags = MediaSender::FLAG_MANUALLY_PREPEND_SPS_PPS; 802 803 ssize_t mediaSenderTrackIndex = 804 mMediaSender->addTrack(format, flags); 805 CHECK_GE(mediaSenderTrackIndex, 0); 806 807 track->setMediaSenderTrackIndex(mediaSenderTrackIndex); 808 809 if ((haveAudio || !enableAudio) && (haveVideo || !enableVideo)) { 810 break; 811 } 812 } 813 814 return OK; 815 } 816 817 void WifiDisplaySource::PlaybackSession::schedulePullExtractor() { 818 if (mPullExtractorPending) { 819 return; 820 } 821 822 int64_t sampleTimeUs; 823 status_t err = mExtractor->getSampleTime(&sampleTimeUs); 824 825 int64_t nowUs = ALooper::GetNowUs(); 826 827 if (mFirstSampleTimeRealUs < 0ll) { 828 mFirstSampleTimeRealUs = nowUs; 829 mFirstSampleTimeUs = sampleTimeUs; 830 } 831 832 int64_t whenUs = sampleTimeUs - mFirstSampleTimeUs + mFirstSampleTimeRealUs; 833 834 sp<AMessage> msg = new AMessage(kWhatPullExtractorSample, id()); 835 msg->setInt32("generation", mPullExtractorGeneration); 836 msg->post(whenUs - nowUs); 837 838 mPullExtractorPending = true; 839 } 840 841 void WifiDisplaySource::PlaybackSession::onPullExtractor() { 842 sp<ABuffer> accessUnit = new ABuffer(1024 * 1024); 843 status_t err = mExtractor->readSampleData(accessUnit); 844 if (err != OK) { 845 // EOS. 846 return; 847 } 848 849 int64_t timeUs; 850 CHECK_EQ((status_t)OK, mExtractor->getSampleTime(&timeUs)); 851 852 accessUnit->meta()->setInt64( 853 "timeUs", mFirstSampleTimeRealUs + timeUs - mFirstSampleTimeUs); 854 855 size_t trackIndex; 856 CHECK_EQ((status_t)OK, mExtractor->getSampleTrackIndex(&trackIndex)); 857 858 sp<AMessage> msg = new AMessage(kWhatConverterNotify, id()); 859 860 msg->setSize( 861 "trackIndex", mExtractorTrackToInternalTrack.valueFor(trackIndex)); 862 863 msg->setInt32("what", Converter::kWhatAccessUnit); 864 msg->setBuffer("accessUnit", accessUnit); 865 msg->post(); 866 867 mExtractor->advance(); 868 869 schedulePullExtractor(); 870 } 871 872 status_t WifiDisplaySource::PlaybackSession::setupPacketizer( 873 bool enableAudio, 874 bool usePCMAudio, 875 bool enableVideo, 876 VideoFormats::ResolutionType videoResolutionType, 877 size_t videoResolutionIndex, 878 VideoFormats::ProfileType videoProfileType, 879 VideoFormats::LevelType videoLevelType) { 880 CHECK(enableAudio || enableVideo); 881 882 if (!mMediaPath.empty()) { 883 return setupMediaPacketizer(enableAudio, enableVideo); 884 } 885 886 if (enableVideo) { 887 status_t err = addVideoSource( 888 videoResolutionType, videoResolutionIndex, videoProfileType, 889 videoLevelType); 890 891 if (err != OK) { 892 return err; 893 } 894 } 895 896 if (!enableAudio) { 897 return OK; 898 } 899 900 return addAudioSource(usePCMAudio); 901 } 902 903 status_t WifiDisplaySource::PlaybackSession::addSource( 904 bool isVideo, const sp<MediaSource> &source, bool isRepeaterSource, 905 bool usePCMAudio, unsigned profileIdc, unsigned levelIdc, 906 unsigned constraintSet, size_t *numInputBuffers) { 907 CHECK(!usePCMAudio || !isVideo); 908 CHECK(!isRepeaterSource || isVideo); 909 CHECK(!profileIdc || isVideo); 910 CHECK(!levelIdc || isVideo); 911 CHECK(!constraintSet || isVideo); 912 913 sp<ALooper> pullLooper = new ALooper; 914 pullLooper->setName("pull_looper"); 915 916 pullLooper->start( 917 false /* runOnCallingThread */, 918 false /* canCallJava */, 919 PRIORITY_AUDIO); 920 921 sp<ALooper> codecLooper = new ALooper; 922 codecLooper->setName("codec_looper"); 923 924 codecLooper->start( 925 false /* runOnCallingThread */, 926 false /* canCallJava */, 927 PRIORITY_AUDIO); 928 929 size_t trackIndex; 930 931 sp<AMessage> notify; 932 933 trackIndex = mTracks.size(); 934 935 sp<AMessage> format; 936 status_t err = convertMetaDataToMessage(source->getFormat(), &format); 937 CHECK_EQ(err, (status_t)OK); 938 939 if (isVideo) { 940 format->setString("mime", MEDIA_MIMETYPE_VIDEO_AVC); 941 format->setInt32("store-metadata-in-buffers", true); 942 format->setInt32("store-metadata-in-buffers-output", (mHDCP != NULL) 943 && (mHDCP->getCaps() & HDCPModule::HDCP_CAPS_ENCRYPT_NATIVE)); 944 format->setInt32( 945 "color-format", OMX_COLOR_FormatAndroidOpaque); 946 format->setInt32("profile-idc", profileIdc); 947 format->setInt32("level-idc", levelIdc); 948 format->setInt32("constraint-set", constraintSet); 949 } else { 950 format->setString( 951 "mime", 952 usePCMAudio 953 ? MEDIA_MIMETYPE_AUDIO_RAW : MEDIA_MIMETYPE_AUDIO_AAC); 954 } 955 956 notify = new AMessage(kWhatConverterNotify, id()); 957 notify->setSize("trackIndex", trackIndex); 958 959 sp<Converter> converter = new Converter(notify, codecLooper, format); 960 961 looper()->registerHandler(converter); 962 963 err = converter->init(); 964 if (err != OK) { 965 ALOGE("%s converter returned err %d", isVideo ? "video" : "audio", err); 966 967 looper()->unregisterHandler(converter->id()); 968 return err; 969 } 970 971 notify = new AMessage(Converter::kWhatMediaPullerNotify, converter->id()); 972 notify->setSize("trackIndex", trackIndex); 973 974 sp<MediaPuller> puller = new MediaPuller(source, notify); 975 pullLooper->registerHandler(puller); 976 977 if (numInputBuffers != NULL) { 978 *numInputBuffers = converter->getInputBufferCount(); 979 } 980 981 notify = new AMessage(kWhatTrackNotify, id()); 982 notify->setSize("trackIndex", trackIndex); 983 984 sp<Track> track = new Track( 985 notify, pullLooper, codecLooper, puller, converter); 986 987 if (isRepeaterSource) { 988 track->setRepeaterSource(static_cast<RepeaterSource *>(source.get())); 989 } 990 991 looper()->registerHandler(track); 992 993 mTracks.add(trackIndex, track); 994 995 if (isVideo) { 996 mVideoTrackIndex = trackIndex; 997 } 998 999 uint32_t flags = 0; 1000 if (converter->needToManuallyPrependSPSPPS()) { 1001 flags |= MediaSender::FLAG_MANUALLY_PREPEND_SPS_PPS; 1002 } 1003 1004 ssize_t mediaSenderTrackIndex = 1005 mMediaSender->addTrack(converter->getOutputFormat(), flags); 1006 CHECK_GE(mediaSenderTrackIndex, 0); 1007 1008 track->setMediaSenderTrackIndex(mediaSenderTrackIndex); 1009 1010 return OK; 1011 } 1012 1013 status_t WifiDisplaySource::PlaybackSession::addVideoSource( 1014 VideoFormats::ResolutionType videoResolutionType, 1015 size_t videoResolutionIndex, 1016 VideoFormats::ProfileType videoProfileType, 1017 VideoFormats::LevelType videoLevelType) { 1018 size_t width, height, framesPerSecond; 1019 bool interlaced; 1020 CHECK(VideoFormats::GetConfiguration( 1021 videoResolutionType, 1022 videoResolutionIndex, 1023 &width, 1024 &height, 1025 &framesPerSecond, 1026 &interlaced)); 1027 1028 unsigned profileIdc, levelIdc, constraintSet; 1029 CHECK(VideoFormats::GetProfileLevel( 1030 videoProfileType, 1031 videoLevelType, 1032 &profileIdc, 1033 &levelIdc, 1034 &constraintSet)); 1035 1036 sp<SurfaceMediaSource> source = new SurfaceMediaSource(width, height); 1037 1038 source->setUseAbsoluteTimestamps(); 1039 1040 sp<RepeaterSource> videoSource = 1041 new RepeaterSource(source, framesPerSecond); 1042 1043 size_t numInputBuffers; 1044 status_t err = addSource( 1045 true /* isVideo */, videoSource, true /* isRepeaterSource */, 1046 false /* usePCMAudio */, profileIdc, levelIdc, constraintSet, 1047 &numInputBuffers); 1048 1049 if (err != OK) { 1050 return err; 1051 } 1052 1053 err = source->setMaxAcquiredBufferCount(numInputBuffers); 1054 CHECK_EQ(err, (status_t)OK); 1055 1056 mBufferQueue = source->getBufferQueue(); 1057 1058 return OK; 1059 } 1060 1061 status_t WifiDisplaySource::PlaybackSession::addAudioSource(bool usePCMAudio) { 1062 sp<AudioSource> audioSource = new AudioSource( 1063 AUDIO_SOURCE_REMOTE_SUBMIX, 1064 48000 /* sampleRate */, 1065 2 /* channelCount */); 1066 1067 if (audioSource->initCheck() == OK) { 1068 return addSource( 1069 false /* isVideo */, audioSource, false /* isRepeaterSource */, 1070 usePCMAudio, 0 /* profileIdc */, 0 /* levelIdc */, 1071 0 /* constraintSet */, NULL /* numInputBuffers */); 1072 } 1073 1074 ALOGW("Unable to instantiate audio source"); 1075 1076 return OK; 1077 } 1078 1079 sp<IGraphicBufferProducer> WifiDisplaySource::PlaybackSession::getSurfaceTexture() { 1080 return mBufferQueue; 1081 } 1082 1083 void WifiDisplaySource::PlaybackSession::requestIDRFrame() { 1084 for (size_t i = 0; i < mTracks.size(); ++i) { 1085 const sp<Track> &track = mTracks.valueAt(i); 1086 1087 track->requestIDRFrame(); 1088 } 1089 } 1090 1091 void WifiDisplaySource::PlaybackSession::notifySessionDead() { 1092 // Inform WifiDisplaySource of our premature death (wish). 1093 sp<AMessage> notify = mNotify->dup(); 1094 notify->setInt32("what", kWhatSessionDead); 1095 notify->post(); 1096 1097 mWeAreDead = true; 1098 } 1099 1100 } // namespace android 1101 1102