Home | History | Annotate | Download | only in source 
      1 /*
      2  * Copyright 2012, The Android Open Source Project
      3  *
      4  * Licensed under the Apache License, Version 2.0 (the "License");
      5  * you may not use this file except in compliance with the License.
      6  * You may obtain a copy of the License at
      7  *
      8  *     http://www.apache.org/licenses/LICENSE-2.0
      9  *
     10  * Unless required by applicable law or agreed to in writing, software
     11  * distributed under the License is distributed on an "AS IS" BASIS,
     12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
     13  * See the License for the specific language governing permissions and
     14  * limitations under the License.
     15  */
     16 
     17 //#define LOG_NDEBUG 0
     18 #define LOG_TAG "PlaybackSession"
     19 #include <utils/Log.h>
     20 
     21 #include "PlaybackSession.h"
     22 
     23 #include "Converter.h"
     24 #include "MediaPuller.h"
     25 #include "RepeaterSource.h"
     26 #include "include/avc_utils.h"
     27 #include "WifiDisplaySource.h"
     28 
     29 #include <binder/IServiceManager.h>
     30 #include <cutils/properties.h>
     31 #include <media/IHDCP.h>
     32 #include <media/IMediaHTTPService.h>
     33 #include <media/stagefright/foundation/ABitReader.h>
     34 #include <media/stagefright/foundation/ABuffer.h>
     35 #include <media/stagefright/foundation/ADebug.h>
     36 #include <media/stagefright/foundation/AMessage.h>
     37 #include <media/stagefright/foundation/hexdump.h>
     38 #include <media/stagefright/AudioSource.h>
     39 #include <media/stagefright/DataSource.h>
     40 #include <media/stagefright/MediaDefs.h>
     41 #include <media/stagefright/MediaErrors.h>
     42 #include <media/stagefright/MediaSource.h>
     43 #include <media/stagefright/MetaData.h>
     44 #include <media/stagefright/NuMediaExtractor.h>
     45 #include <media/stagefright/SurfaceMediaSource.h>
     46 #include <media/stagefright/Utils.h>
     47 
     48 #include <OMX_IVCommon.h>
     49 
     50 namespace android {
     51 
     52 struct WifiDisplaySource::PlaybackSession::Track : public AHandler {
     53     enum {
     54         kWhatStopped,
     55     };
     56 
     57     Track(const sp<AMessage> &notify,
     58           const sp<ALooper> &pullLooper,
     59           const sp<ALooper> &codecLooper,
     60           const sp<MediaPuller> &mediaPuller,
     61           const sp<Converter> &converter);
     62 
     63     Track(const sp<AMessage> &notify, const sp<AMessage> &format);
     64 
     65     void setRepeaterSource(const sp<RepeaterSource> &source);
     66 
     67     sp<AMessage> getFormat();
     68     bool isAudio() const;
     69 
     70     const sp<Converter> &converter() const;
     71     const sp<RepeaterSource> &repeaterSource() const;
     72 
     73     ssize_t mediaSenderTrackIndex() const;
     74     void setMediaSenderTrackIndex(size_t index);
     75 
     76     status_t start();
     77     void stopAsync();
     78 
     79     void pause();
     80     void resume();
     81 
     82     void queueAccessUnit(const sp<ABuffer> &accessUnit);
     83     sp<ABuffer> dequeueAccessUnit();
     84 
     85     bool hasOutputBuffer(int64_t *timeUs) const;
     86     void queueOutputBuffer(const sp<ABuffer> &accessUnit);
     87     sp<ABuffer> dequeueOutputBuffer();
     88 
     89 #if SUSPEND_VIDEO_IF_IDLE
     90     bool isSuspended() const;
     91 #endif
     92 
     93     size_t countQueuedOutputBuffers() const {
     94         return mQueuedOutputBuffers.size();
     95     }
     96 
     97     void requestIDRFrame();
     98 
     99 protected:
    100     virtual void onMessageReceived(const sp<AMessage> &msg);
    101     virtual ~Track();
    102 
    103 private:
    104     enum {
    105         kWhatMediaPullerStopped,
    106     };
    107 
    108     sp<AMessage> mNotify;
    109     sp<ALooper> mPullLooper;
    110     sp<ALooper> mCodecLooper;
    111     sp<MediaPuller> mMediaPuller;
    112     sp<Converter> mConverter;
    113     sp<AMessage> mFormat;
    114     bool mStarted;
    115     ssize_t mMediaSenderTrackIndex;
    116     bool mIsAudio;
    117     List<sp<ABuffer> > mQueuedAccessUnits;
    118     sp<RepeaterSource> mRepeaterSource;
    119     List<sp<ABuffer> > mQueuedOutputBuffers;
    120     int64_t mLastOutputBufferQueuedTimeUs;
    121 
    122     static bool IsAudioFormat(const sp<AMessage> &format);
    123 
    124     DISALLOW_EVIL_CONSTRUCTORS(Track);
    125 };
    126 
    127 WifiDisplaySource::PlaybackSession::Track::Track(
    128         const sp<AMessage> &notify,
    129         const sp<ALooper> &pullLooper,
    130         const sp<ALooper> &codecLooper,
    131         const sp<MediaPuller> &mediaPuller,
    132         const sp<Converter> &converter)
    133     : mNotify(notify),
    134       mPullLooper(pullLooper),
    135       mCodecLooper(codecLooper),
    136       mMediaPuller(mediaPuller),
    137       mConverter(converter),
    138       mStarted(false),
    139       mIsAudio(IsAudioFormat(mConverter->getOutputFormat())),
    140       mLastOutputBufferQueuedTimeUs(-1ll) {
    141 }
    142 
    143 WifiDisplaySource::PlaybackSession::Track::Track(
    144         const sp<AMessage> &notify, const sp<AMessage> &format)
    145     : mNotify(notify),
    146       mFormat(format),
    147       mStarted(false),
    148       mIsAudio(IsAudioFormat(format)),
    149       mLastOutputBufferQueuedTimeUs(-1ll) {
    150 }
    151 
    152 WifiDisplaySource::PlaybackSession::Track::~Track() {
    153     CHECK(!mStarted);
    154 }
    155 
    156 // static
    157 bool WifiDisplaySource::PlaybackSession::Track::IsAudioFormat(
    158         const sp<AMessage> &format) {
    159     AString mime;
    160     CHECK(format->findString("mime", &mime));
    161 
    162     return !strncasecmp(mime.c_str(), "audio/", 6);
    163 }
    164 
    165 sp<AMessage> WifiDisplaySource::PlaybackSession::Track::getFormat() {
    166     return mFormat != NULL ? mFormat : mConverter->getOutputFormat();
    167 }
    168 
    169 bool WifiDisplaySource::PlaybackSession::Track::isAudio() const {
    170     return mIsAudio;
    171 }
    172 
    173 const sp<Converter> &WifiDisplaySource::PlaybackSession::Track::converter() const {
    174     return mConverter;
    175 }
    176 
    177 const sp<RepeaterSource> &
    178 WifiDisplaySource::PlaybackSession::Track::repeaterSource() const {
    179     return mRepeaterSource;
    180 }
    181 
    182 ssize_t WifiDisplaySource::PlaybackSession::Track::mediaSenderTrackIndex() const {
    183     CHECK_GE(mMediaSenderTrackIndex, 0);
    184     return mMediaSenderTrackIndex;
    185 }
    186 
    187 void WifiDisplaySource::PlaybackSession::Track::setMediaSenderTrackIndex(
    188         size_t index) {
    189     mMediaSenderTrackIndex = index;
    190 }
    191 
    192 status_t WifiDisplaySource::PlaybackSession::Track::start() {
    193     ALOGV("Track::start isAudio=%d", mIsAudio);
    194 
    195     CHECK(!mStarted);
    196 
    197     status_t err = OK;
    198 
    199     if (mMediaPuller != NULL) {
    200         err = mMediaPuller->start();
    201     }
    202 
    203     if (err == OK) {
    204         mStarted = true;
    205     }
    206 
    207     return err;
    208 }
    209 
    210 void WifiDisplaySource::PlaybackSession::Track::stopAsync() {
    211     ALOGV("Track::stopAsync isAudio=%d", mIsAudio);
    212 
    213     if (mConverter != NULL) {
    214         mConverter->shutdownAsync();
    215     }
    216 
    217     sp<AMessage> msg = new AMessage(kWhatMediaPullerStopped, id());
    218 
    219     if (mStarted && mMediaPuller != NULL) {
    220         if (mRepeaterSource != NULL) {
    221             // Let's unblock MediaPuller's MediaSource::read().
    222             mRepeaterSource->wakeUp();
    223         }
    224 
    225         mMediaPuller->stopAsync(msg);
    226     } else {
    227         mStarted = false;
    228         msg->post();
    229     }
    230 }
    231 
    232 void WifiDisplaySource::PlaybackSession::Track::pause() {
    233     mMediaPuller->pause();
    234 }
    235 
    236 void WifiDisplaySource::PlaybackSession::Track::resume() {
    237     mMediaPuller->resume();
    238 }
    239 
    240 void WifiDisplaySource::PlaybackSession::Track::onMessageReceived(
    241         const sp<AMessage> &msg) {
    242     switch (msg->what()) {
    243         case kWhatMediaPullerStopped:
    244         {
    245             mConverter.clear();
    246 
    247             mStarted = false;
    248 
    249             sp<AMessage> notify = mNotify->dup();
    250             notify->setInt32("what", kWhatStopped);
    251             notify->post();
    252 
    253             ALOGI("kWhatStopped %s posted", mIsAudio ? "audio" : "video");
    254             break;
    255         }
    256 
    257         default:
    258             TRESPASS();
    259     }
    260 }
    261 
    262 void WifiDisplaySource::PlaybackSession::Track::queueAccessUnit(
    263         const sp<ABuffer> &accessUnit) {
    264     mQueuedAccessUnits.push_back(accessUnit);
    265 }
    266 
    267 sp<ABuffer> WifiDisplaySource::PlaybackSession::Track::dequeueAccessUnit() {
    268     if (mQueuedAccessUnits.empty()) {
    269         return NULL;
    270     }
    271 
    272     sp<ABuffer> accessUnit = *mQueuedAccessUnits.begin();
    273     CHECK(accessUnit != NULL);
    274 
    275     mQueuedAccessUnits.erase(mQueuedAccessUnits.begin());
    276 
    277     return accessUnit;
    278 }
    279 
    280 void WifiDisplaySource::PlaybackSession::Track::setRepeaterSource(
    281         const sp<RepeaterSource> &source) {
    282     mRepeaterSource = source;
    283 }
    284 
    285 void WifiDisplaySource::PlaybackSession::Track::requestIDRFrame() {
    286     if (mIsAudio) {
    287         return;
    288     }
    289 
    290     if (mRepeaterSource != NULL) {
    291         mRepeaterSource->wakeUp();
    292     }
    293 
    294     mConverter->requestIDRFrame();
    295 }
    296 
    297 bool WifiDisplaySource::PlaybackSession::Track::hasOutputBuffer(
    298         int64_t *timeUs) const {
    299     *timeUs = 0ll;
    300 
    301     if (mQueuedOutputBuffers.empty()) {
    302         return false;
    303     }
    304 
    305     const sp<ABuffer> &outputBuffer = *mQueuedOutputBuffers.begin();
    306 
    307     CHECK(outputBuffer->meta()->findInt64("timeUs", timeUs));
    308 
    309     return true;
    310 }
    311 
    312 void WifiDisplaySource::PlaybackSession::Track::queueOutputBuffer(
    313         const sp<ABuffer> &accessUnit) {
    314     mQueuedOutputBuffers.push_back(accessUnit);
    315     mLastOutputBufferQueuedTimeUs = ALooper::GetNowUs();
    316 }
    317 
    318 sp<ABuffer> WifiDisplaySource::PlaybackSession::Track::dequeueOutputBuffer() {
    319     CHECK(!mQueuedOutputBuffers.empty());
    320 
    321     sp<ABuffer> outputBuffer = *mQueuedOutputBuffers.begin();
    322     mQueuedOutputBuffers.erase(mQueuedOutputBuffers.begin());
    323 
    324     return outputBuffer;
    325 }
    326 
    327 #if SUSPEND_VIDEO_IF_IDLE
    328 bool WifiDisplaySource::PlaybackSession::Track::isSuspended() const {
    329     if (!mQueuedOutputBuffers.empty()) {
    330         return false;
    331     }
    332 
    333     if (mLastOutputBufferQueuedTimeUs < 0ll) {
    334         // We've never seen an output buffer queued, but tracks start
    335         // out live, not suspended.
    336         return false;
    337     }
    338 
    339     // If we've not seen new output data for 60ms or more, we consider
    340     // this track suspended for the time being.
    341     return (ALooper::GetNowUs() - mLastOutputBufferQueuedTimeUs) > 60000ll;
    342 }
    343 #endif
    344 
    345 ////////////////////////////////////////////////////////////////////////////////
    346 
    347 WifiDisplaySource::PlaybackSession::PlaybackSession(
    348         const sp<ANetworkSession> &netSession,
    349         const sp<AMessage> &notify,
    350         const in_addr &interfaceAddr,
    351         const sp<IHDCP> &hdcp,
    352         const char *path)
    353     : mNetSession(netSession),
    354       mNotify(notify),
    355       mInterfaceAddr(interfaceAddr),
    356       mHDCP(hdcp),
    357       mLocalRTPPort(-1),
    358       mWeAreDead(false),
    359       mPaused(false),
    360       mLastLifesignUs(),
    361       mVideoTrackIndex(-1),
    362       mPrevTimeUs(-1ll),
    363       mPullExtractorPending(false),
    364       mPullExtractorGeneration(0),
    365       mFirstSampleTimeRealUs(-1ll),
    366       mFirstSampleTimeUs(-1ll) {
    367     if (path != NULL) {
    368         mMediaPath.setTo(path);
    369     }
    370 }
    371 
    372 status_t WifiDisplaySource::PlaybackSession::init(
    373         const char *clientIP,
    374         int32_t clientRtp,
    375         RTPSender::TransportMode rtpMode,
    376         int32_t clientRtcp,
    377         RTPSender::TransportMode rtcpMode,
    378         bool enableAudio,
    379         bool usePCMAudio,
    380         bool enableVideo,
    381         VideoFormats::ResolutionType videoResolutionType,
    382         size_t videoResolutionIndex,
    383         VideoFormats::ProfileType videoProfileType,
    384         VideoFormats::LevelType videoLevelType) {
    385     sp<AMessage> notify = new AMessage(kWhatMediaSenderNotify, id());
    386     mMediaSender = new MediaSender(mNetSession, notify);
    387     looper()->registerHandler(mMediaSender);
    388 
    389     mMediaSender->setHDCP(mHDCP);
    390 
    391     status_t err = setupPacketizer(
    392             enableAudio,
    393             usePCMAudio,
    394             enableVideo,
    395             videoResolutionType,
    396             videoResolutionIndex,
    397             videoProfileType,
    398             videoLevelType);
    399 
    400     if (err == OK) {
    401         err = mMediaSender->initAsync(
    402                 -1 /* trackIndex */,
    403                 clientIP,
    404                 clientRtp,
    405                 rtpMode,
    406                 clientRtcp,
    407                 rtcpMode,
    408                 &mLocalRTPPort);
    409     }
    410 
    411     if (err != OK) {
    412         mLocalRTPPort = -1;
    413 
    414         looper()->unregisterHandler(mMediaSender->id());
    415         mMediaSender.clear();
    416 
    417         return err;
    418     }
    419 
    420     updateLiveness();
    421 
    422     return OK;
    423 }
    424 
    425 WifiDisplaySource::PlaybackSession::~PlaybackSession() {
    426 }
    427 
    428 int32_t WifiDisplaySource::PlaybackSession::getRTPPort() const {
    429     return mLocalRTPPort;
    430 }
    431 
    432 int64_t WifiDisplaySource::PlaybackSession::getLastLifesignUs() const {
    433     return mLastLifesignUs;
    434 }
    435 
    436 void WifiDisplaySource::PlaybackSession::updateLiveness() {
    437     mLastLifesignUs = ALooper::GetNowUs();
    438 }
    439 
    440 status_t WifiDisplaySource::PlaybackSession::play() {
    441     updateLiveness();
    442 
    443     (new AMessage(kWhatResume, id()))->post();
    444 
    445     return OK;
    446 }
    447 
    448 status_t WifiDisplaySource::PlaybackSession::onMediaSenderInitialized() {
    449     for (size_t i = 0; i < mTracks.size(); ++i) {
    450         CHECK_EQ((status_t)OK, mTracks.editValueAt(i)->start());
    451     }
    452 
    453     sp<AMessage> notify = mNotify->dup();
    454     notify->setInt32("what", kWhatSessionEstablished);
    455     notify->post();
    456 
    457     return OK;
    458 }
    459 
    460 status_t WifiDisplaySource::PlaybackSession::pause() {
    461     updateLiveness();
    462 
    463     (new AMessage(kWhatPause, id()))->post();
    464 
    465     return OK;
    466 }
    467 
    468 void WifiDisplaySource::PlaybackSession::destroyAsync() {
    469     ALOGI("destroyAsync");
    470 
    471     for (size_t i = 0; i < mTracks.size(); ++i) {
    472         mTracks.valueAt(i)->stopAsync();
    473     }
    474 }
    475 
    476 void WifiDisplaySource::PlaybackSession::onMessageReceived(
    477         const sp<AMessage> &msg) {
    478     switch (msg->what()) {
    479         case kWhatConverterNotify:
    480         {
    481             if (mWeAreDead) {
    482                 ALOGV("dropping msg '%s' because we're dead",
    483                       msg->debugString().c_str());
    484 
    485                 break;
    486             }
    487 
    488             int32_t what;
    489             CHECK(msg->findInt32("what", &what));
    490 
    491             size_t trackIndex;
    492             CHECK(msg->findSize("trackIndex", &trackIndex));
    493 
    494             if (what == Converter::kWhatAccessUnit) {
    495                 sp<ABuffer> accessUnit;
    496                 CHECK(msg->findBuffer("accessUnit", &accessUnit));
    497 
    498                 const sp<Track> &track = mTracks.valueFor(trackIndex);
    499 
    500                 status_t err = mMediaSender->queueAccessUnit(
    501                         track->mediaSenderTrackIndex(),
    502                         accessUnit);
    503 
    504                 if (err != OK) {
    505                     notifySessionDead();
    506                 }
    507                 break;
    508             } else if (what == Converter::kWhatEOS) {
    509                 CHECK_EQ(what, Converter::kWhatEOS);
    510 
    511                 ALOGI("output EOS on track %d", trackIndex);
    512 
    513                 ssize_t index = mTracks.indexOfKey(trackIndex);
    514                 CHECK_GE(index, 0);
    515 
    516                 const sp<Converter> &converter =
    517                     mTracks.valueAt(index)->converter();
    518                 looper()->unregisterHandler(converter->id());
    519 
    520                 mTracks.removeItemsAt(index);
    521 
    522                 if (mTracks.isEmpty()) {
    523                     ALOGI("Reached EOS");
    524                 }
    525             } else if (what != Converter::kWhatShutdownCompleted) {
    526                 CHECK_EQ(what, Converter::kWhatError);
    527 
    528                 status_t err;
    529                 CHECK(msg->findInt32("err", &err));
    530 
    531                 ALOGE("converter signaled error %d", err);
    532 
    533                 notifySessionDead();
    534             }
    535             break;
    536         }
    537 
    538         case kWhatMediaSenderNotify:
    539         {
    540             int32_t what;
    541             CHECK(msg->findInt32("what", &what));
    542 
    543             if (what == MediaSender::kWhatInitDone) {
    544                 status_t err;
    545                 CHECK(msg->findInt32("err", &err));
    546 
    547                 if (err == OK) {
    548                     onMediaSenderInitialized();
    549                 } else {
    550                     notifySessionDead();
    551                 }
    552             } else if (what == MediaSender::kWhatError) {
    553                 notifySessionDead();
    554             } else if (what == MediaSender::kWhatNetworkStall) {
    555                 size_t numBytesQueued;
    556                 CHECK(msg->findSize("numBytesQueued", &numBytesQueued));
    557 
    558                 if (mVideoTrackIndex >= 0) {
    559                     const sp<Track> &videoTrack =
    560                         mTracks.valueFor(mVideoTrackIndex);
    561 
    562                     sp<Converter> converter = videoTrack->converter();
    563                     if (converter != NULL) {
    564                         converter->dropAFrame();
    565                     }
    566                 }
    567             } else if (what == MediaSender::kWhatInformSender) {
    568                 onSinkFeedback(msg);
    569             } else {
    570                 TRESPASS();
    571             }
    572             break;
    573         }
    574 
    575         case kWhatTrackNotify:
    576         {
    577             int32_t what;
    578             CHECK(msg->findInt32("what", &what));
    579 
    580             size_t trackIndex;
    581             CHECK(msg->findSize("trackIndex", &trackIndex));
    582 
    583             if (what == Track::kWhatStopped) {
    584                 ALOGI("Track %d stopped", trackIndex);
    585 
    586                 sp<Track> track = mTracks.valueFor(trackIndex);
    587                 looper()->unregisterHandler(track->id());
    588                 mTracks.removeItem(trackIndex);
    589                 track.clear();
    590 
    591                 if (!mTracks.isEmpty()) {
    592                     ALOGI("not all tracks are stopped yet");
    593                     break;
    594                 }
    595 
    596                 looper()->unregisterHandler(mMediaSender->id());
    597                 mMediaSender.clear();
    598 
    599                 sp<AMessage> notify = mNotify->dup();
    600                 notify->setInt32("what", kWhatSessionDestroyed);
    601                 notify->post();
    602             }
    603             break;
    604         }
    605 
    606         case kWhatPause:
    607         {
    608             if (mExtractor != NULL) {
    609                 ++mPullExtractorGeneration;
    610                 mFirstSampleTimeRealUs = -1ll;
    611                 mFirstSampleTimeUs = -1ll;
    612             }
    613 
    614             if (mPaused) {
    615                 break;
    616             }
    617 
    618             for (size_t i = 0; i < mTracks.size(); ++i) {
    619                 mTracks.editValueAt(i)->pause();
    620             }
    621 
    622             mPaused = true;
    623             break;
    624         }
    625 
    626         case kWhatResume:
    627         {
    628             if (mExtractor != NULL) {
    629                 schedulePullExtractor();
    630             }
    631 
    632             if (!mPaused) {
    633                 break;
    634             }
    635 
    636             for (size_t i = 0; i < mTracks.size(); ++i) {
    637                 mTracks.editValueAt(i)->resume();
    638             }
    639 
    640             mPaused = false;
    641             break;
    642         }
    643 
    644         case kWhatPullExtractorSample:
    645         {
    646             int32_t generation;
    647             CHECK(msg->findInt32("generation", &generation));
    648 
    649             if (generation != mPullExtractorGeneration) {
    650                 break;
    651             }
    652 
    653             mPullExtractorPending = false;
    654 
    655             onPullExtractor();
    656             break;
    657         }
    658 
    659         default:
    660             TRESPASS();
    661     }
    662 }
    663 
    664 void WifiDisplaySource::PlaybackSession::onSinkFeedback(const sp<AMessage> &msg) {
    665     int64_t avgLatencyUs;
    666     CHECK(msg->findInt64("avgLatencyUs", &avgLatencyUs));
    667 
    668     int64_t maxLatencyUs;
    669     CHECK(msg->findInt64("maxLatencyUs", &maxLatencyUs));
    670 
    671     ALOGI("sink reports avg. latency of %lld ms (max %lld ms)",
    672           avgLatencyUs / 1000ll,
    673           maxLatencyUs / 1000ll);
    674 
    675     if (mVideoTrackIndex >= 0) {
    676         const sp<Track> &videoTrack = mTracks.valueFor(mVideoTrackIndex);
    677         sp<Converter> converter = videoTrack->converter();
    678 
    679         if (converter != NULL) {
    680             int32_t videoBitrate =
    681                 Converter::GetInt32Property("media.wfd.video-bitrate", -1);
    682 
    683             char val[PROPERTY_VALUE_MAX];
    684             if (videoBitrate < 0
    685                     && property_get("media.wfd.video-bitrate", val, NULL)
    686                     && !strcasecmp("adaptive", val)) {
    687                 videoBitrate = converter->getVideoBitrate();
    688 
    689                 if (avgLatencyUs > 300000ll) {
    690                     videoBitrate *= 0.6;
    691                 } else if (avgLatencyUs < 100000ll) {
    692                     videoBitrate *= 1.1;
    693                 }
    694             }
    695 
    696             if (videoBitrate > 0) {
    697                 if (videoBitrate < 500000) {
    698                     videoBitrate = 500000;
    699                 } else if (videoBitrate > 10000000) {
    700                     videoBitrate = 10000000;
    701                 }
    702 
    703                 if (videoBitrate != converter->getVideoBitrate()) {
    704                     ALOGI("setting video bitrate to %d bps", videoBitrate);
    705 
    706                     converter->setVideoBitrate(videoBitrate);
    707                 }
    708             }
    709         }
    710 
    711         sp<RepeaterSource> repeaterSource = videoTrack->repeaterSource();
    712         if (repeaterSource != NULL) {
    713             double rateHz =
    714                 Converter::GetInt32Property(
    715                         "media.wfd.video-framerate", -1);
    716 
    717             char val[PROPERTY_VALUE_MAX];
    718             if (rateHz < 0.0
    719                     && property_get("media.wfd.video-framerate", val, NULL)
    720                     && !strcasecmp("adaptive", val)) {
    721                  rateHz = repeaterSource->getFrameRate();
    722 
    723                 if (avgLatencyUs > 300000ll) {
    724                     rateHz *= 0.9;
    725                 } else if (avgLatencyUs < 200000ll) {
    726                     rateHz *= 1.1;
    727                 }
    728             }
    729 
    730             if (rateHz > 0) {
    731                 if (rateHz < 5.0) {
    732                     rateHz = 5.0;
    733                 } else if (rateHz > 30.0) {
    734                     rateHz = 30.0;
    735                 }
    736 
    737                 if (rateHz != repeaterSource->getFrameRate()) {
    738                     ALOGI("setting frame rate to %.2f Hz", rateHz);
    739 
    740                     repeaterSource->setFrameRate(rateHz);
    741                 }
    742             }
    743         }
    744     }
    745 }
    746 
    747 status_t WifiDisplaySource::PlaybackSession::setupMediaPacketizer(
    748         bool enableAudio, bool enableVideo) {
    749     DataSource::RegisterDefaultSniffers();
    750 
    751     mExtractor = new NuMediaExtractor;
    752 
    753     status_t err = mExtractor->setDataSource(
    754             NULL /* httpService */, mMediaPath.c_str());
    755 
    756     if (err != OK) {
    757         return err;
    758     }
    759 
    760     size_t n = mExtractor->countTracks();
    761     bool haveAudio = false;
    762     bool haveVideo = false;
    763     for (size_t i = 0; i < n; ++i) {
    764         sp<AMessage> format;
    765         err = mExtractor->getTrackFormat(i, &format);
    766 
    767         if (err != OK) {
    768             continue;
    769         }
    770 
    771         AString mime;
    772         CHECK(format->findString("mime", &mime));
    773 
    774         bool isAudio = !strncasecmp(mime.c_str(), "audio/", 6);
    775         bool isVideo = !strncasecmp(mime.c_str(), "video/", 6);
    776 
    777         if (isAudio && enableAudio && !haveAudio) {
    778             haveAudio = true;
    779         } else if (isVideo && enableVideo && !haveVideo) {
    780             haveVideo = true;
    781         } else {
    782             continue;
    783         }
    784 
    785         err = mExtractor->selectTrack(i);
    786 
    787         size_t trackIndex = mTracks.size();
    788 
    789         sp<AMessage> notify = new AMessage(kWhatTrackNotify, id());
    790         notify->setSize("trackIndex", trackIndex);
    791 
    792         sp<Track> track = new Track(notify, format);
    793         looper()->registerHandler(track);
    794 
    795         mTracks.add(trackIndex, track);
    796 
    797         mExtractorTrackToInternalTrack.add(i, trackIndex);
    798 
    799         if (isVideo) {
    800             mVideoTrackIndex = trackIndex;
    801         }
    802 
    803         uint32_t flags = MediaSender::FLAG_MANUALLY_PREPEND_SPS_PPS;
    804 
    805         ssize_t mediaSenderTrackIndex =
    806             mMediaSender->addTrack(format, flags);
    807         CHECK_GE(mediaSenderTrackIndex, 0);
    808 
    809         track->setMediaSenderTrackIndex(mediaSenderTrackIndex);
    810 
    811         if ((haveAudio || !enableAudio) && (haveVideo || !enableVideo)) {
    812             break;
    813         }
    814     }
    815 
    816     return OK;
    817 }
    818 
    819 void WifiDisplaySource::PlaybackSession::schedulePullExtractor() {
    820     if (mPullExtractorPending) {
    821         return;
    822     }
    823 
    824     int64_t sampleTimeUs;
    825     status_t err = mExtractor->getSampleTime(&sampleTimeUs);
    826 
    827     int64_t nowUs = ALooper::GetNowUs();
    828 
    829     if (mFirstSampleTimeRealUs < 0ll) {
    830         mFirstSampleTimeRealUs = nowUs;
    831         mFirstSampleTimeUs = sampleTimeUs;
    832     }
    833 
    834     int64_t whenUs = sampleTimeUs - mFirstSampleTimeUs + mFirstSampleTimeRealUs;
    835 
    836     sp<AMessage> msg = new AMessage(kWhatPullExtractorSample, id());
    837     msg->setInt32("generation", mPullExtractorGeneration);
    838     msg->post(whenUs - nowUs);
    839 
    840     mPullExtractorPending = true;
    841 }
    842 
    843 void WifiDisplaySource::PlaybackSession::onPullExtractor() {
    844     sp<ABuffer> accessUnit = new ABuffer(1024 * 1024);
    845     status_t err = mExtractor->readSampleData(accessUnit);
    846     if (err != OK) {
    847         // EOS.
    848         return;
    849     }
    850 
    851     int64_t timeUs;
    852     CHECK_EQ((status_t)OK, mExtractor->getSampleTime(&timeUs));
    853 
    854     accessUnit->meta()->setInt64(
    855             "timeUs", mFirstSampleTimeRealUs + timeUs - mFirstSampleTimeUs);
    856 
    857     size_t trackIndex;
    858     CHECK_EQ((status_t)OK, mExtractor->getSampleTrackIndex(&trackIndex));
    859 
    860     sp<AMessage> msg = new AMessage(kWhatConverterNotify, id());
    861 
    862     msg->setSize(
    863             "trackIndex", mExtractorTrackToInternalTrack.valueFor(trackIndex));
    864 
    865     msg->setInt32("what", Converter::kWhatAccessUnit);
    866     msg->setBuffer("accessUnit", accessUnit);
    867     msg->post();
    868 
    869     mExtractor->advance();
    870 
    871     schedulePullExtractor();
    872 }
    873 
    874 status_t WifiDisplaySource::PlaybackSession::setupPacketizer(
    875         bool enableAudio,
    876         bool usePCMAudio,
    877         bool enableVideo,
    878         VideoFormats::ResolutionType videoResolutionType,
    879         size_t videoResolutionIndex,
    880         VideoFormats::ProfileType videoProfileType,
    881         VideoFormats::LevelType videoLevelType) {
    882     CHECK(enableAudio || enableVideo);
    883 
    884     if (!mMediaPath.empty()) {
    885         return setupMediaPacketizer(enableAudio, enableVideo);
    886     }
    887 
    888     if (enableVideo) {
    889         status_t err = addVideoSource(
    890                 videoResolutionType, videoResolutionIndex, videoProfileType,
    891                 videoLevelType);
    892 
    893         if (err != OK) {
    894             return err;
    895         }
    896     }
    897 
    898     if (!enableAudio) {
    899         return OK;
    900     }
    901 
    902     return addAudioSource(usePCMAudio);
    903 }
    904 
    905 status_t WifiDisplaySource::PlaybackSession::addSource(
    906         bool isVideo, const sp<MediaSource> &source, bool isRepeaterSource,
    907         bool usePCMAudio, unsigned profileIdc, unsigned levelIdc,
    908         unsigned constraintSet, size_t *numInputBuffers) {
    909     CHECK(!usePCMAudio || !isVideo);
    910     CHECK(!isRepeaterSource || isVideo);
    911     CHECK(!profileIdc || isVideo);
    912     CHECK(!levelIdc || isVideo);
    913     CHECK(!constraintSet || isVideo);
    914 
    915     sp<ALooper> pullLooper = new ALooper;
    916     pullLooper->setName("pull_looper");
    917 
    918     pullLooper->start(
    919             false /* runOnCallingThread */,
    920             false /* canCallJava */,
    921             PRIORITY_AUDIO);
    922 
    923     sp<ALooper> codecLooper = new ALooper;
    924     codecLooper->setName("codec_looper");
    925 
    926     codecLooper->start(
    927             false /* runOnCallingThread */,
    928             false /* canCallJava */,
    929             PRIORITY_AUDIO);
    930 
    931     size_t trackIndex;
    932 
    933     sp<AMessage> notify;
    934 
    935     trackIndex = mTracks.size();
    936 
    937     sp<AMessage> format;
    938     status_t err = convertMetaDataToMessage(source->getFormat(), &format);
    939     CHECK_EQ(err, (status_t)OK);
    940 
    941     if (isVideo) {
    942         format->setString("mime", MEDIA_MIMETYPE_VIDEO_AVC);
    943         format->setInt32("store-metadata-in-buffers", true);
    944         format->setInt32("store-metadata-in-buffers-output", (mHDCP != NULL)
    945                 && (mHDCP->getCaps() & HDCPModule::HDCP_CAPS_ENCRYPT_NATIVE));
    946         format->setInt32(
    947                 "color-format", OMX_COLOR_FormatAndroidOpaque);
    948         format->setInt32("profile-idc", profileIdc);
    949         format->setInt32("level-idc", levelIdc);
    950         format->setInt32("constraint-set", constraintSet);
    951     } else {
    952         format->setString(
    953                 "mime",
    954                 usePCMAudio
    955                     ? MEDIA_MIMETYPE_AUDIO_RAW : MEDIA_MIMETYPE_AUDIO_AAC);
    956     }
    957 
    958     notify = new AMessage(kWhatConverterNotify, id());
    959     notify->setSize("trackIndex", trackIndex);
    960 
    961     sp<Converter> converter = new Converter(notify, codecLooper, format);
    962 
    963     looper()->registerHandler(converter);
    964 
    965     err = converter->init();
    966     if (err != OK) {
    967         ALOGE("%s converter returned err %d", isVideo ? "video" : "audio", err);
    968 
    969         looper()->unregisterHandler(converter->id());
    970         return err;
    971     }
    972 
    973     notify = new AMessage(Converter::kWhatMediaPullerNotify, converter->id());
    974     notify->setSize("trackIndex", trackIndex);
    975 
    976     sp<MediaPuller> puller = new MediaPuller(source, notify);
    977     pullLooper->registerHandler(puller);
    978 
    979     if (numInputBuffers != NULL) {
    980         *numInputBuffers = converter->getInputBufferCount();
    981     }
    982 
    983     notify = new AMessage(kWhatTrackNotify, id());
    984     notify->setSize("trackIndex", trackIndex);
    985 
    986     sp<Track> track = new Track(
    987             notify, pullLooper, codecLooper, puller, converter);
    988 
    989     if (isRepeaterSource) {
    990         track->setRepeaterSource(static_cast<RepeaterSource *>(source.get()));
    991     }
    992 
    993     looper()->registerHandler(track);
    994 
    995     mTracks.add(trackIndex, track);
    996 
    997     if (isVideo) {
    998         mVideoTrackIndex = trackIndex;
    999     }
   1000 
   1001     uint32_t flags = 0;
   1002     if (converter->needToManuallyPrependSPSPPS()) {
   1003         flags |= MediaSender::FLAG_MANUALLY_PREPEND_SPS_PPS;
   1004     }
   1005 
   1006     ssize_t mediaSenderTrackIndex =
   1007         mMediaSender->addTrack(converter->getOutputFormat(), flags);
   1008     CHECK_GE(mediaSenderTrackIndex, 0);
   1009 
   1010     track->setMediaSenderTrackIndex(mediaSenderTrackIndex);
   1011 
   1012     return OK;
   1013 }
   1014 
   1015 status_t WifiDisplaySource::PlaybackSession::addVideoSource(
   1016         VideoFormats::ResolutionType videoResolutionType,
   1017         size_t videoResolutionIndex,
   1018         VideoFormats::ProfileType videoProfileType,
   1019         VideoFormats::LevelType videoLevelType) {
   1020     size_t width, height, framesPerSecond;
   1021     bool interlaced;
   1022     CHECK(VideoFormats::GetConfiguration(
   1023                 videoResolutionType,
   1024                 videoResolutionIndex,
   1025                 &width,
   1026                 &height,
   1027                 &framesPerSecond,
   1028                 &interlaced));
   1029 
   1030     unsigned profileIdc, levelIdc, constraintSet;
   1031     CHECK(VideoFormats::GetProfileLevel(
   1032                 videoProfileType,
   1033                 videoLevelType,
   1034                 &profileIdc,
   1035                 &levelIdc,
   1036                 &constraintSet));
   1037 
   1038     sp<SurfaceMediaSource> source = new SurfaceMediaSource(width, height);
   1039 
   1040     source->setUseAbsoluteTimestamps();
   1041 
   1042     sp<RepeaterSource> videoSource =
   1043         new RepeaterSource(source, framesPerSecond);
   1044 
   1045     size_t numInputBuffers;
   1046     status_t err = addSource(
   1047             true /* isVideo */, videoSource, true /* isRepeaterSource */,
   1048             false /* usePCMAudio */, profileIdc, levelIdc, constraintSet,
   1049             &numInputBuffers);
   1050 
   1051     if (err != OK) {
   1052         return err;
   1053     }
   1054 
   1055     err = source->setMaxAcquiredBufferCount(numInputBuffers);
   1056     CHECK_EQ(err, (status_t)OK);
   1057 
   1058     mProducer = source->getProducer();
   1059 
   1060     return OK;
   1061 }
   1062 
   1063 status_t WifiDisplaySource::PlaybackSession::addAudioSource(bool usePCMAudio) {
   1064     sp<AudioSource> audioSource = new AudioSource(
   1065             AUDIO_SOURCE_REMOTE_SUBMIX,
   1066             48000 /* sampleRate */,
   1067             2 /* channelCount */);
   1068 
   1069     if (audioSource->initCheck() == OK) {
   1070         return addSource(
   1071                 false /* isVideo */, audioSource, false /* isRepeaterSource */,
   1072                 usePCMAudio, 0 /* profileIdc */, 0 /* levelIdc */,
   1073                 0 /* constraintSet */, NULL /* numInputBuffers */);
   1074     }
   1075 
   1076     ALOGW("Unable to instantiate audio source");
   1077 
   1078     return OK;
   1079 }
   1080 
   1081 sp<IGraphicBufferProducer> WifiDisplaySource::PlaybackSession::getSurfaceTexture() {
   1082     return mProducer;
   1083 }
   1084 
   1085 void WifiDisplaySource::PlaybackSession::requestIDRFrame() {
   1086     for (size_t i = 0; i < mTracks.size(); ++i) {
   1087         const sp<Track> &track = mTracks.valueAt(i);
   1088 
   1089         track->requestIDRFrame();
   1090     }
   1091 }
   1092 
   1093 void WifiDisplaySource::PlaybackSession::notifySessionDead() {
   1094     // Inform WifiDisplaySource of our premature death (wish).
   1095     sp<AMessage> notify = mNotify->dup();
   1096     notify->setInt32("what", kWhatSessionDead);
   1097     notify->post();
   1098 
   1099     mWeAreDead = true;
   1100 }
   1101 
   1102 }  // namespace android
   1103 
   1104