Home | History | Annotate | Download | only in source 
      1 /*
      2  * Copyright 2012, The Android Open Source Project
      3  *
      4  * Licensed under the Apache License, Version 2.0 (the "License");
      5  * you may not use this file except in compliance with the License.
      6  * You may obtain a copy of the License at
      7  *
      8  *     http://www.apache.org/licenses/LICENSE-2.0
      9  *
     10  * Unless required by applicable law or agreed to in writing, software
     11  * distributed under the License is distributed on an "AS IS" BASIS,
     12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
     13  * See the License for the specific language governing permissions and
     14  * limitations under the License.
     15  */
     16 
     17 //#define LOG_NDEBUG 0
     18 #define LOG_TAG "PlaybackSession"
     19 #include <utils/Log.h>
     20 
     21 #include "PlaybackSession.h"
     22 
     23 #include "Converter.h"
     24 #include "MediaPuller.h"
     25 #include "RepeaterSource.h"
     26 #include "include/avc_utils.h"
     27 #include "WifiDisplaySource.h"
     28 
     29 #include <binder/IServiceManager.h>
     30 #include <cutils/properties.h>
     31 #include <media/IHDCP.h>
     32 #include <media/IMediaHTTPService.h>
     33 #include <media/stagefright/foundation/ABitReader.h>
     34 #include <media/stagefright/foundation/ABuffer.h>
     35 #include <media/stagefright/foundation/ADebug.h>
     36 #include <media/stagefright/foundation/AMessage.h>
     37 #include <media/stagefright/foundation/hexdump.h>
     38 #include <media/stagefright/AudioSource.h>
     39 #include <media/stagefright/MediaDefs.h>
     40 #include <media/stagefright/MediaErrors.h>
     41 #include <media/stagefright/MediaSource.h>
     42 #include <media/stagefright/MetaData.h>
     43 #include <media/stagefright/NuMediaExtractor.h>
     44 #include <media/stagefright/SurfaceMediaSource.h>
     45 #include <media/stagefright/Utils.h>
     46 
     47 #include <OMX_IVCommon.h>
     48 
     49 namespace android {
     50 
     51 struct WifiDisplaySource::PlaybackSession::Track : public AHandler {
     52     enum {
     53         kWhatStopped,
     54     };
     55 
     56     Track(const sp<AMessage> &notify,
     57           const sp<ALooper> &pullLooper,
     58           const sp<ALooper> &codecLooper,
     59           const sp<MediaPuller> &mediaPuller,
     60           const sp<Converter> &converter);
     61 
     62     Track(const sp<AMessage> &notify, const sp<AMessage> &format);
     63 
     64     void setRepeaterSource(const sp<RepeaterSource> &source);
     65 
     66     sp<AMessage> getFormat();
     67     bool isAudio() const;
     68 
     69     const sp<Converter> &converter() const;
     70     const sp<RepeaterSource> &repeaterSource() const;
     71 
     72     ssize_t mediaSenderTrackIndex() const;
     73     void setMediaSenderTrackIndex(size_t index);
     74 
     75     status_t start();
     76     void stopAsync();
     77 
     78     void pause();
     79     void resume();
     80 
     81     void queueAccessUnit(const sp<ABuffer> &accessUnit);
     82     sp<ABuffer> dequeueAccessUnit();
     83 
     84     bool hasOutputBuffer(int64_t *timeUs) const;
     85     void queueOutputBuffer(const sp<ABuffer> &accessUnit);
     86     sp<ABuffer> dequeueOutputBuffer();
     87 
     88 #if SUSPEND_VIDEO_IF_IDLE
     89     bool isSuspended() const;
     90 #endif
     91 
     92     size_t countQueuedOutputBuffers() const {
     93         return mQueuedOutputBuffers.size();
     94     }
     95 
     96     void requestIDRFrame();
     97 
     98 protected:
     99     virtual void onMessageReceived(const sp<AMessage> &msg);
    100     virtual ~Track();
    101 
    102 private:
    103     enum {
    104         kWhatMediaPullerStopped,
    105     };
    106 
    107     sp<AMessage> mNotify;
    108     sp<ALooper> mPullLooper;
    109     sp<ALooper> mCodecLooper;
    110     sp<MediaPuller> mMediaPuller;
    111     sp<Converter> mConverter;
    112     sp<AMessage> mFormat;
    113     bool mStarted;
    114     ssize_t mMediaSenderTrackIndex;
    115     bool mIsAudio;
    116     List<sp<ABuffer> > mQueuedAccessUnits;
    117     sp<RepeaterSource> mRepeaterSource;
    118     List<sp<ABuffer> > mQueuedOutputBuffers;
    119     int64_t mLastOutputBufferQueuedTimeUs;
    120 
    121     static bool IsAudioFormat(const sp<AMessage> &format);
    122 
    123     DISALLOW_EVIL_CONSTRUCTORS(Track);
    124 };
    125 
    126 WifiDisplaySource::PlaybackSession::Track::Track(
    127         const sp<AMessage> &notify,
    128         const sp<ALooper> &pullLooper,
    129         const sp<ALooper> &codecLooper,
    130         const sp<MediaPuller> &mediaPuller,
    131         const sp<Converter> &converter)
    132     : mNotify(notify),
    133       mPullLooper(pullLooper),
    134       mCodecLooper(codecLooper),
    135       mMediaPuller(mediaPuller),
    136       mConverter(converter),
    137       mStarted(false),
    138       mIsAudio(IsAudioFormat(mConverter->getOutputFormat())),
    139       mLastOutputBufferQueuedTimeUs(-1ll) {
    140 }
    141 
    142 WifiDisplaySource::PlaybackSession::Track::Track(
    143         const sp<AMessage> &notify, const sp<AMessage> &format)
    144     : mNotify(notify),
    145       mFormat(format),
    146       mStarted(false),
    147       mIsAudio(IsAudioFormat(format)),
    148       mLastOutputBufferQueuedTimeUs(-1ll) {
    149 }
    150 
    151 WifiDisplaySource::PlaybackSession::Track::~Track() {
    152     CHECK(!mStarted);
    153 }
    154 
    155 // static
    156 bool WifiDisplaySource::PlaybackSession::Track::IsAudioFormat(
    157         const sp<AMessage> &format) {
    158     AString mime;
    159     CHECK(format->findString("mime", &mime));
    160 
    161     return !strncasecmp(mime.c_str(), "audio/", 6);
    162 }
    163 
    164 sp<AMessage> WifiDisplaySource::PlaybackSession::Track::getFormat() {
    165     return mFormat != NULL ? mFormat : mConverter->getOutputFormat();
    166 }
    167 
    168 bool WifiDisplaySource::PlaybackSession::Track::isAudio() const {
    169     return mIsAudio;
    170 }
    171 
    172 const sp<Converter> &WifiDisplaySource::PlaybackSession::Track::converter() const {
    173     return mConverter;
    174 }
    175 
    176 const sp<RepeaterSource> &
    177 WifiDisplaySource::PlaybackSession::Track::repeaterSource() const {
    178     return mRepeaterSource;
    179 }
    180 
    181 ssize_t WifiDisplaySource::PlaybackSession::Track::mediaSenderTrackIndex() const {
    182     CHECK_GE(mMediaSenderTrackIndex, 0);
    183     return mMediaSenderTrackIndex;
    184 }
    185 
    186 void WifiDisplaySource::PlaybackSession::Track::setMediaSenderTrackIndex(
    187         size_t index) {
    188     mMediaSenderTrackIndex = index;
    189 }
    190 
    191 status_t WifiDisplaySource::PlaybackSession::Track::start() {
    192     ALOGV("Track::start isAudio=%d", mIsAudio);
    193 
    194     CHECK(!mStarted);
    195 
    196     status_t err = OK;
    197 
    198     if (mMediaPuller != NULL) {
    199         err = mMediaPuller->start();
    200     }
    201 
    202     if (err == OK) {
    203         mStarted = true;
    204     }
    205 
    206     return err;
    207 }
    208 
    209 void WifiDisplaySource::PlaybackSession::Track::stopAsync() {
    210     ALOGV("Track::stopAsync isAudio=%d", mIsAudio);
    211 
    212     if (mConverter != NULL) {
    213         mConverter->shutdownAsync();
    214     }
    215 
    216     sp<AMessage> msg = new AMessage(kWhatMediaPullerStopped, this);
    217 
    218     if (mStarted && mMediaPuller != NULL) {
    219         if (mRepeaterSource != NULL) {
    220             // Let's unblock MediaPuller's MediaSource::read().
    221             mRepeaterSource->wakeUp();
    222         }
    223 
    224         mMediaPuller->stopAsync(msg);
    225     } else {
    226         mStarted = false;
    227         msg->post();
    228     }
    229 }
    230 
    231 void WifiDisplaySource::PlaybackSession::Track::pause() {
    232     mMediaPuller->pause();
    233 }
    234 
    235 void WifiDisplaySource::PlaybackSession::Track::resume() {
    236     mMediaPuller->resume();
    237 }
    238 
    239 void WifiDisplaySource::PlaybackSession::Track::onMessageReceived(
    240         const sp<AMessage> &msg) {
    241     switch (msg->what()) {
    242         case kWhatMediaPullerStopped:
    243         {
    244             mConverter.clear();
    245 
    246             mStarted = false;
    247 
    248             sp<AMessage> notify = mNotify->dup();
    249             notify->setInt32("what", kWhatStopped);
    250             notify->post();
    251 
    252             ALOGI("kWhatStopped %s posted", mIsAudio ? "audio" : "video");
    253             break;
    254         }
    255 
    256         default:
    257             TRESPASS();
    258     }
    259 }
    260 
    261 void WifiDisplaySource::PlaybackSession::Track::queueAccessUnit(
    262         const sp<ABuffer> &accessUnit) {
    263     mQueuedAccessUnits.push_back(accessUnit);
    264 }
    265 
    266 sp<ABuffer> WifiDisplaySource::PlaybackSession::Track::dequeueAccessUnit() {
    267     if (mQueuedAccessUnits.empty()) {
    268         return NULL;
    269     }
    270 
    271     sp<ABuffer> accessUnit = *mQueuedAccessUnits.begin();
    272     CHECK(accessUnit != NULL);
    273 
    274     mQueuedAccessUnits.erase(mQueuedAccessUnits.begin());
    275 
    276     return accessUnit;
    277 }
    278 
    279 void WifiDisplaySource::PlaybackSession::Track::setRepeaterSource(
    280         const sp<RepeaterSource> &source) {
    281     mRepeaterSource = source;
    282 }
    283 
    284 void WifiDisplaySource::PlaybackSession::Track::requestIDRFrame() {
    285     if (mIsAudio) {
    286         return;
    287     }
    288 
    289     if (mRepeaterSource != NULL) {
    290         mRepeaterSource->wakeUp();
    291     }
    292 
    293     mConverter->requestIDRFrame();
    294 }
    295 
    296 bool WifiDisplaySource::PlaybackSession::Track::hasOutputBuffer(
    297         int64_t *timeUs) const {
    298     *timeUs = 0ll;
    299 
    300     if (mQueuedOutputBuffers.empty()) {
    301         return false;
    302     }
    303 
    304     const sp<ABuffer> &outputBuffer = *mQueuedOutputBuffers.begin();
    305 
    306     CHECK(outputBuffer->meta()->findInt64("timeUs", timeUs));
    307 
    308     return true;
    309 }
    310 
    311 void WifiDisplaySource::PlaybackSession::Track::queueOutputBuffer(
    312         const sp<ABuffer> &accessUnit) {
    313     mQueuedOutputBuffers.push_back(accessUnit);
    314     mLastOutputBufferQueuedTimeUs = ALooper::GetNowUs();
    315 }
    316 
    317 sp<ABuffer> WifiDisplaySource::PlaybackSession::Track::dequeueOutputBuffer() {
    318     CHECK(!mQueuedOutputBuffers.empty());
    319 
    320     sp<ABuffer> outputBuffer = *mQueuedOutputBuffers.begin();
    321     mQueuedOutputBuffers.erase(mQueuedOutputBuffers.begin());
    322 
    323     return outputBuffer;
    324 }
    325 
    326 #if SUSPEND_VIDEO_IF_IDLE
    327 bool WifiDisplaySource::PlaybackSession::Track::isSuspended() const {
    328     if (!mQueuedOutputBuffers.empty()) {
    329         return false;
    330     }
    331 
    332     if (mLastOutputBufferQueuedTimeUs < 0ll) {
    333         // We've never seen an output buffer queued, but tracks start
    334         // out live, not suspended.
    335         return false;
    336     }
    337 
    338     // If we've not seen new output data for 60ms or more, we consider
    339     // this track suspended for the time being.
    340     return (ALooper::GetNowUs() - mLastOutputBufferQueuedTimeUs) > 60000ll;
    341 }
    342 #endif
    343 
    344 ////////////////////////////////////////////////////////////////////////////////
    345 
    346 WifiDisplaySource::PlaybackSession::PlaybackSession(
    347         const String16 &opPackageName,
    348         const sp<ANetworkSession> &netSession,
    349         const sp<AMessage> &notify,
    350         const in_addr &interfaceAddr,
    351         const sp<IHDCP> &hdcp,
    352         const char *path)
    353     : mOpPackageName(opPackageName),
    354       mNetSession(netSession),
    355       mNotify(notify),
    356       mInterfaceAddr(interfaceAddr),
    357       mHDCP(hdcp),
    358       mLocalRTPPort(-1),
    359       mWeAreDead(false),
    360       mPaused(false),
    361       mLastLifesignUs(),
    362       mVideoTrackIndex(-1),
    363       mPrevTimeUs(-1ll),
    364       mPullExtractorPending(false),
    365       mPullExtractorGeneration(0),
    366       mFirstSampleTimeRealUs(-1ll),
    367       mFirstSampleTimeUs(-1ll) {
    368     if (path != NULL) {
    369         mMediaPath.setTo(path);
    370     }
    371 }
    372 
    373 status_t WifiDisplaySource::PlaybackSession::init(
    374         const char *clientIP,
    375         int32_t clientRtp,
    376         RTPSender::TransportMode rtpMode,
    377         int32_t clientRtcp,
    378         RTPSender::TransportMode rtcpMode,
    379         bool enableAudio,
    380         bool usePCMAudio,
    381         bool enableVideo,
    382         VideoFormats::ResolutionType videoResolutionType,
    383         size_t videoResolutionIndex,
    384         VideoFormats::ProfileType videoProfileType,
    385         VideoFormats::LevelType videoLevelType) {
    386     sp<AMessage> notify = new AMessage(kWhatMediaSenderNotify, this);
    387     mMediaSender = new MediaSender(mNetSession, notify);
    388     looper()->registerHandler(mMediaSender);
    389 
    390     mMediaSender->setHDCP(mHDCP);
    391 
    392     status_t err = setupPacketizer(
    393             enableAudio,
    394             usePCMAudio,
    395             enableVideo,
    396             videoResolutionType,
    397             videoResolutionIndex,
    398             videoProfileType,
    399             videoLevelType);
    400 
    401     if (err == OK) {
    402         err = mMediaSender->initAsync(
    403                 -1 /* trackIndex */,
    404                 clientIP,
    405                 clientRtp,
    406                 rtpMode,
    407                 clientRtcp,
    408                 rtcpMode,
    409                 &mLocalRTPPort);
    410     }
    411 
    412     if (err != OK) {
    413         mLocalRTPPort = -1;
    414 
    415         looper()->unregisterHandler(mMediaSender->id());
    416         mMediaSender.clear();
    417 
    418         return err;
    419     }
    420 
    421     updateLiveness();
    422 
    423     return OK;
    424 }
    425 
    426 WifiDisplaySource::PlaybackSession::~PlaybackSession() {
    427 }
    428 
    429 int32_t WifiDisplaySource::PlaybackSession::getRTPPort() const {
    430     return mLocalRTPPort;
    431 }
    432 
    433 int64_t WifiDisplaySource::PlaybackSession::getLastLifesignUs() const {
    434     return mLastLifesignUs;
    435 }
    436 
    437 void WifiDisplaySource::PlaybackSession::updateLiveness() {
    438     mLastLifesignUs = ALooper::GetNowUs();
    439 }
    440 
    441 status_t WifiDisplaySource::PlaybackSession::play() {
    442     updateLiveness();
    443 
    444     (new AMessage(kWhatResume, this))->post();
    445 
    446     return OK;
    447 }
    448 
    449 status_t WifiDisplaySource::PlaybackSession::onMediaSenderInitialized() {
    450     for (size_t i = 0; i < mTracks.size(); ++i) {
    451         CHECK_EQ((status_t)OK, mTracks.editValueAt(i)->start());
    452     }
    453 
    454     sp<AMessage> notify = mNotify->dup();
    455     notify->setInt32("what", kWhatSessionEstablished);
    456     notify->post();
    457 
    458     return OK;
    459 }
    460 
    461 status_t WifiDisplaySource::PlaybackSession::pause() {
    462     updateLiveness();
    463 
    464     (new AMessage(kWhatPause, this))->post();
    465 
    466     return OK;
    467 }
    468 
    469 void WifiDisplaySource::PlaybackSession::destroyAsync() {
    470     ALOGI("destroyAsync");
    471 
    472     for (size_t i = 0; i < mTracks.size(); ++i) {
    473         mTracks.valueAt(i)->stopAsync();
    474     }
    475 }
    476 
    477 void WifiDisplaySource::PlaybackSession::onMessageReceived(
    478         const sp<AMessage> &msg) {
    479     switch (msg->what()) {
    480         case kWhatConverterNotify:
    481         {
    482             if (mWeAreDead) {
    483                 ALOGV("dropping msg '%s' because we're dead",
    484                       msg->debugString().c_str());
    485 
    486                 break;
    487             }
    488 
    489             int32_t what;
    490             CHECK(msg->findInt32("what", &what));
    491 
    492             size_t trackIndex;
    493             CHECK(msg->findSize("trackIndex", &trackIndex));
    494 
    495             if (what == Converter::kWhatAccessUnit) {
    496                 sp<ABuffer> accessUnit;
    497                 CHECK(msg->findBuffer("accessUnit", &accessUnit));
    498 
    499                 const sp<Track> &track = mTracks.valueFor(trackIndex);
    500 
    501                 status_t err = mMediaSender->queueAccessUnit(
    502                         track->mediaSenderTrackIndex(),
    503                         accessUnit);
    504 
    505                 if (err != OK) {
    506                     notifySessionDead();
    507                 }
    508                 break;
    509             } else if (what == Converter::kWhatEOS) {
    510                 CHECK_EQ(what, Converter::kWhatEOS);
    511 
    512                 ALOGI("output EOS on track %zu", trackIndex);
    513 
    514                 ssize_t index = mTracks.indexOfKey(trackIndex);
    515                 CHECK_GE(index, 0);
    516 
    517                 const sp<Converter> &converter =
    518                     mTracks.valueAt(index)->converter();
    519                 looper()->unregisterHandler(converter->id());
    520 
    521                 mTracks.removeItemsAt(index);
    522 
    523                 if (mTracks.isEmpty()) {
    524                     ALOGI("Reached EOS");
    525                 }
    526             } else if (what != Converter::kWhatShutdownCompleted) {
    527                 CHECK_EQ(what, Converter::kWhatError);
    528 
    529                 status_t err;
    530                 CHECK(msg->findInt32("err", &err));
    531 
    532                 ALOGE("converter signaled error %d", err);
    533 
    534                 notifySessionDead();
    535             }
    536             break;
    537         }
    538 
    539         case kWhatMediaSenderNotify:
    540         {
    541             int32_t what;
    542             CHECK(msg->findInt32("what", &what));
    543 
    544             if (what == MediaSender::kWhatInitDone) {
    545                 status_t err;
    546                 CHECK(msg->findInt32("err", &err));
    547 
    548                 if (err == OK) {
    549                     onMediaSenderInitialized();
    550                 } else {
    551                     notifySessionDead();
    552                 }
    553             } else if (what == MediaSender::kWhatError) {
    554                 notifySessionDead();
    555             } else if (what == MediaSender::kWhatNetworkStall) {
    556                 size_t numBytesQueued;
    557                 CHECK(msg->findSize("numBytesQueued", &numBytesQueued));
    558 
    559                 if (mVideoTrackIndex >= 0) {
    560                     const sp<Track> &videoTrack =
    561                         mTracks.valueFor(mVideoTrackIndex);
    562 
    563                     sp<Converter> converter = videoTrack->converter();
    564                     if (converter != NULL) {
    565                         converter->dropAFrame();
    566                     }
    567                 }
    568             } else if (what == MediaSender::kWhatInformSender) {
    569                 onSinkFeedback(msg);
    570             } else {
    571                 TRESPASS();
    572             }
    573             break;
    574         }
    575 
    576         case kWhatTrackNotify:
    577         {
    578             int32_t what;
    579             CHECK(msg->findInt32("what", &what));
    580 
    581             size_t trackIndex;
    582             CHECK(msg->findSize("trackIndex", &trackIndex));
    583 
    584             if (what == Track::kWhatStopped) {
    585                 ALOGI("Track %zu stopped", trackIndex);
    586 
    587                 sp<Track> track = mTracks.valueFor(trackIndex);
    588                 looper()->unregisterHandler(track->id());
    589                 mTracks.removeItem(trackIndex);
    590                 track.clear();
    591 
    592                 if (!mTracks.isEmpty()) {
    593                     ALOGI("not all tracks are stopped yet");
    594                     break;
    595                 }
    596 
    597                 looper()->unregisterHandler(mMediaSender->id());
    598                 mMediaSender.clear();
    599 
    600                 sp<AMessage> notify = mNotify->dup();
    601                 notify->setInt32("what", kWhatSessionDestroyed);
    602                 notify->post();
    603             }
    604             break;
    605         }
    606 
    607         case kWhatPause:
    608         {
    609             if (mExtractor != NULL) {
    610                 ++mPullExtractorGeneration;
    611                 mFirstSampleTimeRealUs = -1ll;
    612                 mFirstSampleTimeUs = -1ll;
    613             }
    614 
    615             if (mPaused) {
    616                 break;
    617             }
    618 
    619             for (size_t i = 0; i < mTracks.size(); ++i) {
    620                 mTracks.editValueAt(i)->pause();
    621             }
    622 
    623             mPaused = true;
    624             break;
    625         }
    626 
    627         case kWhatResume:
    628         {
    629             if (mExtractor != NULL) {
    630                 schedulePullExtractor();
    631             }
    632 
    633             if (!mPaused) {
    634                 break;
    635             }
    636 
    637             for (size_t i = 0; i < mTracks.size(); ++i) {
    638                 mTracks.editValueAt(i)->resume();
    639             }
    640 
    641             mPaused = false;
    642             break;
    643         }
    644 
    645         case kWhatPullExtractorSample:
    646         {
    647             int32_t generation;
    648             CHECK(msg->findInt32("generation", &generation));
    649 
    650             if (generation != mPullExtractorGeneration) {
    651                 break;
    652             }
    653 
    654             mPullExtractorPending = false;
    655 
    656             onPullExtractor();
    657             break;
    658         }
    659 
    660         default:
    661             TRESPASS();
    662     }
    663 }
    664 
    665 void WifiDisplaySource::PlaybackSession::onSinkFeedback(const sp<AMessage> &msg) {
    666     int64_t avgLatencyUs;
    667     CHECK(msg->findInt64("avgLatencyUs", &avgLatencyUs));
    668 
    669     int64_t maxLatencyUs;
    670     CHECK(msg->findInt64("maxLatencyUs", &maxLatencyUs));
    671 
    672     ALOGI("sink reports avg. latency of %lld ms (max %lld ms)",
    673           avgLatencyUs / 1000ll,
    674           maxLatencyUs / 1000ll);
    675 
    676     if (mVideoTrackIndex >= 0) {
    677         const sp<Track> &videoTrack = mTracks.valueFor(mVideoTrackIndex);
    678         sp<Converter> converter = videoTrack->converter();
    679 
    680         if (converter != NULL) {
    681             int32_t videoBitrate =
    682                 Converter::GetInt32Property("media.wfd.video-bitrate", -1);
    683 
    684             char val[PROPERTY_VALUE_MAX];
    685             if (videoBitrate < 0
    686                     && property_get("media.wfd.video-bitrate", val, NULL)
    687                     && !strcasecmp("adaptive", val)) {
    688                 videoBitrate = converter->getVideoBitrate();
    689 
    690                 if (avgLatencyUs > 300000ll) {
    691                     videoBitrate *= 0.6;
    692                 } else if (avgLatencyUs < 100000ll) {
    693                     videoBitrate *= 1.1;
    694                 }
    695             }
    696 
    697             if (videoBitrate > 0) {
    698                 if (videoBitrate < 500000) {
    699                     videoBitrate = 500000;
    700                 } else if (videoBitrate > 10000000) {
    701                     videoBitrate = 10000000;
    702                 }
    703 
    704                 if (videoBitrate != converter->getVideoBitrate()) {
    705                     ALOGI("setting video bitrate to %d bps", videoBitrate);
    706 
    707                     converter->setVideoBitrate(videoBitrate);
    708                 }
    709             }
    710         }
    711 
    712         sp<RepeaterSource> repeaterSource = videoTrack->repeaterSource();
    713         if (repeaterSource != NULL) {
    714             double rateHz =
    715                 Converter::GetInt32Property(
    716                         "media.wfd.video-framerate", -1);
    717 
    718             char val[PROPERTY_VALUE_MAX];
    719             if (rateHz < 0.0
    720                     && property_get("media.wfd.video-framerate", val, NULL)
    721                     && !strcasecmp("adaptive", val)) {
    722                  rateHz = repeaterSource->getFrameRate();
    723 
    724                 if (avgLatencyUs > 300000ll) {
    725                     rateHz *= 0.9;
    726                 } else if (avgLatencyUs < 200000ll) {
    727                     rateHz *= 1.1;
    728                 }
    729             }
    730 
    731             if (rateHz > 0) {
    732                 if (rateHz < 5.0) {
    733                     rateHz = 5.0;
    734                 } else if (rateHz > 30.0) {
    735                     rateHz = 30.0;
    736                 }
    737 
    738                 if (rateHz != repeaterSource->getFrameRate()) {
    739                     ALOGI("setting frame rate to %.2f Hz", rateHz);
    740 
    741                     repeaterSource->setFrameRate(rateHz);
    742                 }
    743             }
    744         }
    745     }
    746 }
    747 
    748 status_t WifiDisplaySource::PlaybackSession::setupMediaPacketizer(
    749         bool enableAudio, bool enableVideo) {
    750     mExtractor = new NuMediaExtractor;
    751 
    752     status_t err = mExtractor->setDataSource(
    753             NULL /* httpService */, mMediaPath.c_str());
    754 
    755     if (err != OK) {
    756         return err;
    757     }
    758 
    759     size_t n = mExtractor->countTracks();
    760     bool haveAudio = false;
    761     bool haveVideo = false;
    762     for (size_t i = 0; i < n; ++i) {
    763         sp<AMessage> format;
    764         err = mExtractor->getTrackFormat(i, &format);
    765 
    766         if (err != OK) {
    767             continue;
    768         }
    769 
    770         AString mime;
    771         CHECK(format->findString("mime", &mime));
    772 
    773         bool isAudio = !strncasecmp(mime.c_str(), "audio/", 6);
    774         bool isVideo = !strncasecmp(mime.c_str(), "video/", 6);
    775 
    776         if (isAudio && enableAudio && !haveAudio) {
    777             haveAudio = true;
    778         } else if (isVideo && enableVideo && !haveVideo) {
    779             haveVideo = true;
    780         } else {
    781             continue;
    782         }
    783 
    784         err = mExtractor->selectTrack(i);
    785 
    786         size_t trackIndex = mTracks.size();
    787 
    788         sp<AMessage> notify = new AMessage(kWhatTrackNotify, this);
    789         notify->setSize("trackIndex", trackIndex);
    790 
    791         sp<Track> track = new Track(notify, format);
    792         looper()->registerHandler(track);
    793 
    794         mTracks.add(trackIndex, track);
    795 
    796         mExtractorTrackToInternalTrack.add(i, trackIndex);
    797 
    798         if (isVideo) {
    799             mVideoTrackIndex = trackIndex;
    800         }
    801 
    802         uint32_t flags = MediaSender::FLAG_MANUALLY_PREPEND_SPS_PPS;
    803 
    804         ssize_t mediaSenderTrackIndex =
    805             mMediaSender->addTrack(format, flags);
    806         CHECK_GE(mediaSenderTrackIndex, 0);
    807 
    808         track->setMediaSenderTrackIndex(mediaSenderTrackIndex);
    809 
    810         if ((haveAudio || !enableAudio) && (haveVideo || !enableVideo)) {
    811             break;
    812         }
    813     }
    814 
    815     return OK;
    816 }
    817 
    818 void WifiDisplaySource::PlaybackSession::schedulePullExtractor() {
    819     if (mPullExtractorPending) {
    820         return;
    821     }
    822 
    823     int64_t delayUs = 1000000; // default delay is 1 sec
    824     int64_t sampleTimeUs;
    825     status_t err = mExtractor->getSampleTime(&sampleTimeUs);
    826 
    827     if (err == OK) {
    828         int64_t nowUs = ALooper::GetNowUs();
    829 
    830         if (mFirstSampleTimeRealUs < 0ll) {
    831             mFirstSampleTimeRealUs = nowUs;
    832             mFirstSampleTimeUs = sampleTimeUs;
    833         }
    834 
    835         int64_t whenUs = sampleTimeUs - mFirstSampleTimeUs + mFirstSampleTimeRealUs;
    836         delayUs = whenUs - nowUs;
    837     } else {
    838         ALOGW("could not get sample time (%d)", err);
    839     }
    840 
    841     sp<AMessage> msg = new AMessage(kWhatPullExtractorSample, this);
    842     msg->setInt32("generation", mPullExtractorGeneration);
    843     msg->post(delayUs);
    844 
    845     mPullExtractorPending = true;
    846 }
    847 
    848 void WifiDisplaySource::PlaybackSession::onPullExtractor() {
    849     sp<ABuffer> accessUnit = new ABuffer(1024 * 1024);
    850     status_t err = mExtractor->readSampleData(accessUnit);
    851     if (err != OK) {
    852         // EOS.
    853         return;
    854     }
    855 
    856     int64_t timeUs;
    857     CHECK_EQ((status_t)OK, mExtractor->getSampleTime(&timeUs));
    858 
    859     accessUnit->meta()->setInt64(
    860             "timeUs", mFirstSampleTimeRealUs + timeUs - mFirstSampleTimeUs);
    861 
    862     size_t trackIndex;
    863     CHECK_EQ((status_t)OK, mExtractor->getSampleTrackIndex(&trackIndex));
    864 
    865     sp<AMessage> msg = new AMessage(kWhatConverterNotify, this);
    866 
    867     msg->setSize(
    868             "trackIndex", mExtractorTrackToInternalTrack.valueFor(trackIndex));
    869 
    870     msg->setInt32("what", Converter::kWhatAccessUnit);
    871     msg->setBuffer("accessUnit", accessUnit);
    872     msg->post();
    873 
    874     mExtractor->advance();
    875 
    876     schedulePullExtractor();
    877 }
    878 
    879 status_t WifiDisplaySource::PlaybackSession::setupPacketizer(
    880         bool enableAudio,
    881         bool usePCMAudio,
    882         bool enableVideo,
    883         VideoFormats::ResolutionType videoResolutionType,
    884         size_t videoResolutionIndex,
    885         VideoFormats::ProfileType videoProfileType,
    886         VideoFormats::LevelType videoLevelType) {
    887     CHECK(enableAudio || enableVideo);
    888 
    889     if (!mMediaPath.empty()) {
    890         return setupMediaPacketizer(enableAudio, enableVideo);
    891     }
    892 
    893     if (enableVideo) {
    894         status_t err = addVideoSource(
    895                 videoResolutionType, videoResolutionIndex, videoProfileType,
    896                 videoLevelType);
    897 
    898         if (err != OK) {
    899             return err;
    900         }
    901     }
    902 
    903     if (!enableAudio) {
    904         return OK;
    905     }
    906 
    907     return addAudioSource(usePCMAudio);
    908 }
    909 
    910 status_t WifiDisplaySource::PlaybackSession::addSource(
    911         bool isVideo, const sp<MediaSource> &source, bool isRepeaterSource,
    912         bool usePCMAudio, unsigned profileIdc, unsigned levelIdc,
    913         unsigned constraintSet, size_t *numInputBuffers) {
    914     CHECK(!usePCMAudio || !isVideo);
    915     CHECK(!isRepeaterSource || isVideo);
    916     CHECK(!profileIdc || isVideo);
    917     CHECK(!levelIdc || isVideo);
    918     CHECK(!constraintSet || isVideo);
    919 
    920     sp<ALooper> pullLooper = new ALooper;
    921     pullLooper->setName("pull_looper");
    922 
    923     pullLooper->start(
    924             false /* runOnCallingThread */,
    925             false /* canCallJava */,
    926             PRIORITY_AUDIO);
    927 
    928     sp<ALooper> codecLooper = new ALooper;
    929     codecLooper->setName("codec_looper");
    930 
    931     codecLooper->start(
    932             false /* runOnCallingThread */,
    933             false /* canCallJava */,
    934             PRIORITY_AUDIO);
    935 
    936     size_t trackIndex;
    937 
    938     sp<AMessage> notify;
    939 
    940     trackIndex = mTracks.size();
    941 
    942     sp<AMessage> format;
    943     status_t err = convertMetaDataToMessage(source->getFormat(), &format);
    944     CHECK_EQ(err, (status_t)OK);
    945 
    946     if (isVideo) {
    947         format->setString("mime", MEDIA_MIMETYPE_VIDEO_AVC);
    948         format->setInt32(
    949                 "android._input-metadata-buffer-type", kMetadataBufferTypeANWBuffer);
    950         format->setInt32("android._store-metadata-in-buffers-output", (mHDCP != NULL)
    951                 && (mHDCP->getCaps() & HDCPModule::HDCP_CAPS_ENCRYPT_NATIVE));
    952         format->setInt32(
    953                 "color-format", OMX_COLOR_FormatAndroidOpaque);
    954         format->setInt32("profile-idc", profileIdc);
    955         format->setInt32("level-idc", levelIdc);
    956         format->setInt32("constraint-set", constraintSet);
    957     } else {
    958         if (usePCMAudio) {
    959             format->setInt32("pcm-encoding", kAudioEncodingPcm16bit);
    960             format->setString("mime", MEDIA_MIMETYPE_AUDIO_RAW);
    961         } else {
    962             format->setString("mime", MEDIA_MIMETYPE_AUDIO_AAC);
    963         }
    964     }
    965 
    966     notify = new AMessage(kWhatConverterNotify, this);
    967     notify->setSize("trackIndex", trackIndex);
    968 
    969     sp<Converter> converter = new Converter(notify, codecLooper, format);
    970 
    971     looper()->registerHandler(converter);
    972 
    973     err = converter->init();
    974     if (err != OK) {
    975         ALOGE("%s converter returned err %d", isVideo ? "video" : "audio", err);
    976 
    977         looper()->unregisterHandler(converter->id());
    978         return err;
    979     }
    980 
    981     notify = new AMessage(Converter::kWhatMediaPullerNotify, converter);
    982     notify->setSize("trackIndex", trackIndex);
    983 
    984     sp<MediaPuller> puller = new MediaPuller(source, notify);
    985     pullLooper->registerHandler(puller);
    986 
    987     if (numInputBuffers != NULL) {
    988         *numInputBuffers = converter->getInputBufferCount();
    989     }
    990 
    991     notify = new AMessage(kWhatTrackNotify, this);
    992     notify->setSize("trackIndex", trackIndex);
    993 
    994     sp<Track> track = new Track(
    995             notify, pullLooper, codecLooper, puller, converter);
    996 
    997     if (isRepeaterSource) {
    998         track->setRepeaterSource(static_cast<RepeaterSource *>(source.get()));
    999     }
   1000 
   1001     looper()->registerHandler(track);
   1002 
   1003     mTracks.add(trackIndex, track);
   1004 
   1005     if (isVideo) {
   1006         mVideoTrackIndex = trackIndex;
   1007     }
   1008 
   1009     uint32_t flags = 0;
   1010     if (converter->needToManuallyPrependSPSPPS()) {
   1011         flags |= MediaSender::FLAG_MANUALLY_PREPEND_SPS_PPS;
   1012     }
   1013 
   1014     ssize_t mediaSenderTrackIndex =
   1015         mMediaSender->addTrack(converter->getOutputFormat(), flags);
   1016     CHECK_GE(mediaSenderTrackIndex, 0);
   1017 
   1018     track->setMediaSenderTrackIndex(mediaSenderTrackIndex);
   1019 
   1020     return OK;
   1021 }
   1022 
   1023 status_t WifiDisplaySource::PlaybackSession::addVideoSource(
   1024         VideoFormats::ResolutionType videoResolutionType,
   1025         size_t videoResolutionIndex,
   1026         VideoFormats::ProfileType videoProfileType,
   1027         VideoFormats::LevelType videoLevelType) {
   1028     size_t width, height, framesPerSecond;
   1029     bool interlaced;
   1030     CHECK(VideoFormats::GetConfiguration(
   1031                 videoResolutionType,
   1032                 videoResolutionIndex,
   1033                 &width,
   1034                 &height,
   1035                 &framesPerSecond,
   1036                 &interlaced));
   1037 
   1038     unsigned profileIdc, levelIdc, constraintSet;
   1039     CHECK(VideoFormats::GetProfileLevel(
   1040                 videoProfileType,
   1041                 videoLevelType,
   1042                 &profileIdc,
   1043                 &levelIdc,
   1044                 &constraintSet));
   1045 
   1046     sp<SurfaceMediaSource> source = new SurfaceMediaSource(width, height);
   1047 
   1048     source->setUseAbsoluteTimestamps();
   1049 
   1050     sp<RepeaterSource> videoSource =
   1051         new RepeaterSource(source, framesPerSecond);
   1052 
   1053     size_t numInputBuffers;
   1054     status_t err = addSource(
   1055             true /* isVideo */, videoSource, true /* isRepeaterSource */,
   1056             false /* usePCMAudio */, profileIdc, levelIdc, constraintSet,
   1057             &numInputBuffers);
   1058 
   1059     if (err != OK) {
   1060         return err;
   1061     }
   1062 
   1063     err = source->setMaxAcquiredBufferCount(numInputBuffers);
   1064     CHECK_EQ(err, (status_t)OK);
   1065 
   1066     mProducer = source->getProducer();
   1067 
   1068     return OK;
   1069 }
   1070 
   1071 status_t WifiDisplaySource::PlaybackSession::addAudioSource(bool usePCMAudio) {
   1072     sp<AudioSource> audioSource = new AudioSource(
   1073             AUDIO_SOURCE_REMOTE_SUBMIX,
   1074             mOpPackageName,
   1075             48000 /* sampleRate */,
   1076             2 /* channelCount */);
   1077 
   1078     if (audioSource->initCheck() == OK) {
   1079         return addSource(
   1080                 false /* isVideo */, audioSource, false /* isRepeaterSource */,
   1081                 usePCMAudio, 0 /* profileIdc */, 0 /* levelIdc */,
   1082                 0 /* constraintSet */, NULL /* numInputBuffers */);
   1083     }
   1084 
   1085     ALOGW("Unable to instantiate audio source");
   1086 
   1087     return OK;
   1088 }
   1089 
   1090 sp<IGraphicBufferProducer> WifiDisplaySource::PlaybackSession::getSurfaceTexture() {
   1091     return mProducer;
   1092 }
   1093 
   1094 void WifiDisplaySource::PlaybackSession::requestIDRFrame() {
   1095     for (size_t i = 0; i < mTracks.size(); ++i) {
   1096         const sp<Track> &track = mTracks.valueAt(i);
   1097 
   1098         track->requestIDRFrame();
   1099     }
   1100 }
   1101 
   1102 void WifiDisplaySource::PlaybackSession::notifySessionDead() {
   1103     // Inform WifiDisplaySource of our premature death (wish).
   1104     sp<AMessage> notify = mNotify->dup();
   1105     notify->setInt32("what", kWhatSessionDead);
   1106     notify->post();
   1107 
   1108     mWeAreDead = true;
   1109 }
   1110 
   1111 }  // namespace android
   1112 
   1113