Home | History | Annotate | Download | only in nuplayer
      1 /*
      2  * Copyright 2014 The Android Open Source Project
      3  *
      4  * Licensed under the Apache License, Version 2.0 (the "License");
      5  * you may not use this file except in compliance with the License.
      6  * You may obtain a copy of the License at
      7  *
      8  *      http://www.apache.org/licenses/LICENSE-2.0
      9  *
     10  * Unless required by applicable law or agreed to in writing, software
     11  * distributed under the License is distributed on an "AS IS" BASIS,
     12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
     13  * See the License for the specific language governing permissions and
     14  * limitations under the License.
     15  */
     16 
     17 //#define LOG_NDEBUG 0
     18 #define LOG_TAG "NuPlayerDecoder"
     19 #include <utils/Log.h>
     20 #include <inttypes.h>
     21 
     22 #include <algorithm>
     23 
     24 #include "NuPlayerCCDecoder.h"
     25 #include "NuPlayerDecoder.h"
     26 #include "NuPlayerRenderer.h"
     27 #include "NuPlayerSource.h"
     28 
     29 #include <cutils/properties.h>
     30 #include <media/ICrypto.h>
     31 #include <media/stagefright/foundation/ABuffer.h>
     32 #include <media/stagefright/foundation/ADebug.h>
     33 #include <media/stagefright/foundation/AMessage.h>
     34 #include <media/stagefright/MediaBuffer.h>
     35 #include <media/stagefright/MediaCodec.h>
     36 #include <media/stagefright/MediaDefs.h>
     37 #include <media/stagefright/MediaErrors.h>
     38 
     39 #include <gui/Surface.h>
     40 
     41 #include "avc_utils.h"
     42 #include "ATSParser.h"
     43 
     44 namespace android {
     45 
     46 static float kDisplayRefreshingRate = 60.f; // TODO: get this from the display
     47 
     48 // The default total video frame rate of a stream when that info is not available from
     49 // the source.
     50 static float kDefaultVideoFrameRateTotal = 30.f;
     51 
     52 static inline bool getAudioDeepBufferSetting() {
     53     return property_get_bool("media.stagefright.audio.deep", false /* default_value */);
     54 }
     55 
     56 NuPlayer::Decoder::Decoder(
     57         const sp<AMessage> &notify,
     58         const sp<Source> &source,
     59         pid_t pid,
     60         const sp<Renderer> &renderer,
     61         const sp<Surface> &surface,
     62         const sp<CCDecoder> &ccDecoder)
     63     : DecoderBase(notify),
     64       mSurface(surface),
     65       mSource(source),
     66       mRenderer(renderer),
     67       mCCDecoder(ccDecoder),
     68       mPid(pid),
     69       mSkipRenderingUntilMediaTimeUs(-1ll),
     70       mNumFramesTotal(0ll),
     71       mNumInputFramesDropped(0ll),
     72       mNumOutputFramesDropped(0ll),
     73       mVideoWidth(0),
     74       mVideoHeight(0),
     75       mIsAudio(true),
     76       mIsVideoAVC(false),
     77       mIsSecure(false),
     78       mFormatChangePending(false),
     79       mTimeChangePending(false),
     80       mFrameRateTotal(kDefaultVideoFrameRateTotal),
     81       mPlaybackSpeed(1.0f),
     82       mNumVideoTemporalLayerTotal(1), // decode all layers
     83       mNumVideoTemporalLayerAllowed(1),
     84       mCurrentMaxVideoTemporalLayerId(0),
     85       mResumePending(false),
     86       mComponentName("decoder") {
     87     mCodecLooper = new ALooper;
     88     mCodecLooper->setName("NPDecoder-CL");
     89     mCodecLooper->start(false, false, ANDROID_PRIORITY_AUDIO);
     90     mVideoTemporalLayerAggregateFps[0] = mFrameRateTotal;
     91 }
     92 
     93 NuPlayer::Decoder::~Decoder() {
     94     mCodec->release();
     95     releaseAndResetMediaBuffers();
     96 }
     97 
     98 sp<AMessage> NuPlayer::Decoder::getStats() const {
     99     mStats->setInt64("frames-total", mNumFramesTotal);
    100     mStats->setInt64("frames-dropped-input", mNumInputFramesDropped);
    101     mStats->setInt64("frames-dropped-output", mNumOutputFramesDropped);
    102     return mStats;
    103 }
    104 
    105 status_t NuPlayer::Decoder::setVideoSurface(const sp<Surface> &surface) {
    106     if (surface == NULL || ADebug::isExperimentEnabled("legacy-setsurface")) {
    107         return BAD_VALUE;
    108     }
    109 
    110     sp<AMessage> msg = new AMessage(kWhatSetVideoSurface, this);
    111 
    112     msg->setObject("surface", surface);
    113     sp<AMessage> response;
    114     status_t err = msg->postAndAwaitResponse(&response);
    115     if (err == OK && response != NULL) {
    116         CHECK(response->findInt32("err", &err));
    117     }
    118     return err;
    119 }
    120 
    121 void NuPlayer::Decoder::onMessageReceived(const sp<AMessage> &msg) {
    122     ALOGV("[%s] onMessage: %s", mComponentName.c_str(), msg->debugString().c_str());
    123 
    124     switch (msg->what()) {
    125         case kWhatCodecNotify:
    126         {
    127             int32_t cbID;
    128             CHECK(msg->findInt32("callbackID", &cbID));
    129 
    130             ALOGV("[%s] kWhatCodecNotify: cbID = %d, paused = %d",
    131                     mIsAudio ? "audio" : "video", cbID, mPaused);
    132 
    133             if (mPaused) {
    134                 break;
    135             }
    136 
    137             switch (cbID) {
    138                 case MediaCodec::CB_INPUT_AVAILABLE:
    139                 {
    140                     int32_t index;
    141                     CHECK(msg->findInt32("index", &index));
    142 
    143                     handleAnInputBuffer(index);
    144                     break;
    145                 }
    146 
    147                 case MediaCodec::CB_OUTPUT_AVAILABLE:
    148                 {
    149                     int32_t index;
    150                     size_t offset;
    151                     size_t size;
    152                     int64_t timeUs;
    153                     int32_t flags;
    154 
    155                     CHECK(msg->findInt32("index", &index));
    156                     CHECK(msg->findSize("offset", &offset));
    157                     CHECK(msg->findSize("size", &size));
    158                     CHECK(msg->findInt64("timeUs", &timeUs));
    159                     CHECK(msg->findInt32("flags", &flags));
    160 
    161                     handleAnOutputBuffer(index, offset, size, timeUs, flags);
    162                     break;
    163                 }
    164 
    165                 case MediaCodec::CB_OUTPUT_FORMAT_CHANGED:
    166                 {
    167                     sp<AMessage> format;
    168                     CHECK(msg->findMessage("format", &format));
    169 
    170                     handleOutputFormatChange(format);
    171                     break;
    172                 }
    173 
    174                 case MediaCodec::CB_ERROR:
    175                 {
    176                     status_t err;
    177                     CHECK(msg->findInt32("err", &err));
    178                     ALOGE("Decoder (%s) reported error : 0x%x",
    179                             mIsAudio ? "audio" : "video", err);
    180 
    181                     handleError(err);
    182                     break;
    183                 }
    184 
    185                 default:
    186                 {
    187                     TRESPASS();
    188                     break;
    189                 }
    190             }
    191 
    192             break;
    193         }
    194 
    195         case kWhatRenderBuffer:
    196         {
    197             if (!isStaleReply(msg)) {
    198                 onRenderBuffer(msg);
    199             }
    200             break;
    201         }
    202 
    203         case kWhatSetVideoSurface:
    204         {
    205             sp<AReplyToken> replyID;
    206             CHECK(msg->senderAwaitsResponse(&replyID));
    207 
    208             sp<RefBase> obj;
    209             CHECK(msg->findObject("surface", &obj));
    210             sp<Surface> surface = static_cast<Surface *>(obj.get()); // non-null
    211             int32_t err = INVALID_OPERATION;
    212             // NOTE: in practice mSurface is always non-null, but checking here for completeness
    213             if (mCodec != NULL && mSurface != NULL) {
    214                 // TODO: once AwesomePlayer is removed, remove this automatic connecting
    215                 // to the surface by MediaPlayerService.
    216                 //
    217                 // at this point MediaPlayerService::client has already connected to the
    218                 // surface, which MediaCodec does not expect
    219                 err = native_window_api_disconnect(surface.get(), NATIVE_WINDOW_API_MEDIA);
    220                 if (err == OK) {
    221                     err = mCodec->setSurface(surface);
    222                     ALOGI_IF(err, "codec setSurface returned: %d", err);
    223                     if (err == OK) {
    224                         // reconnect to the old surface as MPS::Client will expect to
    225                         // be able to disconnect from it.
    226                         (void)native_window_api_connect(mSurface.get(), NATIVE_WINDOW_API_MEDIA);
    227                         mSurface = surface;
    228                     }
    229                 }
    230                 if (err != OK) {
    231                     // reconnect to the new surface on error as MPS::Client will expect to
    232                     // be able to disconnect from it.
    233                     (void)native_window_api_connect(surface.get(), NATIVE_WINDOW_API_MEDIA);
    234                 }
    235             }
    236 
    237             sp<AMessage> response = new AMessage;
    238             response->setInt32("err", err);
    239             response->postReply(replyID);
    240             break;
    241         }
    242 
    243         default:
    244             DecoderBase::onMessageReceived(msg);
    245             break;
    246     }
    247 }
    248 
    249 void NuPlayer::Decoder::onConfigure(const sp<AMessage> &format) {
    250     CHECK(mCodec == NULL);
    251 
    252     mFormatChangePending = false;
    253     mTimeChangePending = false;
    254 
    255     ++mBufferGeneration;
    256 
    257     AString mime;
    258     CHECK(format->findString("mime", &mime));
    259 
    260     mIsAudio = !strncasecmp("audio/", mime.c_str(), 6);
    261     mIsVideoAVC = !strcasecmp(MEDIA_MIMETYPE_VIDEO_AVC, mime.c_str());
    262 
    263     mComponentName = mime;
    264     mComponentName.append(" decoder");
    265     ALOGV("[%s] onConfigure (surface=%p)", mComponentName.c_str(), mSurface.get());
    266 
    267     mCodec = MediaCodec::CreateByType(
    268             mCodecLooper, mime.c_str(), false /* encoder */, NULL /* err */, mPid);
    269     int32_t secure = 0;
    270     if (format->findInt32("secure", &secure) && secure != 0) {
    271         if (mCodec != NULL) {
    272             mCodec->getName(&mComponentName);
    273             mComponentName.append(".secure");
    274             mCodec->release();
    275             ALOGI("[%s] creating", mComponentName.c_str());
    276             mCodec = MediaCodec::CreateByComponentName(
    277                     mCodecLooper, mComponentName.c_str(), NULL /* err */, mPid);
    278         }
    279     }
    280     if (mCodec == NULL) {
    281         ALOGE("Failed to create %s%s decoder",
    282                 (secure ? "secure " : ""), mime.c_str());
    283         handleError(UNKNOWN_ERROR);
    284         return;
    285     }
    286     mIsSecure = secure;
    287 
    288     mCodec->getName(&mComponentName);
    289 
    290     status_t err;
    291     if (mSurface != NULL) {
    292         // disconnect from surface as MediaCodec will reconnect
    293         err = native_window_api_disconnect(
    294                 mSurface.get(), NATIVE_WINDOW_API_MEDIA);
    295         // We treat this as a warning, as this is a preparatory step.
    296         // Codec will try to connect to the surface, which is where
    297         // any error signaling will occur.
    298         ALOGW_IF(err != OK, "failed to disconnect from surface: %d", err);
    299     }
    300     err = mCodec->configure(
    301             format, mSurface, NULL /* crypto */, 0 /* flags */);
    302     if (err != OK) {
    303         ALOGE("Failed to configure %s decoder (err=%d)", mComponentName.c_str(), err);
    304         mCodec->release();
    305         mCodec.clear();
    306         handleError(err);
    307         return;
    308     }
    309     rememberCodecSpecificData(format);
    310 
    311     // the following should work in configured state
    312     CHECK_EQ((status_t)OK, mCodec->getOutputFormat(&mOutputFormat));
    313     CHECK_EQ((status_t)OK, mCodec->getInputFormat(&mInputFormat));
    314 
    315     mStats->setString("mime", mime.c_str());
    316     mStats->setString("component-name", mComponentName.c_str());
    317 
    318     if (!mIsAudio) {
    319         int32_t width, height;
    320         if (mOutputFormat->findInt32("width", &width)
    321                 && mOutputFormat->findInt32("height", &height)) {
    322             mStats->setInt32("width", width);
    323             mStats->setInt32("height", height);
    324         }
    325     }
    326 
    327     sp<AMessage> reply = new AMessage(kWhatCodecNotify, this);
    328     mCodec->setCallback(reply);
    329 
    330     err = mCodec->start();
    331     if (err != OK) {
    332         ALOGE("Failed to start %s decoder (err=%d)", mComponentName.c_str(), err);
    333         mCodec->release();
    334         mCodec.clear();
    335         handleError(err);
    336         return;
    337     }
    338 
    339     releaseAndResetMediaBuffers();
    340 
    341     mPaused = false;
    342     mResumePending = false;
    343 }
    344 
    345 void NuPlayer::Decoder::onSetParameters(const sp<AMessage> &params) {
    346     bool needAdjustLayers = false;
    347     float frameRateTotal;
    348     if (params->findFloat("frame-rate-total", &frameRateTotal)
    349             && mFrameRateTotal != frameRateTotal) {
    350         needAdjustLayers = true;
    351         mFrameRateTotal = frameRateTotal;
    352     }
    353 
    354     int32_t numVideoTemporalLayerTotal;
    355     if (params->findInt32("temporal-layer-count", &numVideoTemporalLayerTotal)
    356             && numVideoTemporalLayerTotal >= 0
    357             && numVideoTemporalLayerTotal <= kMaxNumVideoTemporalLayers
    358             && mNumVideoTemporalLayerTotal != numVideoTemporalLayerTotal) {
    359         needAdjustLayers = true;
    360         mNumVideoTemporalLayerTotal = std::max(numVideoTemporalLayerTotal, 1);
    361     }
    362 
    363     if (needAdjustLayers && mNumVideoTemporalLayerTotal > 1) {
    364         // TODO: For now, layer fps is calculated for some specific architectures.
    365         // But it really should be extracted from the stream.
    366         mVideoTemporalLayerAggregateFps[0] =
    367             mFrameRateTotal / (float)(1ll << (mNumVideoTemporalLayerTotal - 1));
    368         for (int32_t i = 1; i < mNumVideoTemporalLayerTotal; ++i) {
    369             mVideoTemporalLayerAggregateFps[i] =
    370                 mFrameRateTotal / (float)(1ll << (mNumVideoTemporalLayerTotal - i))
    371                 + mVideoTemporalLayerAggregateFps[i - 1];
    372         }
    373     }
    374 
    375     float playbackSpeed;
    376     if (params->findFloat("playback-speed", &playbackSpeed)
    377             && mPlaybackSpeed != playbackSpeed) {
    378         needAdjustLayers = true;
    379         mPlaybackSpeed = playbackSpeed;
    380     }
    381 
    382     if (needAdjustLayers) {
    383         float decodeFrameRate = mFrameRateTotal;
    384         // enable temporal layering optimization only if we know the layering depth
    385         if (mNumVideoTemporalLayerTotal > 1) {
    386             int32_t layerId;
    387             for (layerId = 0; layerId < mNumVideoTemporalLayerTotal - 1; ++layerId) {
    388                 if (mVideoTemporalLayerAggregateFps[layerId] * mPlaybackSpeed
    389                         >= kDisplayRefreshingRate * 0.9) {
    390                     break;
    391                 }
    392             }
    393             mNumVideoTemporalLayerAllowed = layerId + 1;
    394             decodeFrameRate = mVideoTemporalLayerAggregateFps[layerId];
    395         }
    396         ALOGV("onSetParameters: allowed layers=%d, decodeFps=%g",
    397                 mNumVideoTemporalLayerAllowed, decodeFrameRate);
    398 
    399         if (mCodec == NULL) {
    400             ALOGW("onSetParameters called before codec is created.");
    401             return;
    402         }
    403 
    404         sp<AMessage> codecParams = new AMessage();
    405         codecParams->setFloat("operating-rate", decodeFrameRate * mPlaybackSpeed);
    406         mCodec->setParameters(codecParams);
    407     }
    408 }
    409 
    410 void NuPlayer::Decoder::onSetRenderer(const sp<Renderer> &renderer) {
    411     bool hadNoRenderer = (mRenderer == NULL);
    412     mRenderer = renderer;
    413     if (hadNoRenderer && mRenderer != NULL) {
    414         // this means that the widevine legacy source is ready
    415         onRequestInputBuffers();
    416     }
    417 }
    418 
    419 void NuPlayer::Decoder::onGetInputBuffers(
    420         Vector<sp<ABuffer> > *dstBuffers) {
    421     CHECK_EQ((status_t)OK, mCodec->getWidevineLegacyBuffers(dstBuffers));
    422 }
    423 
    424 void NuPlayer::Decoder::onResume(bool notifyComplete) {
    425     mPaused = false;
    426 
    427     if (notifyComplete) {
    428         mResumePending = true;
    429     }
    430     mCodec->start();
    431 }
    432 
    433 void NuPlayer::Decoder::doFlush(bool notifyComplete) {
    434     if (mCCDecoder != NULL) {
    435         mCCDecoder->flush();
    436     }
    437 
    438     if (mRenderer != NULL) {
    439         mRenderer->flush(mIsAudio, notifyComplete);
    440         mRenderer->signalTimeDiscontinuity();
    441     }
    442 
    443     status_t err = OK;
    444     if (mCodec != NULL) {
    445         err = mCodec->flush();
    446         mCSDsToSubmit = mCSDsForCurrentFormat; // copy operator
    447         ++mBufferGeneration;
    448     }
    449 
    450     if (err != OK) {
    451         ALOGE("failed to flush %s (err=%d)", mComponentName.c_str(), err);
    452         handleError(err);
    453         // finish with posting kWhatFlushCompleted.
    454         // we attempt to release the buffers even if flush fails.
    455     }
    456     releaseAndResetMediaBuffers();
    457     mPaused = true;
    458 }
    459 
    460 
    461 void NuPlayer::Decoder::onFlush() {
    462     doFlush(true);
    463 
    464     if (isDiscontinuityPending()) {
    465         // This could happen if the client starts seeking/shutdown
    466         // after we queued an EOS for discontinuities.
    467         // We can consider discontinuity handled.
    468         finishHandleDiscontinuity(false /* flushOnTimeChange */);
    469     }
    470 
    471     sp<AMessage> notify = mNotify->dup();
    472     notify->setInt32("what", kWhatFlushCompleted);
    473     notify->post();
    474 }
    475 
    476 void NuPlayer::Decoder::onShutdown(bool notifyComplete) {
    477     status_t err = OK;
    478 
    479     // if there is a pending resume request, notify complete now
    480     notifyResumeCompleteIfNecessary();
    481 
    482     if (mCodec != NULL) {
    483         err = mCodec->release();
    484         mCodec = NULL;
    485         ++mBufferGeneration;
    486 
    487         if (mSurface != NULL) {
    488             // reconnect to surface as MediaCodec disconnected from it
    489             status_t error =
    490                     native_window_api_connect(mSurface.get(), NATIVE_WINDOW_API_MEDIA);
    491             ALOGW_IF(error != NO_ERROR,
    492                     "[%s] failed to connect to native window, error=%d",
    493                     mComponentName.c_str(), error);
    494         }
    495         mComponentName = "decoder";
    496     }
    497 
    498     releaseAndResetMediaBuffers();
    499 
    500     if (err != OK) {
    501         ALOGE("failed to release %s (err=%d)", mComponentName.c_str(), err);
    502         handleError(err);
    503         // finish with posting kWhatShutdownCompleted.
    504     }
    505 
    506     if (notifyComplete) {
    507         sp<AMessage> notify = mNotify->dup();
    508         notify->setInt32("what", kWhatShutdownCompleted);
    509         notify->post();
    510         mPaused = true;
    511     }
    512 }
    513 
    514 /*
    515  * returns true if we should request more data
    516  */
    517 bool NuPlayer::Decoder::doRequestBuffers() {
    518     // mRenderer is only NULL if we have a legacy widevine source that
    519     // is not yet ready. In this case we must not fetch input.
    520     if (isDiscontinuityPending() || mRenderer == NULL) {
    521         return false;
    522     }
    523     status_t err = OK;
    524     while (err == OK && !mDequeuedInputBuffers.empty()) {
    525         size_t bufferIx = *mDequeuedInputBuffers.begin();
    526         sp<AMessage> msg = new AMessage();
    527         msg->setSize("buffer-ix", bufferIx);
    528         err = fetchInputData(msg);
    529         if (err != OK && err != ERROR_END_OF_STREAM) {
    530             // if EOS, need to queue EOS buffer
    531             break;
    532         }
    533         mDequeuedInputBuffers.erase(mDequeuedInputBuffers.begin());
    534 
    535         if (!mPendingInputMessages.empty()
    536                 || !onInputBufferFetched(msg)) {
    537             mPendingInputMessages.push_back(msg);
    538         }
    539     }
    540 
    541     return err == -EWOULDBLOCK
    542             && mSource->feedMoreTSData() == OK;
    543 }
    544 
    545 void NuPlayer::Decoder::handleError(int32_t err)
    546 {
    547     // We cannot immediately release the codec due to buffers still outstanding
    548     // in the renderer.  We signal to the player the error so it can shutdown/release the
    549     // decoder after flushing and increment the generation to discard unnecessary messages.
    550 
    551     ++mBufferGeneration;
    552 
    553     sp<AMessage> notify = mNotify->dup();
    554     notify->setInt32("what", kWhatError);
    555     notify->setInt32("err", err);
    556     notify->post();
    557 }
    558 
    559 bool NuPlayer::Decoder::handleAnInputBuffer(size_t index) {
    560     if (isDiscontinuityPending()) {
    561         return false;
    562     }
    563 
    564     sp<ABuffer> buffer;
    565     mCodec->getInputBuffer(index, &buffer);
    566 
    567     if (buffer == NULL) {
    568         handleError(UNKNOWN_ERROR);
    569         return false;
    570     }
    571 
    572     if (index >= mInputBuffers.size()) {
    573         for (size_t i = mInputBuffers.size(); i <= index; ++i) {
    574             mInputBuffers.add();
    575             mMediaBuffers.add();
    576             mInputBufferIsDequeued.add();
    577             mMediaBuffers.editItemAt(i) = NULL;
    578             mInputBufferIsDequeued.editItemAt(i) = false;
    579         }
    580     }
    581     mInputBuffers.editItemAt(index) = buffer;
    582 
    583     //CHECK_LT(bufferIx, mInputBuffers.size());
    584 
    585     if (mMediaBuffers[index] != NULL) {
    586         mMediaBuffers[index]->release();
    587         mMediaBuffers.editItemAt(index) = NULL;
    588     }
    589     mInputBufferIsDequeued.editItemAt(index) = true;
    590 
    591     if (!mCSDsToSubmit.isEmpty()) {
    592         sp<AMessage> msg = new AMessage();
    593         msg->setSize("buffer-ix", index);
    594 
    595         sp<ABuffer> buffer = mCSDsToSubmit.itemAt(0);
    596         ALOGI("[%s] resubmitting CSD", mComponentName.c_str());
    597         msg->setBuffer("buffer", buffer);
    598         mCSDsToSubmit.removeAt(0);
    599         if (!onInputBufferFetched(msg)) {
    600             handleError(UNKNOWN_ERROR);
    601             return false;
    602         }
    603         return true;
    604     }
    605 
    606     while (!mPendingInputMessages.empty()) {
    607         sp<AMessage> msg = *mPendingInputMessages.begin();
    608         if (!onInputBufferFetched(msg)) {
    609             break;
    610         }
    611         mPendingInputMessages.erase(mPendingInputMessages.begin());
    612     }
    613 
    614     if (!mInputBufferIsDequeued.editItemAt(index)) {
    615         return true;
    616     }
    617 
    618     mDequeuedInputBuffers.push_back(index);
    619 
    620     onRequestInputBuffers();
    621     return true;
    622 }
    623 
    624 bool NuPlayer::Decoder::handleAnOutputBuffer(
    625         size_t index,
    626         size_t offset,
    627         size_t size,
    628         int64_t timeUs,
    629         int32_t flags) {
    630 //    CHECK_LT(bufferIx, mOutputBuffers.size());
    631     sp<ABuffer> buffer;
    632     mCodec->getOutputBuffer(index, &buffer);
    633 
    634     if (index >= mOutputBuffers.size()) {
    635         for (size_t i = mOutputBuffers.size(); i <= index; ++i) {
    636             mOutputBuffers.add();
    637         }
    638     }
    639 
    640     mOutputBuffers.editItemAt(index) = buffer;
    641 
    642     buffer->setRange(offset, size);
    643     buffer->meta()->clear();
    644     buffer->meta()->setInt64("timeUs", timeUs);
    645 
    646     bool eos = flags & MediaCodec::BUFFER_FLAG_EOS;
    647     // we do not expect CODECCONFIG or SYNCFRAME for decoder
    648 
    649     sp<AMessage> reply = new AMessage(kWhatRenderBuffer, this);
    650     reply->setSize("buffer-ix", index);
    651     reply->setInt32("generation", mBufferGeneration);
    652 
    653     if (eos) {
    654         ALOGI("[%s] saw output EOS", mIsAudio ? "audio" : "video");
    655 
    656         buffer->meta()->setInt32("eos", true);
    657         reply->setInt32("eos", true);
    658     } else if (mSkipRenderingUntilMediaTimeUs >= 0) {
    659         if (timeUs < mSkipRenderingUntilMediaTimeUs) {
    660             ALOGV("[%s] dropping buffer at time %lld as requested.",
    661                      mComponentName.c_str(), (long long)timeUs);
    662 
    663             reply->post();
    664             return true;
    665         }
    666 
    667         mSkipRenderingUntilMediaTimeUs = -1;
    668     }
    669 
    670     mNumFramesTotal += !mIsAudio;
    671 
    672     // wait until 1st frame comes out to signal resume complete
    673     notifyResumeCompleteIfNecessary();
    674 
    675     if (mRenderer != NULL) {
    676         // send the buffer to renderer.
    677         mRenderer->queueBuffer(mIsAudio, buffer, reply);
    678         if (eos && !isDiscontinuityPending()) {
    679             mRenderer->queueEOS(mIsAudio, ERROR_END_OF_STREAM);
    680         }
    681     }
    682 
    683     return true;
    684 }
    685 
    686 void NuPlayer::Decoder::handleOutputFormatChange(const sp<AMessage> &format) {
    687     if (!mIsAudio) {
    688         int32_t width, height;
    689         if (format->findInt32("width", &width)
    690                 && format->findInt32("height", &height)) {
    691             mStats->setInt32("width", width);
    692             mStats->setInt32("height", height);
    693         }
    694         sp<AMessage> notify = mNotify->dup();
    695         notify->setInt32("what", kWhatVideoSizeChanged);
    696         notify->setMessage("format", format);
    697         notify->post();
    698     } else if (mRenderer != NULL) {
    699         uint32_t flags;
    700         int64_t durationUs;
    701         bool hasVideo = (mSource->getFormat(false /* audio */) != NULL);
    702         if (getAudioDeepBufferSetting() // override regardless of source duration
    703                 || (!hasVideo
    704                         && mSource->getDuration(&durationUs) == OK
    705                         && durationUs > AUDIO_SINK_MIN_DEEP_BUFFER_DURATION_US)) {
    706             flags = AUDIO_OUTPUT_FLAG_DEEP_BUFFER;
    707         } else {
    708             flags = AUDIO_OUTPUT_FLAG_NONE;
    709         }
    710 
    711         status_t err = mRenderer->openAudioSink(
    712                 format, false /* offloadOnly */, hasVideo, flags, NULL /* isOffloaed */);
    713         if (err != OK) {
    714             handleError(err);
    715         }
    716     }
    717 }
    718 
    719 void NuPlayer::Decoder::releaseAndResetMediaBuffers() {
    720     for (size_t i = 0; i < mMediaBuffers.size(); i++) {
    721         if (mMediaBuffers[i] != NULL) {
    722             mMediaBuffers[i]->release();
    723             mMediaBuffers.editItemAt(i) = NULL;
    724         }
    725     }
    726     mMediaBuffers.resize(mInputBuffers.size());
    727     for (size_t i = 0; i < mMediaBuffers.size(); i++) {
    728         mMediaBuffers.editItemAt(i) = NULL;
    729     }
    730     mInputBufferIsDequeued.clear();
    731     mInputBufferIsDequeued.resize(mInputBuffers.size());
    732     for (size_t i = 0; i < mInputBufferIsDequeued.size(); i++) {
    733         mInputBufferIsDequeued.editItemAt(i) = false;
    734     }
    735 
    736     mPendingInputMessages.clear();
    737     mDequeuedInputBuffers.clear();
    738     mSkipRenderingUntilMediaTimeUs = -1;
    739 }
    740 
    741 void NuPlayer::Decoder::requestCodecNotification() {
    742     if (mCodec != NULL) {
    743         sp<AMessage> reply = new AMessage(kWhatCodecNotify, this);
    744         reply->setInt32("generation", mBufferGeneration);
    745         mCodec->requestActivityNotification(reply);
    746     }
    747 }
    748 
    749 bool NuPlayer::Decoder::isStaleReply(const sp<AMessage> &msg) {
    750     int32_t generation;
    751     CHECK(msg->findInt32("generation", &generation));
    752     return generation != mBufferGeneration;
    753 }
    754 
    755 status_t NuPlayer::Decoder::fetchInputData(sp<AMessage> &reply) {
    756     sp<ABuffer> accessUnit;
    757     bool dropAccessUnit = true;
    758     do {
    759         status_t err = mSource->dequeueAccessUnit(mIsAudio, &accessUnit);
    760 
    761         if (err == -EWOULDBLOCK) {
    762             return err;
    763         } else if (err != OK) {
    764             if (err == INFO_DISCONTINUITY) {
    765                 int32_t type;
    766                 CHECK(accessUnit->meta()->findInt32("discontinuity", &type));
    767 
    768                 bool formatChange =
    769                     (mIsAudio &&
    770                      (type & ATSParser::DISCONTINUITY_AUDIO_FORMAT))
    771                     || (!mIsAudio &&
    772                             (type & ATSParser::DISCONTINUITY_VIDEO_FORMAT));
    773 
    774                 bool timeChange = (type & ATSParser::DISCONTINUITY_TIME) != 0;
    775 
    776                 ALOGI("%s discontinuity (format=%d, time=%d)",
    777                         mIsAudio ? "audio" : "video", formatChange, timeChange);
    778 
    779                 bool seamlessFormatChange = false;
    780                 sp<AMessage> newFormat = mSource->getFormat(mIsAudio);
    781                 if (formatChange) {
    782                     seamlessFormatChange =
    783                         supportsSeamlessFormatChange(newFormat);
    784                     // treat seamless format change separately
    785                     formatChange = !seamlessFormatChange;
    786                 }
    787 
    788                 // For format or time change, return EOS to queue EOS input,
    789                 // then wait for EOS on output.
    790                 if (formatChange /* not seamless */) {
    791                     mFormatChangePending = true;
    792                     err = ERROR_END_OF_STREAM;
    793                 } else if (timeChange) {
    794                     rememberCodecSpecificData(newFormat);
    795                     mTimeChangePending = true;
    796                     err = ERROR_END_OF_STREAM;
    797                 } else if (seamlessFormatChange) {
    798                     // reuse existing decoder and don't flush
    799                     rememberCodecSpecificData(newFormat);
    800                     continue;
    801                 } else {
    802                     // This stream is unaffected by the discontinuity
    803                     return -EWOULDBLOCK;
    804                 }
    805             }
    806 
    807             // reply should only be returned without a buffer set
    808             // when there is an error (including EOS)
    809             CHECK(err != OK);
    810 
    811             reply->setInt32("err", err);
    812             return ERROR_END_OF_STREAM;
    813         }
    814 
    815         dropAccessUnit = false;
    816         if (!mIsAudio && !mIsSecure) {
    817             int32_t layerId = 0;
    818             bool haveLayerId = accessUnit->meta()->findInt32("temporal-layer-id", &layerId);
    819             if (mRenderer->getVideoLateByUs() > 100000ll
    820                     && mIsVideoAVC
    821                     && !IsAVCReferenceFrame(accessUnit)) {
    822                 dropAccessUnit = true;
    823             } else if (haveLayerId && mNumVideoTemporalLayerTotal > 1) {
    824                 // Add only one layer each time.
    825                 if (layerId > mCurrentMaxVideoTemporalLayerId + 1
    826                         || layerId >= mNumVideoTemporalLayerAllowed) {
    827                     dropAccessUnit = true;
    828                     ALOGV("dropping layer(%d), speed=%g, allowed layer count=%d, max layerId=%d",
    829                             layerId, mPlaybackSpeed, mNumVideoTemporalLayerAllowed,
    830                             mCurrentMaxVideoTemporalLayerId);
    831                 } else if (layerId > mCurrentMaxVideoTemporalLayerId) {
    832                     mCurrentMaxVideoTemporalLayerId = layerId;
    833                 } else if (layerId == 0 && mNumVideoTemporalLayerTotal > 1 && IsIDR(accessUnit)) {
    834                     mCurrentMaxVideoTemporalLayerId = mNumVideoTemporalLayerTotal - 1;
    835                 }
    836             }
    837             if (dropAccessUnit) {
    838                 if (layerId <= mCurrentMaxVideoTemporalLayerId && layerId > 0) {
    839                     mCurrentMaxVideoTemporalLayerId = layerId - 1;
    840                 }
    841                 ++mNumInputFramesDropped;
    842             }
    843         }
    844     } while (dropAccessUnit);
    845 
    846     // ALOGV("returned a valid buffer of %s data", mIsAudio ? "mIsAudio" : "video");
    847 #if 0
    848     int64_t mediaTimeUs;
    849     CHECK(accessUnit->meta()->findInt64("timeUs", &mediaTimeUs));
    850     ALOGV("[%s] feeding input buffer at media time %.3f",
    851          mIsAudio ? "audio" : "video",
    852          mediaTimeUs / 1E6);
    853 #endif
    854 
    855     if (mCCDecoder != NULL) {
    856         mCCDecoder->decode(accessUnit);
    857     }
    858 
    859     reply->setBuffer("buffer", accessUnit);
    860 
    861     return OK;
    862 }
    863 
    864 bool NuPlayer::Decoder::onInputBufferFetched(const sp<AMessage> &msg) {
    865     size_t bufferIx;
    866     CHECK(msg->findSize("buffer-ix", &bufferIx));
    867     CHECK_LT(bufferIx, mInputBuffers.size());
    868     sp<ABuffer> codecBuffer = mInputBuffers[bufferIx];
    869 
    870     sp<ABuffer> buffer;
    871     bool hasBuffer = msg->findBuffer("buffer", &buffer);
    872 
    873     // handle widevine classic source - that fills an arbitrary input buffer
    874     MediaBuffer *mediaBuffer = NULL;
    875     if (hasBuffer) {
    876         mediaBuffer = (MediaBuffer *)(buffer->getMediaBufferBase());
    877         if (mediaBuffer != NULL) {
    878             // likely filled another buffer than we requested: adjust buffer index
    879             size_t ix;
    880             for (ix = 0; ix < mInputBuffers.size(); ix++) {
    881                 const sp<ABuffer> &buf = mInputBuffers[ix];
    882                 if (buf->data() == mediaBuffer->data()) {
    883                     // all input buffers are dequeued on start, hence the check
    884                     if (!mInputBufferIsDequeued[ix]) {
    885                         ALOGV("[%s] received MediaBuffer for #%zu instead of #%zu",
    886                                 mComponentName.c_str(), ix, bufferIx);
    887                         mediaBuffer->release();
    888                         return false;
    889                     }
    890 
    891                     // TRICKY: need buffer for the metadata, so instead, set
    892                     // codecBuffer to the same (though incorrect) buffer to
    893                     // avoid a memcpy into the codecBuffer
    894                     codecBuffer = buffer;
    895                     codecBuffer->setRange(
    896                             mediaBuffer->range_offset(),
    897                             mediaBuffer->range_length());
    898                     bufferIx = ix;
    899                     break;
    900                 }
    901             }
    902             CHECK(ix < mInputBuffers.size());
    903         }
    904     }
    905 
    906     if (buffer == NULL /* includes !hasBuffer */) {
    907         int32_t streamErr = ERROR_END_OF_STREAM;
    908         CHECK(msg->findInt32("err", &streamErr) || !hasBuffer);
    909 
    910         CHECK(streamErr != OK);
    911 
    912         // attempt to queue EOS
    913         status_t err = mCodec->queueInputBuffer(
    914                 bufferIx,
    915                 0,
    916                 0,
    917                 0,
    918                 MediaCodec::BUFFER_FLAG_EOS);
    919         if (err == OK) {
    920             mInputBufferIsDequeued.editItemAt(bufferIx) = false;
    921         } else if (streamErr == ERROR_END_OF_STREAM) {
    922             streamErr = err;
    923             // err will not be ERROR_END_OF_STREAM
    924         }
    925 
    926         if (streamErr != ERROR_END_OF_STREAM) {
    927             ALOGE("Stream error for %s (err=%d), EOS %s queued",
    928                     mComponentName.c_str(),
    929                     streamErr,
    930                     err == OK ? "successfully" : "unsuccessfully");
    931             handleError(streamErr);
    932         }
    933     } else {
    934         sp<AMessage> extra;
    935         if (buffer->meta()->findMessage("extra", &extra) && extra != NULL) {
    936             int64_t resumeAtMediaTimeUs;
    937             if (extra->findInt64(
    938                         "resume-at-mediaTimeUs", &resumeAtMediaTimeUs)) {
    939                 ALOGI("[%s] suppressing rendering until %lld us",
    940                         mComponentName.c_str(), (long long)resumeAtMediaTimeUs);
    941                 mSkipRenderingUntilMediaTimeUs = resumeAtMediaTimeUs;
    942             }
    943         }
    944 
    945         int64_t timeUs = 0;
    946         uint32_t flags = 0;
    947         CHECK(buffer->meta()->findInt64("timeUs", &timeUs));
    948 
    949         int32_t eos, csd;
    950         // we do not expect SYNCFRAME for decoder
    951         if (buffer->meta()->findInt32("eos", &eos) && eos) {
    952             flags |= MediaCodec::BUFFER_FLAG_EOS;
    953         } else if (buffer->meta()->findInt32("csd", &csd) && csd) {
    954             flags |= MediaCodec::BUFFER_FLAG_CODECCONFIG;
    955         }
    956 
    957         // copy into codec buffer
    958         if (buffer != codecBuffer) {
    959             if (buffer->size() > codecBuffer->capacity()) {
    960                 handleError(ERROR_BUFFER_TOO_SMALL);
    961                 mDequeuedInputBuffers.push_back(bufferIx);
    962                 return false;
    963             }
    964             codecBuffer->setRange(0, buffer->size());
    965             memcpy(codecBuffer->data(), buffer->data(), buffer->size());
    966         }
    967 
    968         status_t err = mCodec->queueInputBuffer(
    969                         bufferIx,
    970                         codecBuffer->offset(),
    971                         codecBuffer->size(),
    972                         timeUs,
    973                         flags);
    974         if (err != OK) {
    975             if (mediaBuffer != NULL) {
    976                 mediaBuffer->release();
    977             }
    978             ALOGE("Failed to queue input buffer for %s (err=%d)",
    979                     mComponentName.c_str(), err);
    980             handleError(err);
    981         } else {
    982             mInputBufferIsDequeued.editItemAt(bufferIx) = false;
    983             if (mediaBuffer != NULL) {
    984                 CHECK(mMediaBuffers[bufferIx] == NULL);
    985                 mMediaBuffers.editItemAt(bufferIx) = mediaBuffer;
    986             }
    987         }
    988     }
    989     return true;
    990 }
    991 
    992 void NuPlayer::Decoder::onRenderBuffer(const sp<AMessage> &msg) {
    993     status_t err;
    994     int32_t render;
    995     size_t bufferIx;
    996     int32_t eos;
    997     CHECK(msg->findSize("buffer-ix", &bufferIx));
    998 
    999     if (!mIsAudio) {
   1000         int64_t timeUs;
   1001         sp<ABuffer> buffer = mOutputBuffers[bufferIx];
   1002         buffer->meta()->findInt64("timeUs", &timeUs);
   1003 
   1004         if (mCCDecoder != NULL && mCCDecoder->isSelected()) {
   1005             mCCDecoder->display(timeUs);
   1006         }
   1007     }
   1008 
   1009     if (msg->findInt32("render", &render) && render) {
   1010         int64_t timestampNs;
   1011         CHECK(msg->findInt64("timestampNs", &timestampNs));
   1012         err = mCodec->renderOutputBufferAndRelease(bufferIx, timestampNs);
   1013     } else {
   1014         mNumOutputFramesDropped += !mIsAudio;
   1015         err = mCodec->releaseOutputBuffer(bufferIx);
   1016     }
   1017     if (err != OK) {
   1018         ALOGE("failed to release output buffer for %s (err=%d)",
   1019                 mComponentName.c_str(), err);
   1020         handleError(err);
   1021     }
   1022     if (msg->findInt32("eos", &eos) && eos
   1023             && isDiscontinuityPending()) {
   1024         finishHandleDiscontinuity(true /* flushOnTimeChange */);
   1025     }
   1026 }
   1027 
   1028 bool NuPlayer::Decoder::isDiscontinuityPending() const {
   1029     return mFormatChangePending || mTimeChangePending;
   1030 }
   1031 
   1032 void NuPlayer::Decoder::finishHandleDiscontinuity(bool flushOnTimeChange) {
   1033     ALOGV("finishHandleDiscontinuity: format %d, time %d, flush %d",
   1034             mFormatChangePending, mTimeChangePending, flushOnTimeChange);
   1035 
   1036     // If we have format change, pause and wait to be killed;
   1037     // If we have time change only, flush and restart fetching.
   1038 
   1039     if (mFormatChangePending) {
   1040         mPaused = true;
   1041     } else if (mTimeChangePending) {
   1042         if (flushOnTimeChange) {
   1043             doFlush(false /* notifyComplete */);
   1044             signalResume(false /* notifyComplete */);
   1045         }
   1046     }
   1047 
   1048     // Notify NuPlayer to either shutdown decoder, or rescan sources
   1049     sp<AMessage> msg = mNotify->dup();
   1050     msg->setInt32("what", kWhatInputDiscontinuity);
   1051     msg->setInt32("formatChange", mFormatChangePending);
   1052     msg->post();
   1053 
   1054     mFormatChangePending = false;
   1055     mTimeChangePending = false;
   1056 }
   1057 
   1058 bool NuPlayer::Decoder::supportsSeamlessAudioFormatChange(
   1059         const sp<AMessage> &targetFormat) const {
   1060     if (targetFormat == NULL) {
   1061         return true;
   1062     }
   1063 
   1064     AString mime;
   1065     if (!targetFormat->findString("mime", &mime)) {
   1066         return false;
   1067     }
   1068 
   1069     if (!strcasecmp(mime.c_str(), MEDIA_MIMETYPE_AUDIO_AAC)) {
   1070         // field-by-field comparison
   1071         const char * keys[] = { "channel-count", "sample-rate", "is-adts" };
   1072         for (unsigned int i = 0; i < sizeof(keys) / sizeof(keys[0]); i++) {
   1073             int32_t oldVal, newVal;
   1074             if (!mInputFormat->findInt32(keys[i], &oldVal) ||
   1075                     !targetFormat->findInt32(keys[i], &newVal) ||
   1076                     oldVal != newVal) {
   1077                 return false;
   1078             }
   1079         }
   1080 
   1081         sp<ABuffer> oldBuf, newBuf;
   1082         if (mInputFormat->findBuffer("csd-0", &oldBuf) &&
   1083                 targetFormat->findBuffer("csd-0", &newBuf)) {
   1084             if (oldBuf->size() != newBuf->size()) {
   1085                 return false;
   1086             }
   1087             return !memcmp(oldBuf->data(), newBuf->data(), oldBuf->size());
   1088         }
   1089     }
   1090     return false;
   1091 }
   1092 
   1093 bool NuPlayer::Decoder::supportsSeamlessFormatChange(const sp<AMessage> &targetFormat) const {
   1094     if (mInputFormat == NULL) {
   1095         return false;
   1096     }
   1097 
   1098     if (targetFormat == NULL) {
   1099         return true;
   1100     }
   1101 
   1102     AString oldMime, newMime;
   1103     if (!mInputFormat->findString("mime", &oldMime)
   1104             || !targetFormat->findString("mime", &newMime)
   1105             || !(oldMime == newMime)) {
   1106         return false;
   1107     }
   1108 
   1109     bool audio = !strncasecmp(oldMime.c_str(), "audio/", strlen("audio/"));
   1110     bool seamless;
   1111     if (audio) {
   1112         seamless = supportsSeamlessAudioFormatChange(targetFormat);
   1113     } else {
   1114         int32_t isAdaptive;
   1115         seamless = (mCodec != NULL &&
   1116                 mInputFormat->findInt32("adaptive-playback", &isAdaptive) &&
   1117                 isAdaptive);
   1118     }
   1119 
   1120     ALOGV("%s seamless support for %s", seamless ? "yes" : "no", oldMime.c_str());
   1121     return seamless;
   1122 }
   1123 
   1124 void NuPlayer::Decoder::rememberCodecSpecificData(const sp<AMessage> &format) {
   1125     if (format == NULL) {
   1126         return;
   1127     }
   1128     mCSDsForCurrentFormat.clear();
   1129     for (int32_t i = 0; ; ++i) {
   1130         AString tag = "csd-";
   1131         tag.append(i);
   1132         sp<ABuffer> buffer;
   1133         if (!format->findBuffer(tag.c_str(), &buffer)) {
   1134             break;
   1135         }
   1136         mCSDsForCurrentFormat.push(buffer);
   1137     }
   1138 }
   1139 
   1140 void NuPlayer::Decoder::notifyResumeCompleteIfNecessary() {
   1141     if (mResumePending) {
   1142         mResumePending = false;
   1143 
   1144         sp<AMessage> notify = mNotify->dup();
   1145         notify->setInt32("what", kWhatResumeCompleted);
   1146         notify->post();
   1147     }
   1148 }
   1149 
   1150 }  // namespace android
   1151 
   1152