Home | History | Annotate | Download | only in nuplayer 
      1 /*
      2  * Copyright 2014 The Android Open Source Project
      3  *
      4  * Licensed under the Apache License, Version 2.0 (the "License");
      5  * you may not use this file except in compliance with the License.
      6  * You may obtain a copy of the License at
      7  *
      8  *      http://www.apache.org/licenses/LICENSE-2.0
      9  *
     10  * Unless required by applicable law or agreed to in writing, software
     11  * distributed under the License is distributed on an "AS IS" BASIS,
     12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
     13  * See the License for the specific language governing permissions and
     14  * limitations under the License.
     15  */
     16 
     17 //#define LOG_NDEBUG 0
     18 #define LOG_TAG "NuPlayerDecoder"
     19 #include <utils/Log.h>
     20 #include <inttypes.h>
     21 
     22 #include "NuPlayerCCDecoder.h"
     23 #include "NuPlayerDecoder.h"
     24 #include "NuPlayerRenderer.h"
     25 #include "NuPlayerSource.h"
     26 
     27 #include <cutils/properties.h>
     28 #include <media/ICrypto.h>
     29 #include <media/stagefright/foundation/ABuffer.h>
     30 #include <media/stagefright/foundation/ADebug.h>
     31 #include <media/stagefright/foundation/AMessage.h>
     32 #include <media/stagefright/MediaBuffer.h>
     33 #include <media/stagefright/MediaCodec.h>
     34 #include <media/stagefright/MediaDefs.h>
     35 #include <media/stagefright/MediaErrors.h>
     36 
     37 #include <gui/Surface.h>
     38 
     39 #include "avc_utils.h"
     40 #include "ATSParser.h"
     41 
     42 namespace android {
     43 
     44 static inline bool getAudioDeepBufferSetting() {
     45     return property_get_bool("media.stagefright.audio.deep", false /* default_value */);
     46 }
     47 
     48 NuPlayer::Decoder::Decoder(
     49         const sp<AMessage> &notify,
     50         const sp<Source> &source,
     51         pid_t pid,
     52         const sp<Renderer> &renderer,
     53         const sp<Surface> &surface,
     54         const sp<CCDecoder> &ccDecoder)
     55     : DecoderBase(notify),
     56       mSurface(surface),
     57       mSource(source),
     58       mRenderer(renderer),
     59       mCCDecoder(ccDecoder),
     60       mPid(pid),
     61       mSkipRenderingUntilMediaTimeUs(-1ll),
     62       mNumFramesTotal(0ll),
     63       mNumInputFramesDropped(0ll),
     64       mNumOutputFramesDropped(0ll),
     65       mVideoWidth(0),
     66       mVideoHeight(0),
     67       mIsAudio(true),
     68       mIsVideoAVC(false),
     69       mIsSecure(false),
     70       mFormatChangePending(false),
     71       mTimeChangePending(false),
     72       mPaused(true),
     73       mResumePending(false),
     74       mComponentName("decoder") {
     75     mCodecLooper = new ALooper;
     76     mCodecLooper->setName("NPDecoder-CL");
     77     mCodecLooper->start(false, false, ANDROID_PRIORITY_AUDIO);
     78 }
     79 
     80 NuPlayer::Decoder::~Decoder() {
     81     mCodec->release();
     82     releaseAndResetMediaBuffers();
     83 }
     84 
     85 sp<AMessage> NuPlayer::Decoder::getStats() const {
     86     mStats->setInt64("frames-total", mNumFramesTotal);
     87     mStats->setInt64("frames-dropped-input", mNumInputFramesDropped);
     88     mStats->setInt64("frames-dropped-output", mNumOutputFramesDropped);
     89     return mStats;
     90 }
     91 
     92 status_t NuPlayer::Decoder::setVideoSurface(const sp<Surface> &surface) {
     93     if (surface == NULL || ADebug::isExperimentEnabled("legacy-setsurface")) {
     94         return BAD_VALUE;
     95     }
     96 
     97     sp<AMessage> msg = new AMessage(kWhatSetVideoSurface, this);
     98 
     99     msg->setObject("surface", surface);
    100     sp<AMessage> response;
    101     status_t err = msg->postAndAwaitResponse(&response);
    102     if (err == OK && response != NULL) {
    103         CHECK(response->findInt32("err", &err));
    104     }
    105     return err;
    106 }
    107 
    108 void NuPlayer::Decoder::onMessageReceived(const sp<AMessage> &msg) {
    109     ALOGV("[%s] onMessage: %s", mComponentName.c_str(), msg->debugString().c_str());
    110 
    111     switch (msg->what()) {
    112         case kWhatCodecNotify:
    113         {
    114             int32_t cbID;
    115             CHECK(msg->findInt32("callbackID", &cbID));
    116 
    117             ALOGV("[%s] kWhatCodecNotify: cbID = %d, paused = %d",
    118                     mIsAudio ? "audio" : "video", cbID, mPaused);
    119 
    120             if (mPaused) {
    121                 break;
    122             }
    123 
    124             switch (cbID) {
    125                 case MediaCodec::CB_INPUT_AVAILABLE:
    126                 {
    127                     int32_t index;
    128                     CHECK(msg->findInt32("index", &index));
    129 
    130                     handleAnInputBuffer(index);
    131                     break;
    132                 }
    133 
    134                 case MediaCodec::CB_OUTPUT_AVAILABLE:
    135                 {
    136                     int32_t index;
    137                     size_t offset;
    138                     size_t size;
    139                     int64_t timeUs;
    140                     int32_t flags;
    141 
    142                     CHECK(msg->findInt32("index", &index));
    143                     CHECK(msg->findSize("offset", &offset));
    144                     CHECK(msg->findSize("size", &size));
    145                     CHECK(msg->findInt64("timeUs", &timeUs));
    146                     CHECK(msg->findInt32("flags", &flags));
    147 
    148                     handleAnOutputBuffer(index, offset, size, timeUs, flags);
    149                     break;
    150                 }
    151 
    152                 case MediaCodec::CB_OUTPUT_FORMAT_CHANGED:
    153                 {
    154                     sp<AMessage> format;
    155                     CHECK(msg->findMessage("format", &format));
    156 
    157                     handleOutputFormatChange(format);
    158                     break;
    159                 }
    160 
    161                 case MediaCodec::CB_ERROR:
    162                 {
    163                     status_t err;
    164                     CHECK(msg->findInt32("err", &err));
    165                     ALOGE("Decoder (%s) reported error : 0x%x",
    166                             mIsAudio ? "audio" : "video", err);
    167 
    168                     handleError(err);
    169                     break;
    170                 }
    171 
    172                 default:
    173                 {
    174                     TRESPASS();
    175                     break;
    176                 }
    177             }
    178 
    179             break;
    180         }
    181 
    182         case kWhatRenderBuffer:
    183         {
    184             if (!isStaleReply(msg)) {
    185                 onRenderBuffer(msg);
    186             }
    187             break;
    188         }
    189 
    190         case kWhatSetVideoSurface:
    191         {
    192             sp<AReplyToken> replyID;
    193             CHECK(msg->senderAwaitsResponse(&replyID));
    194 
    195             sp<RefBase> obj;
    196             CHECK(msg->findObject("surface", &obj));
    197             sp<Surface> surface = static_cast<Surface *>(obj.get()); // non-null
    198             int32_t err = INVALID_OPERATION;
    199             // NOTE: in practice mSurface is always non-null, but checking here for completeness
    200             if (mCodec != NULL && mSurface != NULL) {
    201                 // TODO: once AwesomePlayer is removed, remove this automatic connecting
    202                 // to the surface by MediaPlayerService.
    203                 //
    204                 // at this point MediaPlayerService::client has already connected to the
    205                 // surface, which MediaCodec does not expect
    206                 err = native_window_api_disconnect(surface.get(), NATIVE_WINDOW_API_MEDIA);
    207                 if (err == OK) {
    208                     err = mCodec->setSurface(surface);
    209                     ALOGI_IF(err, "codec setSurface returned: %d", err);
    210                     if (err == OK) {
    211                         // reconnect to the old surface as MPS::Client will expect to
    212                         // be able to disconnect from it.
    213                         (void)native_window_api_connect(mSurface.get(), NATIVE_WINDOW_API_MEDIA);
    214                         mSurface = surface;
    215                     }
    216                 }
    217                 if (err != OK) {
    218                     // reconnect to the new surface on error as MPS::Client will expect to
    219                     // be able to disconnect from it.
    220                     (void)native_window_api_connect(surface.get(), NATIVE_WINDOW_API_MEDIA);
    221                 }
    222             }
    223 
    224             sp<AMessage> response = new AMessage;
    225             response->setInt32("err", err);
    226             response->postReply(replyID);
    227             break;
    228         }
    229 
    230         default:
    231             DecoderBase::onMessageReceived(msg);
    232             break;
    233     }
    234 }
    235 
    236 void NuPlayer::Decoder::onConfigure(const sp<AMessage> &format) {
    237     CHECK(mCodec == NULL);
    238 
    239     mFormatChangePending = false;
    240     mTimeChangePending = false;
    241 
    242     ++mBufferGeneration;
    243 
    244     AString mime;
    245     CHECK(format->findString("mime", &mime));
    246 
    247     mIsAudio = !strncasecmp("audio/", mime.c_str(), 6);
    248     mIsVideoAVC = !strcasecmp(MEDIA_MIMETYPE_VIDEO_AVC, mime.c_str());
    249 
    250     mComponentName = mime;
    251     mComponentName.append(" decoder");
    252     ALOGV("[%s] onConfigure (surface=%p)", mComponentName.c_str(), mSurface.get());
    253 
    254     mCodec = MediaCodec::CreateByType(
    255             mCodecLooper, mime.c_str(), false /* encoder */, NULL /* err */, mPid);
    256     int32_t secure = 0;
    257     if (format->findInt32("secure", &secure) && secure != 0) {
    258         if (mCodec != NULL) {
    259             mCodec->getName(&mComponentName);
    260             mComponentName.append(".secure");
    261             mCodec->release();
    262             ALOGI("[%s] creating", mComponentName.c_str());
    263             mCodec = MediaCodec::CreateByComponentName(
    264                     mCodecLooper, mComponentName.c_str(), NULL /* err */, mPid);
    265         }
    266     }
    267     if (mCodec == NULL) {
    268         ALOGE("Failed to create %s%s decoder",
    269                 (secure ? "secure " : ""), mime.c_str());
    270         handleError(UNKNOWN_ERROR);
    271         return;
    272     }
    273     mIsSecure = secure;
    274 
    275     mCodec->getName(&mComponentName);
    276 
    277     status_t err;
    278     if (mSurface != NULL) {
    279         // disconnect from surface as MediaCodec will reconnect
    280         err = native_window_api_disconnect(
    281                 mSurface.get(), NATIVE_WINDOW_API_MEDIA);
    282         // We treat this as a warning, as this is a preparatory step.
    283         // Codec will try to connect to the surface, which is where
    284         // any error signaling will occur.
    285         ALOGW_IF(err != OK, "failed to disconnect from surface: %d", err);
    286     }
    287     err = mCodec->configure(
    288             format, mSurface, NULL /* crypto */, 0 /* flags */);
    289     if (err != OK) {
    290         ALOGE("Failed to configure %s decoder (err=%d)", mComponentName.c_str(), err);
    291         mCodec->release();
    292         mCodec.clear();
    293         handleError(err);
    294         return;
    295     }
    296     rememberCodecSpecificData(format);
    297 
    298     // the following should work in configured state
    299     CHECK_EQ((status_t)OK, mCodec->getOutputFormat(&mOutputFormat));
    300     CHECK_EQ((status_t)OK, mCodec->getInputFormat(&mInputFormat));
    301 
    302     mStats->setString("mime", mime.c_str());
    303     mStats->setString("component-name", mComponentName.c_str());
    304 
    305     if (!mIsAudio) {
    306         int32_t width, height;
    307         if (mOutputFormat->findInt32("width", &width)
    308                 && mOutputFormat->findInt32("height", &height)) {
    309             mStats->setInt32("width", width);
    310             mStats->setInt32("height", height);
    311         }
    312     }
    313 
    314     sp<AMessage> reply = new AMessage(kWhatCodecNotify, this);
    315     mCodec->setCallback(reply);
    316 
    317     err = mCodec->start();
    318     if (err != OK) {
    319         ALOGE("Failed to start %s decoder (err=%d)", mComponentName.c_str(), err);
    320         mCodec->release();
    321         mCodec.clear();
    322         handleError(err);
    323         return;
    324     }
    325 
    326     releaseAndResetMediaBuffers();
    327 
    328     mPaused = false;
    329     mResumePending = false;
    330 }
    331 
    332 void NuPlayer::Decoder::onSetParameters(const sp<AMessage> &params) {
    333     if (mCodec == NULL) {
    334         ALOGW("onSetParameters called before codec is created.");
    335         return;
    336     }
    337     mCodec->setParameters(params);
    338 }
    339 
    340 void NuPlayer::Decoder::onSetRenderer(const sp<Renderer> &renderer) {
    341     bool hadNoRenderer = (mRenderer == NULL);
    342     mRenderer = renderer;
    343     if (hadNoRenderer && mRenderer != NULL) {
    344         // this means that the widevine legacy source is ready
    345         onRequestInputBuffers();
    346     }
    347 }
    348 
    349 void NuPlayer::Decoder::onGetInputBuffers(
    350         Vector<sp<ABuffer> > *dstBuffers) {
    351     CHECK_EQ((status_t)OK, mCodec->getWidevineLegacyBuffers(dstBuffers));
    352 }
    353 
    354 void NuPlayer::Decoder::onResume(bool notifyComplete) {
    355     mPaused = false;
    356 
    357     if (notifyComplete) {
    358         mResumePending = true;
    359     }
    360     mCodec->start();
    361 }
    362 
    363 void NuPlayer::Decoder::doFlush(bool notifyComplete) {
    364     if (mCCDecoder != NULL) {
    365         mCCDecoder->flush();
    366     }
    367 
    368     if (mRenderer != NULL) {
    369         mRenderer->flush(mIsAudio, notifyComplete);
    370         mRenderer->signalTimeDiscontinuity();
    371     }
    372 
    373     status_t err = OK;
    374     if (mCodec != NULL) {
    375         err = mCodec->flush();
    376         mCSDsToSubmit = mCSDsForCurrentFormat; // copy operator
    377         ++mBufferGeneration;
    378     }
    379 
    380     if (err != OK) {
    381         ALOGE("failed to flush %s (err=%d)", mComponentName.c_str(), err);
    382         handleError(err);
    383         // finish with posting kWhatFlushCompleted.
    384         // we attempt to release the buffers even if flush fails.
    385     }
    386     releaseAndResetMediaBuffers();
    387     mPaused = true;
    388 }
    389 
    390 
    391 void NuPlayer::Decoder::onFlush() {
    392     doFlush(true);
    393 
    394     if (isDiscontinuityPending()) {
    395         // This could happen if the client starts seeking/shutdown
    396         // after we queued an EOS for discontinuities.
    397         // We can consider discontinuity handled.
    398         finishHandleDiscontinuity(false /* flushOnTimeChange */);
    399     }
    400 
    401     sp<AMessage> notify = mNotify->dup();
    402     notify->setInt32("what", kWhatFlushCompleted);
    403     notify->post();
    404 }
    405 
    406 void NuPlayer::Decoder::onShutdown(bool notifyComplete) {
    407     status_t err = OK;
    408 
    409     // if there is a pending resume request, notify complete now
    410     notifyResumeCompleteIfNecessary();
    411 
    412     if (mCodec != NULL) {
    413         err = mCodec->release();
    414         mCodec = NULL;
    415         ++mBufferGeneration;
    416 
    417         if (mSurface != NULL) {
    418             // reconnect to surface as MediaCodec disconnected from it
    419             status_t error =
    420                     native_window_api_connect(mSurface.get(), NATIVE_WINDOW_API_MEDIA);
    421             ALOGW_IF(error != NO_ERROR,
    422                     "[%s] failed to connect to native window, error=%d",
    423                     mComponentName.c_str(), error);
    424         }
    425         mComponentName = "decoder";
    426     }
    427 
    428     releaseAndResetMediaBuffers();
    429 
    430     if (err != OK) {
    431         ALOGE("failed to release %s (err=%d)", mComponentName.c_str(), err);
    432         handleError(err);
    433         // finish with posting kWhatShutdownCompleted.
    434     }
    435 
    436     if (notifyComplete) {
    437         sp<AMessage> notify = mNotify->dup();
    438         notify->setInt32("what", kWhatShutdownCompleted);
    439         notify->post();
    440         mPaused = true;
    441     }
    442 }
    443 
    444 /*
    445  * returns true if we should request more data
    446  */
    447 bool NuPlayer::Decoder::doRequestBuffers() {
    448     // mRenderer is only NULL if we have a legacy widevine source that
    449     // is not yet ready. In this case we must not fetch input.
    450     if (isDiscontinuityPending() || mRenderer == NULL) {
    451         return false;
    452     }
    453     status_t err = OK;
    454     while (err == OK && !mDequeuedInputBuffers.empty()) {
    455         size_t bufferIx = *mDequeuedInputBuffers.begin();
    456         sp<AMessage> msg = new AMessage();
    457         msg->setSize("buffer-ix", bufferIx);
    458         err = fetchInputData(msg);
    459         if (err != OK && err != ERROR_END_OF_STREAM) {
    460             // if EOS, need to queue EOS buffer
    461             break;
    462         }
    463         mDequeuedInputBuffers.erase(mDequeuedInputBuffers.begin());
    464 
    465         if (!mPendingInputMessages.empty()
    466                 || !onInputBufferFetched(msg)) {
    467             mPendingInputMessages.push_back(msg);
    468         }
    469     }
    470 
    471     return err == -EWOULDBLOCK
    472             && mSource->feedMoreTSData() == OK;
    473 }
    474 
    475 void NuPlayer::Decoder::handleError(int32_t err)
    476 {
    477     // We cannot immediately release the codec due to buffers still outstanding
    478     // in the renderer.  We signal to the player the error so it can shutdown/release the
    479     // decoder after flushing and increment the generation to discard unnecessary messages.
    480 
    481     ++mBufferGeneration;
    482 
    483     sp<AMessage> notify = mNotify->dup();
    484     notify->setInt32("what", kWhatError);
    485     notify->setInt32("err", err);
    486     notify->post();
    487 }
    488 
    489 bool NuPlayer::Decoder::handleAnInputBuffer(size_t index) {
    490     if (isDiscontinuityPending()) {
    491         return false;
    492     }
    493 
    494     sp<ABuffer> buffer;
    495     mCodec->getInputBuffer(index, &buffer);
    496 
    497     if (buffer == NULL) {
    498         handleError(UNKNOWN_ERROR);
    499         return false;
    500     }
    501 
    502     if (index >= mInputBuffers.size()) {
    503         for (size_t i = mInputBuffers.size(); i <= index; ++i) {
    504             mInputBuffers.add();
    505             mMediaBuffers.add();
    506             mInputBufferIsDequeued.add();
    507             mMediaBuffers.editItemAt(i) = NULL;
    508             mInputBufferIsDequeued.editItemAt(i) = false;
    509         }
    510     }
    511     mInputBuffers.editItemAt(index) = buffer;
    512 
    513     //CHECK_LT(bufferIx, mInputBuffers.size());
    514 
    515     if (mMediaBuffers[index] != NULL) {
    516         mMediaBuffers[index]->release();
    517         mMediaBuffers.editItemAt(index) = NULL;
    518     }
    519     mInputBufferIsDequeued.editItemAt(index) = true;
    520 
    521     if (!mCSDsToSubmit.isEmpty()) {
    522         sp<AMessage> msg = new AMessage();
    523         msg->setSize("buffer-ix", index);
    524 
    525         sp<ABuffer> buffer = mCSDsToSubmit.itemAt(0);
    526         ALOGI("[%s] resubmitting CSD", mComponentName.c_str());
    527         msg->setBuffer("buffer", buffer);
    528         mCSDsToSubmit.removeAt(0);
    529         CHECK(onInputBufferFetched(msg));
    530         return true;
    531     }
    532 
    533     while (!mPendingInputMessages.empty()) {
    534         sp<AMessage> msg = *mPendingInputMessages.begin();
    535         if (!onInputBufferFetched(msg)) {
    536             break;
    537         }
    538         mPendingInputMessages.erase(mPendingInputMessages.begin());
    539     }
    540 
    541     if (!mInputBufferIsDequeued.editItemAt(index)) {
    542         return true;
    543     }
    544 
    545     mDequeuedInputBuffers.push_back(index);
    546 
    547     onRequestInputBuffers();
    548     return true;
    549 }
    550 
    551 bool NuPlayer::Decoder::handleAnOutputBuffer(
    552         size_t index,
    553         size_t offset,
    554         size_t size,
    555         int64_t timeUs,
    556         int32_t flags) {
    557 //    CHECK_LT(bufferIx, mOutputBuffers.size());
    558     sp<ABuffer> buffer;
    559     mCodec->getOutputBuffer(index, &buffer);
    560 
    561     if (index >= mOutputBuffers.size()) {
    562         for (size_t i = mOutputBuffers.size(); i <= index; ++i) {
    563             mOutputBuffers.add();
    564         }
    565     }
    566 
    567     mOutputBuffers.editItemAt(index) = buffer;
    568 
    569     buffer->setRange(offset, size);
    570     buffer->meta()->clear();
    571     buffer->meta()->setInt64("timeUs", timeUs);
    572 
    573     bool eos = flags & MediaCodec::BUFFER_FLAG_EOS;
    574     // we do not expect CODECCONFIG or SYNCFRAME for decoder
    575 
    576     sp<AMessage> reply = new AMessage(kWhatRenderBuffer, this);
    577     reply->setSize("buffer-ix", index);
    578     reply->setInt32("generation", mBufferGeneration);
    579 
    580     if (eos) {
    581         ALOGI("[%s] saw output EOS", mIsAudio ? "audio" : "video");
    582 
    583         buffer->meta()->setInt32("eos", true);
    584         reply->setInt32("eos", true);
    585     } else if (mSkipRenderingUntilMediaTimeUs >= 0) {
    586         if (timeUs < mSkipRenderingUntilMediaTimeUs) {
    587             ALOGV("[%s] dropping buffer at time %lld as requested.",
    588                      mComponentName.c_str(), (long long)timeUs);
    589 
    590             reply->post();
    591             return true;
    592         }
    593 
    594         mSkipRenderingUntilMediaTimeUs = -1;
    595     }
    596 
    597     mNumFramesTotal += !mIsAudio;
    598 
    599     // wait until 1st frame comes out to signal resume complete
    600     notifyResumeCompleteIfNecessary();
    601 
    602     if (mRenderer != NULL) {
    603         // send the buffer to renderer.
    604         mRenderer->queueBuffer(mIsAudio, buffer, reply);
    605         if (eos && !isDiscontinuityPending()) {
    606             mRenderer->queueEOS(mIsAudio, ERROR_END_OF_STREAM);
    607         }
    608     }
    609 
    610     return true;
    611 }
    612 
    613 void NuPlayer::Decoder::handleOutputFormatChange(const sp<AMessage> &format) {
    614     if (!mIsAudio) {
    615         int32_t width, height;
    616         if (format->findInt32("width", &width)
    617                 && format->findInt32("height", &height)) {
    618             mStats->setInt32("width", width);
    619             mStats->setInt32("height", height);
    620         }
    621         sp<AMessage> notify = mNotify->dup();
    622         notify->setInt32("what", kWhatVideoSizeChanged);
    623         notify->setMessage("format", format);
    624         notify->post();
    625     } else if (mRenderer != NULL) {
    626         uint32_t flags;
    627         int64_t durationUs;
    628         bool hasVideo = (mSource->getFormat(false /* audio */) != NULL);
    629         if (getAudioDeepBufferSetting() // override regardless of source duration
    630                 || (!hasVideo
    631                         && mSource->getDuration(&durationUs) == OK
    632                         && durationUs > AUDIO_SINK_MIN_DEEP_BUFFER_DURATION_US)) {
    633             flags = AUDIO_OUTPUT_FLAG_DEEP_BUFFER;
    634         } else {
    635             flags = AUDIO_OUTPUT_FLAG_NONE;
    636         }
    637 
    638         mRenderer->openAudioSink(
    639                 format, false /* offloadOnly */, hasVideo, flags, NULL /* isOffloaed */);
    640     }
    641 }
    642 
    643 void NuPlayer::Decoder::releaseAndResetMediaBuffers() {
    644     for (size_t i = 0; i < mMediaBuffers.size(); i++) {
    645         if (mMediaBuffers[i] != NULL) {
    646             mMediaBuffers[i]->release();
    647             mMediaBuffers.editItemAt(i) = NULL;
    648         }
    649     }
    650     mMediaBuffers.resize(mInputBuffers.size());
    651     for (size_t i = 0; i < mMediaBuffers.size(); i++) {
    652         mMediaBuffers.editItemAt(i) = NULL;
    653     }
    654     mInputBufferIsDequeued.clear();
    655     mInputBufferIsDequeued.resize(mInputBuffers.size());
    656     for (size_t i = 0; i < mInputBufferIsDequeued.size(); i++) {
    657         mInputBufferIsDequeued.editItemAt(i) = false;
    658     }
    659 
    660     mPendingInputMessages.clear();
    661     mDequeuedInputBuffers.clear();
    662     mSkipRenderingUntilMediaTimeUs = -1;
    663 }
    664 
    665 void NuPlayer::Decoder::requestCodecNotification() {
    666     if (mCodec != NULL) {
    667         sp<AMessage> reply = new AMessage(kWhatCodecNotify, this);
    668         reply->setInt32("generation", mBufferGeneration);
    669         mCodec->requestActivityNotification(reply);
    670     }
    671 }
    672 
    673 bool NuPlayer::Decoder::isStaleReply(const sp<AMessage> &msg) {
    674     int32_t generation;
    675     CHECK(msg->findInt32("generation", &generation));
    676     return generation != mBufferGeneration;
    677 }
    678 
    679 status_t NuPlayer::Decoder::fetchInputData(sp<AMessage> &reply) {
    680     sp<ABuffer> accessUnit;
    681     bool dropAccessUnit;
    682     do {
    683         status_t err = mSource->dequeueAccessUnit(mIsAudio, &accessUnit);
    684 
    685         if (err == -EWOULDBLOCK) {
    686             return err;
    687         } else if (err != OK) {
    688             if (err == INFO_DISCONTINUITY) {
    689                 int32_t type;
    690                 CHECK(accessUnit->meta()->findInt32("discontinuity", &type));
    691 
    692                 bool formatChange =
    693                     (mIsAudio &&
    694                      (type & ATSParser::DISCONTINUITY_AUDIO_FORMAT))
    695                     || (!mIsAudio &&
    696                             (type & ATSParser::DISCONTINUITY_VIDEO_FORMAT));
    697 
    698                 bool timeChange = (type & ATSParser::DISCONTINUITY_TIME) != 0;
    699 
    700                 ALOGI("%s discontinuity (format=%d, time=%d)",
    701                         mIsAudio ? "audio" : "video", formatChange, timeChange);
    702 
    703                 bool seamlessFormatChange = false;
    704                 sp<AMessage> newFormat = mSource->getFormat(mIsAudio);
    705                 if (formatChange) {
    706                     seamlessFormatChange =
    707                         supportsSeamlessFormatChange(newFormat);
    708                     // treat seamless format change separately
    709                     formatChange = !seamlessFormatChange;
    710                 }
    711 
    712                 // For format or time change, return EOS to queue EOS input,
    713                 // then wait for EOS on output.
    714                 if (formatChange /* not seamless */) {
    715                     mFormatChangePending = true;
    716                     err = ERROR_END_OF_STREAM;
    717                 } else if (timeChange) {
    718                     rememberCodecSpecificData(newFormat);
    719                     mTimeChangePending = true;
    720                     err = ERROR_END_OF_STREAM;
    721                 } else if (seamlessFormatChange) {
    722                     // reuse existing decoder and don't flush
    723                     rememberCodecSpecificData(newFormat);
    724                     continue;
    725                 } else {
    726                     // This stream is unaffected by the discontinuity
    727                     return -EWOULDBLOCK;
    728                 }
    729             }
    730 
    731             // reply should only be returned without a buffer set
    732             // when there is an error (including EOS)
    733             CHECK(err != OK);
    734 
    735             reply->setInt32("err", err);
    736             return ERROR_END_OF_STREAM;
    737         }
    738 
    739         dropAccessUnit = false;
    740         if (!mIsAudio
    741                 && !mIsSecure
    742                 && mRenderer->getVideoLateByUs() > 100000ll
    743                 && mIsVideoAVC
    744                 && !IsAVCReferenceFrame(accessUnit)) {
    745             dropAccessUnit = true;
    746             ++mNumInputFramesDropped;
    747         }
    748     } while (dropAccessUnit);
    749 
    750     // ALOGV("returned a valid buffer of %s data", mIsAudio ? "mIsAudio" : "video");
    751 #if 0
    752     int64_t mediaTimeUs;
    753     CHECK(accessUnit->meta()->findInt64("timeUs", &mediaTimeUs));
    754     ALOGV("[%s] feeding input buffer at media time %.3f",
    755          mIsAudio ? "audio" : "video",
    756          mediaTimeUs / 1E6);
    757 #endif
    758 
    759     if (mCCDecoder != NULL) {
    760         mCCDecoder->decode(accessUnit);
    761     }
    762 
    763     reply->setBuffer("buffer", accessUnit);
    764 
    765     return OK;
    766 }
    767 
    768 bool NuPlayer::Decoder::onInputBufferFetched(const sp<AMessage> &msg) {
    769     size_t bufferIx;
    770     CHECK(msg->findSize("buffer-ix", &bufferIx));
    771     CHECK_LT(bufferIx, mInputBuffers.size());
    772     sp<ABuffer> codecBuffer = mInputBuffers[bufferIx];
    773 
    774     sp<ABuffer> buffer;
    775     bool hasBuffer = msg->findBuffer("buffer", &buffer);
    776 
    777     // handle widevine classic source - that fills an arbitrary input buffer
    778     MediaBuffer *mediaBuffer = NULL;
    779     if (hasBuffer) {
    780         mediaBuffer = (MediaBuffer *)(buffer->getMediaBufferBase());
    781         if (mediaBuffer != NULL) {
    782             // likely filled another buffer than we requested: adjust buffer index
    783             size_t ix;
    784             for (ix = 0; ix < mInputBuffers.size(); ix++) {
    785                 const sp<ABuffer> &buf = mInputBuffers[ix];
    786                 if (buf->data() == mediaBuffer->data()) {
    787                     // all input buffers are dequeued on start, hence the check
    788                     if (!mInputBufferIsDequeued[ix]) {
    789                         ALOGV("[%s] received MediaBuffer for #%zu instead of #%zu",
    790                                 mComponentName.c_str(), ix, bufferIx);
    791                         mediaBuffer->release();
    792                         return false;
    793                     }
    794 
    795                     // TRICKY: need buffer for the metadata, so instead, set
    796                     // codecBuffer to the same (though incorrect) buffer to
    797                     // avoid a memcpy into the codecBuffer
    798                     codecBuffer = buffer;
    799                     codecBuffer->setRange(
    800                             mediaBuffer->range_offset(),
    801                             mediaBuffer->range_length());
    802                     bufferIx = ix;
    803                     break;
    804                 }
    805             }
    806             CHECK(ix < mInputBuffers.size());
    807         }
    808     }
    809 
    810     if (buffer == NULL /* includes !hasBuffer */) {
    811         int32_t streamErr = ERROR_END_OF_STREAM;
    812         CHECK(msg->findInt32("err", &streamErr) || !hasBuffer);
    813 
    814         CHECK(streamErr != OK);
    815 
    816         // attempt to queue EOS
    817         status_t err = mCodec->queueInputBuffer(
    818                 bufferIx,
    819                 0,
    820                 0,
    821                 0,
    822                 MediaCodec::BUFFER_FLAG_EOS);
    823         if (err == OK) {
    824             mInputBufferIsDequeued.editItemAt(bufferIx) = false;
    825         } else if (streamErr == ERROR_END_OF_STREAM) {
    826             streamErr = err;
    827             // err will not be ERROR_END_OF_STREAM
    828         }
    829 
    830         if (streamErr != ERROR_END_OF_STREAM) {
    831             ALOGE("Stream error for %s (err=%d), EOS %s queued",
    832                     mComponentName.c_str(),
    833                     streamErr,
    834                     err == OK ? "successfully" : "unsuccessfully");
    835             handleError(streamErr);
    836         }
    837     } else {
    838         sp<AMessage> extra;
    839         if (buffer->meta()->findMessage("extra", &extra) && extra != NULL) {
    840             int64_t resumeAtMediaTimeUs;
    841             if (extra->findInt64(
    842                         "resume-at-mediaTimeUs", &resumeAtMediaTimeUs)) {
    843                 ALOGI("[%s] suppressing rendering until %lld us",
    844                         mComponentName.c_str(), (long long)resumeAtMediaTimeUs);
    845                 mSkipRenderingUntilMediaTimeUs = resumeAtMediaTimeUs;
    846             }
    847         }
    848 
    849         int64_t timeUs = 0;
    850         uint32_t flags = 0;
    851         CHECK(buffer->meta()->findInt64("timeUs", &timeUs));
    852 
    853         int32_t eos, csd;
    854         // we do not expect SYNCFRAME for decoder
    855         if (buffer->meta()->findInt32("eos", &eos) && eos) {
    856             flags |= MediaCodec::BUFFER_FLAG_EOS;
    857         } else if (buffer->meta()->findInt32("csd", &csd) && csd) {
    858             flags |= MediaCodec::BUFFER_FLAG_CODECCONFIG;
    859         }
    860 
    861         // copy into codec buffer
    862         if (buffer != codecBuffer) {
    863             CHECK_LE(buffer->size(), codecBuffer->capacity());
    864             codecBuffer->setRange(0, buffer->size());
    865             memcpy(codecBuffer->data(), buffer->data(), buffer->size());
    866         }
    867 
    868         status_t err = mCodec->queueInputBuffer(
    869                         bufferIx,
    870                         codecBuffer->offset(),
    871                         codecBuffer->size(),
    872                         timeUs,
    873                         flags);
    874         if (err != OK) {
    875             if (mediaBuffer != NULL) {
    876                 mediaBuffer->release();
    877             }
    878             ALOGE("Failed to queue input buffer for %s (err=%d)",
    879                     mComponentName.c_str(), err);
    880             handleError(err);
    881         } else {
    882             mInputBufferIsDequeued.editItemAt(bufferIx) = false;
    883             if (mediaBuffer != NULL) {
    884                 CHECK(mMediaBuffers[bufferIx] == NULL);
    885                 mMediaBuffers.editItemAt(bufferIx) = mediaBuffer;
    886             }
    887         }
    888     }
    889     return true;
    890 }
    891 
    892 void NuPlayer::Decoder::onRenderBuffer(const sp<AMessage> &msg) {
    893     status_t err;
    894     int32_t render;
    895     size_t bufferIx;
    896     int32_t eos;
    897     CHECK(msg->findSize("buffer-ix", &bufferIx));
    898 
    899     if (!mIsAudio) {
    900         int64_t timeUs;
    901         sp<ABuffer> buffer = mOutputBuffers[bufferIx];
    902         buffer->meta()->findInt64("timeUs", &timeUs);
    903 
    904         if (mCCDecoder != NULL && mCCDecoder->isSelected()) {
    905             mCCDecoder->display(timeUs);
    906         }
    907     }
    908 
    909     if (msg->findInt32("render", &render) && render) {
    910         int64_t timestampNs;
    911         CHECK(msg->findInt64("timestampNs", &timestampNs));
    912         err = mCodec->renderOutputBufferAndRelease(bufferIx, timestampNs);
    913     } else {
    914         mNumOutputFramesDropped += !mIsAudio;
    915         err = mCodec->releaseOutputBuffer(bufferIx);
    916     }
    917     if (err != OK) {
    918         ALOGE("failed to release output buffer for %s (err=%d)",
    919                 mComponentName.c_str(), err);
    920         handleError(err);
    921     }
    922     if (msg->findInt32("eos", &eos) && eos
    923             && isDiscontinuityPending()) {
    924         finishHandleDiscontinuity(true /* flushOnTimeChange */);
    925     }
    926 }
    927 
    928 bool NuPlayer::Decoder::isDiscontinuityPending() const {
    929     return mFormatChangePending || mTimeChangePending;
    930 }
    931 
    932 void NuPlayer::Decoder::finishHandleDiscontinuity(bool flushOnTimeChange) {
    933     ALOGV("finishHandleDiscontinuity: format %d, time %d, flush %d",
    934             mFormatChangePending, mTimeChangePending, flushOnTimeChange);
    935 
    936     // If we have format change, pause and wait to be killed;
    937     // If we have time change only, flush and restart fetching.
    938 
    939     if (mFormatChangePending) {
    940         mPaused = true;
    941     } else if (mTimeChangePending) {
    942         if (flushOnTimeChange) {
    943             doFlush(false /* notifyComplete */);
    944             signalResume(false /* notifyComplete */);
    945         }
    946     }
    947 
    948     // Notify NuPlayer to either shutdown decoder, or rescan sources
    949     sp<AMessage> msg = mNotify->dup();
    950     msg->setInt32("what", kWhatInputDiscontinuity);
    951     msg->setInt32("formatChange", mFormatChangePending);
    952     msg->post();
    953 
    954     mFormatChangePending = false;
    955     mTimeChangePending = false;
    956 }
    957 
    958 bool NuPlayer::Decoder::supportsSeamlessAudioFormatChange(
    959         const sp<AMessage> &targetFormat) const {
    960     if (targetFormat == NULL) {
    961         return true;
    962     }
    963 
    964     AString mime;
    965     if (!targetFormat->findString("mime", &mime)) {
    966         return false;
    967     }
    968 
    969     if (!strcasecmp(mime.c_str(), MEDIA_MIMETYPE_AUDIO_AAC)) {
    970         // field-by-field comparison
    971         const char * keys[] = { "channel-count", "sample-rate", "is-adts" };
    972         for (unsigned int i = 0; i < sizeof(keys) / sizeof(keys[0]); i++) {
    973             int32_t oldVal, newVal;
    974             if (!mInputFormat->findInt32(keys[i], &oldVal) ||
    975                     !targetFormat->findInt32(keys[i], &newVal) ||
    976                     oldVal != newVal) {
    977                 return false;
    978             }
    979         }
    980 
    981         sp<ABuffer> oldBuf, newBuf;
    982         if (mInputFormat->findBuffer("csd-0", &oldBuf) &&
    983                 targetFormat->findBuffer("csd-0", &newBuf)) {
    984             if (oldBuf->size() != newBuf->size()) {
    985                 return false;
    986             }
    987             return !memcmp(oldBuf->data(), newBuf->data(), oldBuf->size());
    988         }
    989     }
    990     return false;
    991 }
    992 
    993 bool NuPlayer::Decoder::supportsSeamlessFormatChange(const sp<AMessage> &targetFormat) const {
    994     if (mInputFormat == NULL) {
    995         return false;
    996     }
    997 
    998     if (targetFormat == NULL) {
    999         return true;
   1000     }
   1001 
   1002     AString oldMime, newMime;
   1003     if (!mInputFormat->findString("mime", &oldMime)
   1004             || !targetFormat->findString("mime", &newMime)
   1005             || !(oldMime == newMime)) {
   1006         return false;
   1007     }
   1008 
   1009     bool audio = !strncasecmp(oldMime.c_str(), "audio/", strlen("audio/"));
   1010     bool seamless;
   1011     if (audio) {
   1012         seamless = supportsSeamlessAudioFormatChange(targetFormat);
   1013     } else {
   1014         int32_t isAdaptive;
   1015         seamless = (mCodec != NULL &&
   1016                 mInputFormat->findInt32("adaptive-playback", &isAdaptive) &&
   1017                 isAdaptive);
   1018     }
   1019 
   1020     ALOGV("%s seamless support for %s", seamless ? "yes" : "no", oldMime.c_str());
   1021     return seamless;
   1022 }
   1023 
   1024 void NuPlayer::Decoder::rememberCodecSpecificData(const sp<AMessage> &format) {
   1025     if (format == NULL) {
   1026         return;
   1027     }
   1028     mCSDsForCurrentFormat.clear();
   1029     for (int32_t i = 0; ; ++i) {
   1030         AString tag = "csd-";
   1031         tag.append(i);
   1032         sp<ABuffer> buffer;
   1033         if (!format->findBuffer(tag.c_str(), &buffer)) {
   1034             break;
   1035         }
   1036         mCSDsForCurrentFormat.push(buffer);
   1037     }
   1038 }
   1039 
   1040 void NuPlayer::Decoder::notifyResumeCompleteIfNecessary() {
   1041     if (mResumePending) {
   1042         mResumePending = false;
   1043 
   1044         sp<AMessage> notify = mNotify->dup();
   1045         notify->setInt32("what", kWhatResumeCompleted);
   1046         notify->post();
   1047     }
   1048 }
   1049 
   1050 }  // namespace android
   1051 
   1052