Home | History | Annotate | Download | only in source 
      1 /*
      2  * Copyright 2012, The Android Open Source Project
      3  *
      4  * Licensed under the Apache License, Version 2.0 (the "License");
      5  * you may not use this file except in compliance with the License.
      6  * You may obtain a copy of the License at
      7  *
      8  *     http://www.apache.org/licenses/LICENSE-2.0
      9  *
     10  * Unless required by applicable law or agreed to in writing, software
     11  * distributed under the License is distributed on an "AS IS" BASIS,
     12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
     13  * See the License for the specific language governing permissions and
     14  * limitations under the License.
     15  */
     16 
     17 //#define LOG_NDEBUG 0
     18 #define LOG_TAG "Converter"
     19 #include <utils/Log.h>
     20 
     21 #include "Converter.h"
     22 
     23 #include "MediaPuller.h"
     24 
     25 #include <cutils/properties.h>
     26 #include <gui/SurfaceTextureClient.h>
     27 #include <media/ICrypto.h>
     28 #include <media/stagefright/foundation/ABuffer.h>
     29 #include <media/stagefright/foundation/ADebug.h>
     30 #include <media/stagefright/foundation/AMessage.h>
     31 #include <media/stagefright/MediaBuffer.h>
     32 #include <media/stagefright/MediaCodec.h>
     33 #include <media/stagefright/MediaDefs.h>
     34 #include <media/stagefright/MediaErrors.h>
     35 
     36 #include <OMX_Video.h>
     37 
     38 namespace android {
     39 
     40 Converter::Converter(
     41         const sp<AMessage> &notify,
     42         const sp<ALooper> &codecLooper,
     43         const sp<AMessage> &format,
     44         bool usePCMAudio)
     45     : mInitCheck(NO_INIT),
     46       mNotify(notify),
     47       mCodecLooper(codecLooper),
     48       mInputFormat(format),
     49       mIsVideo(false),
     50       mIsPCMAudio(usePCMAudio),
     51       mDoMoreWorkPending(false)
     52 #if ENABLE_SILENCE_DETECTION
     53       ,mFirstSilentFrameUs(-1ll)
     54       ,mInSilentMode(false)
     55 #endif
     56     {
     57     AString mime;
     58     CHECK(mInputFormat->findString("mime", &mime));
     59 
     60     if (!strncasecmp("video/", mime.c_str(), 6)) {
     61         mIsVideo = true;
     62     }
     63 
     64     CHECK(!usePCMAudio || !mIsVideo);
     65 
     66     mInitCheck = initEncoder();
     67 
     68     if (mInitCheck != OK) {
     69         if (mEncoder != NULL) {
     70             mEncoder->release();
     71             mEncoder.clear();
     72         }
     73     }
     74 }
     75 
     76 Converter::~Converter() {
     77     CHECK(mEncoder == NULL);
     78 }
     79 
     80 void Converter::shutdownAsync() {
     81     ALOGV("shutdown");
     82     (new AMessage(kWhatShutdown, id()))->post();
     83 }
     84 
     85 status_t Converter::initCheck() const {
     86     return mInitCheck;
     87 }
     88 
     89 size_t Converter::getInputBufferCount() const {
     90     return mEncoderInputBuffers.size();
     91 }
     92 
     93 sp<AMessage> Converter::getOutputFormat() const {
     94     return mOutputFormat;
     95 }
     96 
     97 static int32_t getBitrate(const char *propName, int32_t defaultValue) {
     98     char val[PROPERTY_VALUE_MAX];
     99     if (property_get(propName, val, NULL)) {
    100         char *end;
    101         unsigned long x = strtoul(val, &end, 10);
    102 
    103         if (*end == '\0' && end > val && x > 0) {
    104             return x;
    105         }
    106     }
    107 
    108     return defaultValue;
    109 }
    110 
    111 status_t Converter::initEncoder() {
    112     AString inputMIME;
    113     CHECK(mInputFormat->findString("mime", &inputMIME));
    114 
    115     AString outputMIME;
    116     bool isAudio = false;
    117     if (!strcasecmp(inputMIME.c_str(), MEDIA_MIMETYPE_AUDIO_RAW)) {
    118         if (mIsPCMAudio) {
    119             outputMIME = MEDIA_MIMETYPE_AUDIO_RAW;
    120         } else {
    121             outputMIME = MEDIA_MIMETYPE_AUDIO_AAC;
    122         }
    123         isAudio = true;
    124     } else if (!strcasecmp(inputMIME.c_str(), MEDIA_MIMETYPE_VIDEO_RAW)) {
    125         outputMIME = MEDIA_MIMETYPE_VIDEO_AVC;
    126     } else {
    127         TRESPASS();
    128     }
    129 
    130     if (!mIsPCMAudio) {
    131         mEncoder = MediaCodec::CreateByType(
    132                 mCodecLooper, outputMIME.c_str(), true /* encoder */);
    133 
    134         if (mEncoder == NULL) {
    135             return ERROR_UNSUPPORTED;
    136         }
    137     }
    138 
    139     mOutputFormat = mInputFormat->dup();
    140 
    141     if (mIsPCMAudio) {
    142         return OK;
    143     }
    144 
    145     mOutputFormat->setString("mime", outputMIME.c_str());
    146 
    147     int32_t audioBitrate = getBitrate("media.wfd.audio-bitrate", 128000);
    148     int32_t videoBitrate = getBitrate("media.wfd.video-bitrate", 5000000);
    149 
    150     ALOGI("using audio bitrate of %d bps, video bitrate of %d bps",
    151           audioBitrate, videoBitrate);
    152 
    153     if (isAudio) {
    154         mOutputFormat->setInt32("bitrate", audioBitrate);
    155     } else {
    156         mOutputFormat->setInt32("bitrate", videoBitrate);
    157         mOutputFormat->setInt32("bitrate-mode", OMX_Video_ControlRateConstant);
    158         mOutputFormat->setInt32("frame-rate", 30);
    159         mOutputFormat->setInt32("i-frame-interval", 1);  // Iframes every 1 secs
    160         mOutputFormat->setInt32("prepend-sps-pps-to-idr-frames", 1);
    161     }
    162 
    163     ALOGV("output format is '%s'", mOutputFormat->debugString(0).c_str());
    164 
    165     status_t err = mEncoder->configure(
    166             mOutputFormat,
    167             NULL /* nativeWindow */,
    168             NULL /* crypto */,
    169             MediaCodec::CONFIGURE_FLAG_ENCODE);
    170 
    171     if (err != OK) {
    172         return err;
    173     }
    174 
    175     err = mEncoder->start();
    176 
    177     if (err != OK) {
    178         return err;
    179     }
    180 
    181     err = mEncoder->getInputBuffers(&mEncoderInputBuffers);
    182 
    183     if (err != OK) {
    184         return err;
    185     }
    186 
    187     return mEncoder->getOutputBuffers(&mEncoderOutputBuffers);
    188 }
    189 
    190 void Converter::notifyError(status_t err) {
    191     sp<AMessage> notify = mNotify->dup();
    192     notify->setInt32("what", kWhatError);
    193     notify->setInt32("err", err);
    194     notify->post();
    195 }
    196 
    197 // static
    198 bool Converter::IsSilence(const sp<ABuffer> &accessUnit) {
    199     const uint8_t *ptr = accessUnit->data();
    200     const uint8_t *end = ptr + accessUnit->size();
    201     while (ptr < end) {
    202         if (*ptr != 0) {
    203             return false;
    204         }
    205         ++ptr;
    206     }
    207 
    208     return true;
    209 }
    210 
    211 void Converter::onMessageReceived(const sp<AMessage> &msg) {
    212     switch (msg->what()) {
    213         case kWhatMediaPullerNotify:
    214         {
    215             int32_t what;
    216             CHECK(msg->findInt32("what", &what));
    217 
    218             if (!mIsPCMAudio && mEncoder == NULL) {
    219                 ALOGV("got msg '%s' after encoder shutdown.",
    220                       msg->debugString().c_str());
    221 
    222                 if (what == MediaPuller::kWhatAccessUnit) {
    223                     sp<ABuffer> accessUnit;
    224                     CHECK(msg->findBuffer("accessUnit", &accessUnit));
    225 
    226                     void *mbuf;
    227                     if (accessUnit->meta()->findPointer("mediaBuffer", &mbuf)
    228                             && mbuf != NULL) {
    229                         ALOGV("releasing mbuf %p", mbuf);
    230 
    231                         accessUnit->meta()->setPointer("mediaBuffer", NULL);
    232 
    233                         static_cast<MediaBuffer *>(mbuf)->release();
    234                         mbuf = NULL;
    235                     }
    236                 }
    237                 break;
    238             }
    239 
    240             if (what == MediaPuller::kWhatEOS) {
    241                 mInputBufferQueue.push_back(NULL);
    242 
    243                 feedEncoderInputBuffers();
    244 
    245                 scheduleDoMoreWork();
    246             } else {
    247                 CHECK_EQ(what, MediaPuller::kWhatAccessUnit);
    248 
    249                 sp<ABuffer> accessUnit;
    250                 CHECK(msg->findBuffer("accessUnit", &accessUnit));
    251 
    252 #if 0
    253                 void *mbuf;
    254                 if (accessUnit->meta()->findPointer("mediaBuffer", &mbuf)
    255                         && mbuf != NULL) {
    256                     ALOGI("queueing mbuf %p", mbuf);
    257                 }
    258 #endif
    259 
    260 #if ENABLE_SILENCE_DETECTION
    261                 if (!mIsVideo) {
    262                     if (IsSilence(accessUnit)) {
    263                         if (mInSilentMode) {
    264                             break;
    265                         }
    266 
    267                         int64_t nowUs = ALooper::GetNowUs();
    268 
    269                         if (mFirstSilentFrameUs < 0ll) {
    270                             mFirstSilentFrameUs = nowUs;
    271                         } else if (nowUs >= mFirstSilentFrameUs + 10000000ll) {
    272                             mInSilentMode = true;
    273                             ALOGI("audio in silent mode now.");
    274                             break;
    275                         }
    276                     } else {
    277                         if (mInSilentMode) {
    278                             ALOGI("audio no longer in silent mode.");
    279                         }
    280                         mInSilentMode = false;
    281                         mFirstSilentFrameUs = -1ll;
    282                     }
    283                 }
    284 #endif
    285 
    286                 mInputBufferQueue.push_back(accessUnit);
    287 
    288                 feedEncoderInputBuffers();
    289 
    290                 scheduleDoMoreWork();
    291             }
    292             break;
    293         }
    294 
    295         case kWhatEncoderActivity:
    296         {
    297 #if 0
    298             int64_t whenUs;
    299             if (msg->findInt64("whenUs", &whenUs)) {
    300                 int64_t nowUs = ALooper::GetNowUs();
    301                 ALOGI("[%s] kWhatEncoderActivity after %lld us",
    302                       mIsVideo ? "video" : "audio", nowUs - whenUs);
    303             }
    304 #endif
    305 
    306             mDoMoreWorkPending = false;
    307 
    308             if (mEncoder == NULL) {
    309                 break;
    310             }
    311 
    312             status_t err = doMoreWork();
    313 
    314             if (err != OK) {
    315                 notifyError(err);
    316             } else {
    317                 scheduleDoMoreWork();
    318             }
    319             break;
    320         }
    321 
    322         case kWhatRequestIDRFrame:
    323         {
    324             if (mEncoder == NULL) {
    325                 break;
    326             }
    327 
    328             if (mIsVideo) {
    329                 ALOGI("requesting IDR frame");
    330                 mEncoder->requestIDRFrame();
    331             }
    332             break;
    333         }
    334 
    335         case kWhatShutdown:
    336         {
    337             ALOGI("shutting down encoder");
    338 
    339             if (mEncoder != NULL) {
    340                 mEncoder->release();
    341                 mEncoder.clear();
    342             }
    343 
    344             AString mime;
    345             CHECK(mInputFormat->findString("mime", &mime));
    346             ALOGI("encoder (%s) shut down.", mime.c_str());
    347             break;
    348         }
    349 
    350         default:
    351             TRESPASS();
    352     }
    353 }
    354 
    355 void Converter::scheduleDoMoreWork() {
    356     if (mIsPCMAudio) {
    357         // There's no encoder involved in this case.
    358         return;
    359     }
    360 
    361     if (mDoMoreWorkPending) {
    362         return;
    363     }
    364 
    365     mDoMoreWorkPending = true;
    366 
    367 #if 1
    368     if (mEncoderActivityNotify == NULL) {
    369         mEncoderActivityNotify = new AMessage(kWhatEncoderActivity, id());
    370     }
    371     mEncoder->requestActivityNotification(mEncoderActivityNotify->dup());
    372 #else
    373     sp<AMessage> notify = new AMessage(kWhatEncoderActivity, id());
    374     notify->setInt64("whenUs", ALooper::GetNowUs());
    375     mEncoder->requestActivityNotification(notify);
    376 #endif
    377 }
    378 
    379 status_t Converter::feedRawAudioInputBuffers() {
    380     // Split incoming PCM audio into buffers of 6 AUs of 80 audio frames each
    381     // and add a 4 byte header according to the wifi display specs.
    382 
    383     while (!mInputBufferQueue.empty()) {
    384         sp<ABuffer> buffer = *mInputBufferQueue.begin();
    385         mInputBufferQueue.erase(mInputBufferQueue.begin());
    386 
    387         int16_t *ptr = (int16_t *)buffer->data();
    388         int16_t *stop = (int16_t *)(buffer->data() + buffer->size());
    389         while (ptr < stop) {
    390             *ptr = htons(*ptr);
    391             ++ptr;
    392         }
    393 
    394         static const size_t kFrameSize = 2 * sizeof(int16_t);  // stereo
    395         static const size_t kFramesPerAU = 80;
    396         static const size_t kNumAUsPerPESPacket = 6;
    397 
    398         if (mPartialAudioAU != NULL) {
    399             size_t bytesMissingForFullAU =
    400                 kNumAUsPerPESPacket * kFramesPerAU * kFrameSize
    401                 - mPartialAudioAU->size() + 4;
    402 
    403             size_t copy = buffer->size();
    404             if(copy > bytesMissingForFullAU) {
    405                 copy = bytesMissingForFullAU;
    406             }
    407 
    408             memcpy(mPartialAudioAU->data() + mPartialAudioAU->size(),
    409                    buffer->data(),
    410                    copy);
    411 
    412             mPartialAudioAU->setRange(0, mPartialAudioAU->size() + copy);
    413 
    414             buffer->setRange(buffer->offset() + copy, buffer->size() - copy);
    415 
    416             int64_t timeUs;
    417             CHECK(buffer->meta()->findInt64("timeUs", &timeUs));
    418 
    419             int64_t copyUs = (int64_t)((copy / kFrameSize) * 1E6 / 48000.0);
    420             timeUs += copyUs;
    421             buffer->meta()->setInt64("timeUs", timeUs);
    422 
    423             if (bytesMissingForFullAU == copy) {
    424                 sp<AMessage> notify = mNotify->dup();
    425                 notify->setInt32("what", kWhatAccessUnit);
    426                 notify->setBuffer("accessUnit", mPartialAudioAU);
    427                 notify->post();
    428 
    429                 mPartialAudioAU.clear();
    430             }
    431         }
    432 
    433         while (buffer->size() > 0) {
    434             sp<ABuffer> partialAudioAU =
    435                 new ABuffer(
    436                         4
    437                         + kNumAUsPerPESPacket * kFrameSize * kFramesPerAU);
    438 
    439             uint8_t *ptr = partialAudioAU->data();
    440             ptr[0] = 0xa0;  // 10100000b
    441             ptr[1] = kNumAUsPerPESPacket;
    442             ptr[2] = 0;  // reserved, audio _emphasis_flag = 0
    443 
    444             static const unsigned kQuantizationWordLength = 0;  // 16-bit
    445             static const unsigned kAudioSamplingFrequency = 2;  // 48Khz
    446             static const unsigned kNumberOfAudioChannels = 1;  // stereo
    447 
    448             ptr[3] = (kQuantizationWordLength << 6)
    449                     | (kAudioSamplingFrequency << 3)
    450                     | kNumberOfAudioChannels;
    451 
    452             size_t copy = buffer->size();
    453             if (copy > partialAudioAU->size() - 4) {
    454                 copy = partialAudioAU->size() - 4;
    455             }
    456 
    457             memcpy(&ptr[4], buffer->data(), copy);
    458 
    459             partialAudioAU->setRange(0, 4 + copy);
    460             buffer->setRange(buffer->offset() + copy, buffer->size() - copy);
    461 
    462             int64_t timeUs;
    463             CHECK(buffer->meta()->findInt64("timeUs", &timeUs));
    464 
    465             partialAudioAU->meta()->setInt64("timeUs", timeUs);
    466 
    467             int64_t copyUs = (int64_t)((copy / kFrameSize) * 1E6 / 48000.0);
    468             timeUs += copyUs;
    469             buffer->meta()->setInt64("timeUs", timeUs);
    470 
    471             if (copy == partialAudioAU->capacity() - 4) {
    472                 sp<AMessage> notify = mNotify->dup();
    473                 notify->setInt32("what", kWhatAccessUnit);
    474                 notify->setBuffer("accessUnit", partialAudioAU);
    475                 notify->post();
    476 
    477                 partialAudioAU.clear();
    478                 continue;
    479             }
    480 
    481             mPartialAudioAU = partialAudioAU;
    482         }
    483     }
    484 
    485     return OK;
    486 }
    487 
    488 status_t Converter::feedEncoderInputBuffers() {
    489     if (mIsPCMAudio) {
    490         return feedRawAudioInputBuffers();
    491     }
    492 
    493     while (!mInputBufferQueue.empty()
    494             && !mAvailEncoderInputIndices.empty()) {
    495         sp<ABuffer> buffer = *mInputBufferQueue.begin();
    496         mInputBufferQueue.erase(mInputBufferQueue.begin());
    497 
    498         size_t bufferIndex = *mAvailEncoderInputIndices.begin();
    499         mAvailEncoderInputIndices.erase(mAvailEncoderInputIndices.begin());
    500 
    501         int64_t timeUs = 0ll;
    502         uint32_t flags = 0;
    503 
    504         if (buffer != NULL) {
    505             CHECK(buffer->meta()->findInt64("timeUs", &timeUs));
    506 
    507             memcpy(mEncoderInputBuffers.itemAt(bufferIndex)->data(),
    508                    buffer->data(),
    509                    buffer->size());
    510 
    511             void *mediaBuffer;
    512             if (buffer->meta()->findPointer("mediaBuffer", &mediaBuffer)
    513                     && mediaBuffer != NULL) {
    514                 mEncoderInputBuffers.itemAt(bufferIndex)->meta()
    515                     ->setPointer("mediaBuffer", mediaBuffer);
    516 
    517                 buffer->meta()->setPointer("mediaBuffer", NULL);
    518             }
    519         } else {
    520             flags = MediaCodec::BUFFER_FLAG_EOS;
    521         }
    522 
    523         status_t err = mEncoder->queueInputBuffer(
    524                 bufferIndex, 0, (buffer == NULL) ? 0 : buffer->size(),
    525                 timeUs, flags);
    526 
    527         if (err != OK) {
    528             return err;
    529         }
    530     }
    531 
    532     return OK;
    533 }
    534 
    535 status_t Converter::doMoreWork() {
    536     status_t err;
    537 
    538     for (;;) {
    539         size_t bufferIndex;
    540         err = mEncoder->dequeueInputBuffer(&bufferIndex);
    541 
    542         if (err != OK) {
    543             break;
    544         }
    545 
    546         mAvailEncoderInputIndices.push_back(bufferIndex);
    547     }
    548 
    549     feedEncoderInputBuffers();
    550 
    551     for (;;) {
    552         size_t bufferIndex;
    553         size_t offset;
    554         size_t size;
    555         int64_t timeUs;
    556         uint32_t flags;
    557         err = mEncoder->dequeueOutputBuffer(
    558                 &bufferIndex, &offset, &size, &timeUs, &flags);
    559 
    560         if (err != OK) {
    561             if (err == -EAGAIN) {
    562                 err = OK;
    563             }
    564             break;
    565         }
    566 
    567         if (flags & MediaCodec::BUFFER_FLAG_EOS) {
    568             sp<AMessage> notify = mNotify->dup();
    569             notify->setInt32("what", kWhatEOS);
    570             notify->post();
    571         } else {
    572             sp<ABuffer> buffer = new ABuffer(size);
    573             buffer->meta()->setInt64("timeUs", timeUs);
    574 
    575             ALOGV("[%s] time %lld us (%.2f secs)",
    576                   mIsVideo ? "video" : "audio", timeUs, timeUs / 1E6);
    577 
    578             memcpy(buffer->data(),
    579                    mEncoderOutputBuffers.itemAt(bufferIndex)->base() + offset,
    580                    size);
    581 
    582             if (flags & MediaCodec::BUFFER_FLAG_CODECCONFIG) {
    583                 mOutputFormat->setBuffer("csd-0", buffer);
    584             } else {
    585                 sp<AMessage> notify = mNotify->dup();
    586                 notify->setInt32("what", kWhatAccessUnit);
    587                 notify->setBuffer("accessUnit", buffer);
    588                 notify->post();
    589             }
    590         }
    591 
    592         mEncoder->releaseOutputBuffer(bufferIndex);
    593 
    594         if (flags & MediaCodec::BUFFER_FLAG_EOS) {
    595             break;
    596         }
    597     }
    598 
    599     return err;
    600 }
    601 
    602 void Converter::requestIDRFrame() {
    603     (new AMessage(kWhatRequestIDRFrame, id()))->post();
    604 }
    605 
    606 }  // namespace android
    607