1 /* 2 * Copyright 2012, The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17 //#define LOG_NDEBUG 0 18 #define LOG_TAG "Converter" 19 #include <utils/Log.h> 20 21 #include "Converter.h" 22 23 #include "MediaPuller.h" 24 25 #include <cutils/properties.h> 26 #include <gui/Surface.h> 27 #include <media/ICrypto.h> 28 #include <media/stagefright/foundation/ABuffer.h> 29 #include <media/stagefright/foundation/ADebug.h> 30 #include <media/stagefright/foundation/AMessage.h> 31 #include <media/stagefright/MediaBuffer.h> 32 #include <media/stagefright/MediaCodec.h> 33 #include <media/stagefright/MediaDefs.h> 34 #include <media/stagefright/MediaErrors.h> 35 36 #include <OMX_Video.h> 37 38 namespace android { 39 40 Converter::Converter( 41 const sp<AMessage> ¬ify, 42 const sp<ALooper> &codecLooper, 43 const sp<AMessage> &format, 44 bool usePCMAudio) 45 : mInitCheck(NO_INIT), 46 mNotify(notify), 47 mCodecLooper(codecLooper), 48 mInputFormat(format), 49 mIsVideo(false), 50 mIsPCMAudio(usePCMAudio), 51 mNeedToManuallyPrependSPSPPS(false), 52 mDoMoreWorkPending(false) 53 #if ENABLE_SILENCE_DETECTION 54 ,mFirstSilentFrameUs(-1ll) 55 ,mInSilentMode(false) 56 #endif 57 ,mPrevVideoBitrate(-1) 58 ,mNumFramesToDrop(0) 59 { 60 AString mime; 61 CHECK(mInputFormat->findString("mime", &mime)); 62 63 if (!strncasecmp("video/", mime.c_str(), 6)) { 64 mIsVideo = true; 65 } 66 67 CHECK(!usePCMAudio || !mIsVideo); 68 69 mInitCheck = initEncoder(); 70 71 if (mInitCheck != OK) { 72 releaseEncoder(); 73 } 74 } 75 76 static void ReleaseMediaBufferReference(const sp<ABuffer> &accessUnit) { 77 void *mbuf; 78 if (accessUnit->meta()->findPointer("mediaBuffer", &mbuf) 79 && mbuf != NULL) { 80 ALOGV("releasing mbuf %p", mbuf); 81 82 accessUnit->meta()->setPointer("mediaBuffer", NULL); 83 84 static_cast<MediaBuffer *>(mbuf)->release(); 85 mbuf = NULL; 86 } 87 } 88 89 void Converter::releaseEncoder() { 90 if (mEncoder == NULL) { 91 return; 92 } 93 94 mEncoder->release(); 95 mEncoder.clear(); 96 97 while (!mInputBufferQueue.empty()) { 98 sp<ABuffer> accessUnit = *mInputBufferQueue.begin(); 99 mInputBufferQueue.erase(mInputBufferQueue.begin()); 100 101 ReleaseMediaBufferReference(accessUnit); 102 } 103 104 for (size_t i = 0; i < mEncoderInputBuffers.size(); ++i) { 105 sp<ABuffer> accessUnit = mEncoderInputBuffers.itemAt(i); 106 ReleaseMediaBufferReference(accessUnit); 107 } 108 109 mEncoderInputBuffers.clear(); 110 mEncoderOutputBuffers.clear(); 111 } 112 113 Converter::~Converter() { 114 CHECK(mEncoder == NULL); 115 } 116 117 void Converter::shutdownAsync() { 118 ALOGV("shutdown"); 119 (new AMessage(kWhatShutdown, id()))->post(); 120 } 121 122 status_t Converter::initCheck() const { 123 return mInitCheck; 124 } 125 126 size_t Converter::getInputBufferCount() const { 127 return mEncoderInputBuffers.size(); 128 } 129 130 sp<AMessage> Converter::getOutputFormat() const { 131 return mOutputFormat; 132 } 133 134 bool Converter::needToManuallyPrependSPSPPS() const { 135 return mNeedToManuallyPrependSPSPPS; 136 } 137 138 // static 139 int32_t Converter::GetInt32Property( 140 const char *propName, int32_t defaultValue) { 141 char val[PROPERTY_VALUE_MAX]; 142 if (property_get(propName, val, NULL)) { 143 char *end; 144 unsigned long x = strtoul(val, &end, 10); 145 146 if (*end == '\0' && end > val && x > 0) { 147 return x; 148 } 149 } 150 151 return defaultValue; 152 } 153 154 status_t Converter::initEncoder() { 155 AString inputMIME; 156 CHECK(mInputFormat->findString("mime", &inputMIME)); 157 158 AString outputMIME; 159 bool isAudio = false; 160 if (!strcasecmp(inputMIME.c_str(), MEDIA_MIMETYPE_AUDIO_RAW)) { 161 if (mIsPCMAudio) { 162 outputMIME = MEDIA_MIMETYPE_AUDIO_RAW; 163 } else { 164 outputMIME = MEDIA_MIMETYPE_AUDIO_AAC; 165 } 166 isAudio = true; 167 } else if (!strcasecmp(inputMIME.c_str(), MEDIA_MIMETYPE_VIDEO_RAW)) { 168 outputMIME = MEDIA_MIMETYPE_VIDEO_AVC; 169 } else { 170 TRESPASS(); 171 } 172 173 if (!mIsPCMAudio) { 174 mEncoder = MediaCodec::CreateByType( 175 mCodecLooper, outputMIME.c_str(), true /* encoder */); 176 177 if (mEncoder == NULL) { 178 return ERROR_UNSUPPORTED; 179 } 180 } 181 182 mOutputFormat = mInputFormat->dup(); 183 184 if (mIsPCMAudio) { 185 return OK; 186 } 187 188 mOutputFormat->setString("mime", outputMIME.c_str()); 189 190 int32_t audioBitrate = GetInt32Property("media.wfd.audio-bitrate", 128000); 191 int32_t videoBitrate = GetInt32Property("media.wfd.video-bitrate", 5000000); 192 mPrevVideoBitrate = videoBitrate; 193 194 ALOGI("using audio bitrate of %d bps, video bitrate of %d bps", 195 audioBitrate, videoBitrate); 196 197 if (isAudio) { 198 mOutputFormat->setInt32("bitrate", audioBitrate); 199 } else { 200 mOutputFormat->setInt32("bitrate", videoBitrate); 201 mOutputFormat->setInt32("bitrate-mode", OMX_Video_ControlRateConstant); 202 mOutputFormat->setInt32("frame-rate", 30); 203 mOutputFormat->setInt32("i-frame-interval", 15); // Iframes every 15 secs 204 205 // Configure encoder to use intra macroblock refresh mode 206 mOutputFormat->setInt32("intra-refresh-mode", OMX_VIDEO_IntraRefreshCyclic); 207 208 int width, height, mbs; 209 if (!mOutputFormat->findInt32("width", &width) 210 || !mOutputFormat->findInt32("height", &height)) { 211 return ERROR_UNSUPPORTED; 212 } 213 214 // Update macroblocks in a cyclic fashion with 10% of all MBs within 215 // frame gets updated at one time. It takes about 10 frames to 216 // completely update a whole video frame. If the frame rate is 30, 217 // it takes about 333 ms in the best case (if next frame is not an IDR) 218 // to recover from a lost/corrupted packet. 219 mbs = (((width + 15) / 16) * ((height + 15) / 16) * 10) / 100; 220 mOutputFormat->setInt32("intra-refresh-CIR-mbs", mbs); 221 } 222 223 ALOGV("output format is '%s'", mOutputFormat->debugString(0).c_str()); 224 225 mNeedToManuallyPrependSPSPPS = false; 226 227 status_t err = NO_INIT; 228 229 if (!isAudio) { 230 sp<AMessage> tmp = mOutputFormat->dup(); 231 tmp->setInt32("prepend-sps-pps-to-idr-frames", 1); 232 233 err = mEncoder->configure( 234 tmp, 235 NULL /* nativeWindow */, 236 NULL /* crypto */, 237 MediaCodec::CONFIGURE_FLAG_ENCODE); 238 239 if (err == OK) { 240 // Encoder supported prepending SPS/PPS, we don't need to emulate 241 // it. 242 mOutputFormat = tmp; 243 } else { 244 mNeedToManuallyPrependSPSPPS = true; 245 246 ALOGI("We going to manually prepend SPS and PPS to IDR frames."); 247 } 248 } 249 250 if (err != OK) { 251 // We'll get here for audio or if we failed to configure the encoder 252 // to automatically prepend SPS/PPS in the case of video. 253 254 err = mEncoder->configure( 255 mOutputFormat, 256 NULL /* nativeWindow */, 257 NULL /* crypto */, 258 MediaCodec::CONFIGURE_FLAG_ENCODE); 259 } 260 261 if (err != OK) { 262 return err; 263 } 264 265 err = mEncoder->start(); 266 267 if (err != OK) { 268 return err; 269 } 270 271 err = mEncoder->getInputBuffers(&mEncoderInputBuffers); 272 273 if (err != OK) { 274 return err; 275 } 276 277 return mEncoder->getOutputBuffers(&mEncoderOutputBuffers); 278 } 279 280 void Converter::notifyError(status_t err) { 281 sp<AMessage> notify = mNotify->dup(); 282 notify->setInt32("what", kWhatError); 283 notify->setInt32("err", err); 284 notify->post(); 285 } 286 287 // static 288 bool Converter::IsSilence(const sp<ABuffer> &accessUnit) { 289 const uint8_t *ptr = accessUnit->data(); 290 const uint8_t *end = ptr + accessUnit->size(); 291 while (ptr < end) { 292 if (*ptr != 0) { 293 return false; 294 } 295 ++ptr; 296 } 297 298 return true; 299 } 300 301 void Converter::onMessageReceived(const sp<AMessage> &msg) { 302 switch (msg->what()) { 303 case kWhatMediaPullerNotify: 304 { 305 int32_t what; 306 CHECK(msg->findInt32("what", &what)); 307 308 if (!mIsPCMAudio && mEncoder == NULL) { 309 ALOGV("got msg '%s' after encoder shutdown.", 310 msg->debugString().c_str()); 311 312 if (what == MediaPuller::kWhatAccessUnit) { 313 sp<ABuffer> accessUnit; 314 CHECK(msg->findBuffer("accessUnit", &accessUnit)); 315 316 ReleaseMediaBufferReference(accessUnit); 317 } 318 break; 319 } 320 321 if (what == MediaPuller::kWhatEOS) { 322 mInputBufferQueue.push_back(NULL); 323 324 feedEncoderInputBuffers(); 325 326 scheduleDoMoreWork(); 327 } else { 328 CHECK_EQ(what, MediaPuller::kWhatAccessUnit); 329 330 sp<ABuffer> accessUnit; 331 CHECK(msg->findBuffer("accessUnit", &accessUnit)); 332 333 if (mIsVideo && mNumFramesToDrop) { 334 --mNumFramesToDrop; 335 ALOGI("dropping frame."); 336 ReleaseMediaBufferReference(accessUnit); 337 break; 338 } 339 340 #if 0 341 void *mbuf; 342 if (accessUnit->meta()->findPointer("mediaBuffer", &mbuf) 343 && mbuf != NULL) { 344 ALOGI("queueing mbuf %p", mbuf); 345 } 346 #endif 347 348 #if ENABLE_SILENCE_DETECTION 349 if (!mIsVideo) { 350 if (IsSilence(accessUnit)) { 351 if (mInSilentMode) { 352 break; 353 } 354 355 int64_t nowUs = ALooper::GetNowUs(); 356 357 if (mFirstSilentFrameUs < 0ll) { 358 mFirstSilentFrameUs = nowUs; 359 } else if (nowUs >= mFirstSilentFrameUs + 10000000ll) { 360 mInSilentMode = true; 361 ALOGI("audio in silent mode now."); 362 break; 363 } 364 } else { 365 if (mInSilentMode) { 366 ALOGI("audio no longer in silent mode."); 367 } 368 mInSilentMode = false; 369 mFirstSilentFrameUs = -1ll; 370 } 371 } 372 #endif 373 374 mInputBufferQueue.push_back(accessUnit); 375 376 feedEncoderInputBuffers(); 377 378 scheduleDoMoreWork(); 379 } 380 break; 381 } 382 383 case kWhatEncoderActivity: 384 { 385 #if 0 386 int64_t whenUs; 387 if (msg->findInt64("whenUs", &whenUs)) { 388 int64_t nowUs = ALooper::GetNowUs(); 389 ALOGI("[%s] kWhatEncoderActivity after %lld us", 390 mIsVideo ? "video" : "audio", nowUs - whenUs); 391 } 392 #endif 393 394 mDoMoreWorkPending = false; 395 396 if (mEncoder == NULL) { 397 break; 398 } 399 400 status_t err = doMoreWork(); 401 402 if (err != OK) { 403 notifyError(err); 404 } else { 405 scheduleDoMoreWork(); 406 } 407 break; 408 } 409 410 case kWhatRequestIDRFrame: 411 { 412 if (mEncoder == NULL) { 413 break; 414 } 415 416 if (mIsVideo) { 417 ALOGI("requesting IDR frame"); 418 mEncoder->requestIDRFrame(); 419 } 420 break; 421 } 422 423 case kWhatShutdown: 424 { 425 ALOGI("shutting down %s encoder", mIsVideo ? "video" : "audio"); 426 427 releaseEncoder(); 428 429 AString mime; 430 CHECK(mInputFormat->findString("mime", &mime)); 431 ALOGI("encoder (%s) shut down.", mime.c_str()); 432 break; 433 } 434 435 case kWhatDropAFrame: 436 { 437 ++mNumFramesToDrop; 438 break; 439 } 440 441 default: 442 TRESPASS(); 443 } 444 } 445 446 void Converter::scheduleDoMoreWork() { 447 if (mIsPCMAudio) { 448 // There's no encoder involved in this case. 449 return; 450 } 451 452 if (mDoMoreWorkPending) { 453 return; 454 } 455 456 mDoMoreWorkPending = true; 457 458 #if 1 459 if (mEncoderActivityNotify == NULL) { 460 mEncoderActivityNotify = new AMessage(kWhatEncoderActivity, id()); 461 } 462 mEncoder->requestActivityNotification(mEncoderActivityNotify->dup()); 463 #else 464 sp<AMessage> notify = new AMessage(kWhatEncoderActivity, id()); 465 notify->setInt64("whenUs", ALooper::GetNowUs()); 466 mEncoder->requestActivityNotification(notify); 467 #endif 468 } 469 470 status_t Converter::feedRawAudioInputBuffers() { 471 // Split incoming PCM audio into buffers of 6 AUs of 80 audio frames each 472 // and add a 4 byte header according to the wifi display specs. 473 474 while (!mInputBufferQueue.empty()) { 475 sp<ABuffer> buffer = *mInputBufferQueue.begin(); 476 mInputBufferQueue.erase(mInputBufferQueue.begin()); 477 478 int16_t *ptr = (int16_t *)buffer->data(); 479 int16_t *stop = (int16_t *)(buffer->data() + buffer->size()); 480 while (ptr < stop) { 481 *ptr = htons(*ptr); 482 ++ptr; 483 } 484 485 static const size_t kFrameSize = 2 * sizeof(int16_t); // stereo 486 static const size_t kFramesPerAU = 80; 487 static const size_t kNumAUsPerPESPacket = 6; 488 489 if (mPartialAudioAU != NULL) { 490 size_t bytesMissingForFullAU = 491 kNumAUsPerPESPacket * kFramesPerAU * kFrameSize 492 - mPartialAudioAU->size() + 4; 493 494 size_t copy = buffer->size(); 495 if(copy > bytesMissingForFullAU) { 496 copy = bytesMissingForFullAU; 497 } 498 499 memcpy(mPartialAudioAU->data() + mPartialAudioAU->size(), 500 buffer->data(), 501 copy); 502 503 mPartialAudioAU->setRange(0, mPartialAudioAU->size() + copy); 504 505 buffer->setRange(buffer->offset() + copy, buffer->size() - copy); 506 507 int64_t timeUs; 508 CHECK(buffer->meta()->findInt64("timeUs", &timeUs)); 509 510 int64_t copyUs = (int64_t)((copy / kFrameSize) * 1E6 / 48000.0); 511 timeUs += copyUs; 512 buffer->meta()->setInt64("timeUs", timeUs); 513 514 if (bytesMissingForFullAU == copy) { 515 sp<AMessage> notify = mNotify->dup(); 516 notify->setInt32("what", kWhatAccessUnit); 517 notify->setBuffer("accessUnit", mPartialAudioAU); 518 notify->post(); 519 520 mPartialAudioAU.clear(); 521 } 522 } 523 524 while (buffer->size() > 0) { 525 sp<ABuffer> partialAudioAU = 526 new ABuffer( 527 4 528 + kNumAUsPerPESPacket * kFrameSize * kFramesPerAU); 529 530 uint8_t *ptr = partialAudioAU->data(); 531 ptr[0] = 0xa0; // 10100000b 532 ptr[1] = kNumAUsPerPESPacket; 533 ptr[2] = 0; // reserved, audio _emphasis_flag = 0 534 535 static const unsigned kQuantizationWordLength = 0; // 16-bit 536 static const unsigned kAudioSamplingFrequency = 2; // 48Khz 537 static const unsigned kNumberOfAudioChannels = 1; // stereo 538 539 ptr[3] = (kQuantizationWordLength << 6) 540 | (kAudioSamplingFrequency << 3) 541 | kNumberOfAudioChannels; 542 543 size_t copy = buffer->size(); 544 if (copy > partialAudioAU->size() - 4) { 545 copy = partialAudioAU->size() - 4; 546 } 547 548 memcpy(&ptr[4], buffer->data(), copy); 549 550 partialAudioAU->setRange(0, 4 + copy); 551 buffer->setRange(buffer->offset() + copy, buffer->size() - copy); 552 553 int64_t timeUs; 554 CHECK(buffer->meta()->findInt64("timeUs", &timeUs)); 555 556 partialAudioAU->meta()->setInt64("timeUs", timeUs); 557 558 int64_t copyUs = (int64_t)((copy / kFrameSize) * 1E6 / 48000.0); 559 timeUs += copyUs; 560 buffer->meta()->setInt64("timeUs", timeUs); 561 562 if (copy == partialAudioAU->capacity() - 4) { 563 sp<AMessage> notify = mNotify->dup(); 564 notify->setInt32("what", kWhatAccessUnit); 565 notify->setBuffer("accessUnit", partialAudioAU); 566 notify->post(); 567 568 partialAudioAU.clear(); 569 continue; 570 } 571 572 mPartialAudioAU = partialAudioAU; 573 } 574 } 575 576 return OK; 577 } 578 579 status_t Converter::feedEncoderInputBuffers() { 580 if (mIsPCMAudio) { 581 return feedRawAudioInputBuffers(); 582 } 583 584 while (!mInputBufferQueue.empty() 585 && !mAvailEncoderInputIndices.empty()) { 586 sp<ABuffer> buffer = *mInputBufferQueue.begin(); 587 mInputBufferQueue.erase(mInputBufferQueue.begin()); 588 589 size_t bufferIndex = *mAvailEncoderInputIndices.begin(); 590 mAvailEncoderInputIndices.erase(mAvailEncoderInputIndices.begin()); 591 592 int64_t timeUs = 0ll; 593 uint32_t flags = 0; 594 595 if (buffer != NULL) { 596 CHECK(buffer->meta()->findInt64("timeUs", &timeUs)); 597 598 memcpy(mEncoderInputBuffers.itemAt(bufferIndex)->data(), 599 buffer->data(), 600 buffer->size()); 601 602 void *mediaBuffer; 603 if (buffer->meta()->findPointer("mediaBuffer", &mediaBuffer) 604 && mediaBuffer != NULL) { 605 mEncoderInputBuffers.itemAt(bufferIndex)->meta() 606 ->setPointer("mediaBuffer", mediaBuffer); 607 608 buffer->meta()->setPointer("mediaBuffer", NULL); 609 } 610 } else { 611 flags = MediaCodec::BUFFER_FLAG_EOS; 612 } 613 614 status_t err = mEncoder->queueInputBuffer( 615 bufferIndex, 0, (buffer == NULL) ? 0 : buffer->size(), 616 timeUs, flags); 617 618 if (err != OK) { 619 return err; 620 } 621 } 622 623 return OK; 624 } 625 626 status_t Converter::doMoreWork() { 627 status_t err; 628 629 for (;;) { 630 size_t bufferIndex; 631 err = mEncoder->dequeueInputBuffer(&bufferIndex); 632 633 if (err != OK) { 634 break; 635 } 636 637 mAvailEncoderInputIndices.push_back(bufferIndex); 638 } 639 640 feedEncoderInputBuffers(); 641 642 for (;;) { 643 size_t bufferIndex; 644 size_t offset; 645 size_t size; 646 int64_t timeUs; 647 uint32_t flags; 648 err = mEncoder->dequeueOutputBuffer( 649 &bufferIndex, &offset, &size, &timeUs, &flags); 650 651 if (err != OK) { 652 if (err == INFO_FORMAT_CHANGED) { 653 continue; 654 } else if (err == INFO_OUTPUT_BUFFERS_CHANGED) { 655 mEncoder->getOutputBuffers(&mEncoderOutputBuffers); 656 continue; 657 } 658 659 if (err == -EAGAIN) { 660 err = OK; 661 } 662 break; 663 } 664 665 if (flags & MediaCodec::BUFFER_FLAG_EOS) { 666 sp<AMessage> notify = mNotify->dup(); 667 notify->setInt32("what", kWhatEOS); 668 notify->post(); 669 } else { 670 sp<ABuffer> buffer = new ABuffer(size); 671 buffer->meta()->setInt64("timeUs", timeUs); 672 673 ALOGV("[%s] time %lld us (%.2f secs)", 674 mIsVideo ? "video" : "audio", timeUs, timeUs / 1E6); 675 676 memcpy(buffer->data(), 677 mEncoderOutputBuffers.itemAt(bufferIndex)->base() + offset, 678 size); 679 680 if (flags & MediaCodec::BUFFER_FLAG_CODECCONFIG) { 681 mOutputFormat->setBuffer("csd-0", buffer); 682 } else { 683 sp<AMessage> notify = mNotify->dup(); 684 notify->setInt32("what", kWhatAccessUnit); 685 notify->setBuffer("accessUnit", buffer); 686 notify->post(); 687 } 688 } 689 690 mEncoder->releaseOutputBuffer(bufferIndex); 691 692 if (flags & MediaCodec::BUFFER_FLAG_EOS) { 693 break; 694 } 695 } 696 697 return err; 698 } 699 700 void Converter::requestIDRFrame() { 701 (new AMessage(kWhatRequestIDRFrame, id()))->post(); 702 } 703 704 void Converter::dropAFrame() { 705 (new AMessage(kWhatDropAFrame, id()))->post(); 706 } 707 708 int32_t Converter::getVideoBitrate() const { 709 return mPrevVideoBitrate; 710 } 711 712 void Converter::setVideoBitrate(int32_t bitRate) { 713 if (mIsVideo && mEncoder != NULL && bitRate != mPrevVideoBitrate) { 714 sp<AMessage> params = new AMessage; 715 params->setInt32("videoBitrate", bitRate); 716 717 mEncoder->setParameters(params); 718 719 mPrevVideoBitrate = bitRate; 720 } 721 } 722 723 } // namespace android 724
