1 /* 2 * Copyright 2012, The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17 //#define LOG_NDEBUG 0 18 #define LOG_TAG "MediaCodec" 19 #include <inttypes.h> 20 21 #include "include/avc_utils.h" 22 #include "include/SoftwareRenderer.h" 23 24 #include <binder/IMemory.h> 25 #include <binder/IPCThreadState.h> 26 #include <binder/IServiceManager.h> 27 #include <binder/MemoryDealer.h> 28 #include <gui/BufferQueue.h> 29 #include <gui/Surface.h> 30 #include <media/ICrypto.h> 31 #include <media/IOMX.h> 32 #include <media/IResourceManagerService.h> 33 #include <media/stagefright/foundation/ABuffer.h> 34 #include <media/stagefright/foundation/ADebug.h> 35 #include <media/stagefright/foundation/AMessage.h> 36 #include <media/stagefright/foundation/AString.h> 37 #include <media/stagefright/foundation/hexdump.h> 38 #include <media/stagefright/ACodec.h> 39 #include <media/stagefright/BufferProducerWrapper.h> 40 #include <media/stagefright/MediaCodec.h> 41 #include <media/stagefright/MediaCodecList.h> 42 #include <media/stagefright/MediaDefs.h> 43 #include <media/stagefright/MediaErrors.h> 44 #include <media/stagefright/MediaFilter.h> 45 #include <media/stagefright/MetaData.h> 46 #include <media/stagefright/OMXClient.h> 47 #include <media/stagefright/OMXCodec.h> 48 #include <media/stagefright/PersistentSurface.h> 49 #include <media/stagefright/SurfaceUtils.h> 50 #include <mediautils/BatteryNotifier.h> 51 #include <private/android_filesystem_config.h> 52 #include <utils/Log.h> 53 #include <utils/Singleton.h> 54 55 namespace android { 56 57 static int64_t getId(sp<IResourceManagerClient> client) { 58 return (int64_t) client.get(); 59 } 60 61 static bool isResourceError(status_t err) { 62 return (err == NO_MEMORY); 63 } 64 65 static const int kMaxRetry = 2; 66 67 struct ResourceManagerClient : public BnResourceManagerClient { 68 ResourceManagerClient(MediaCodec* codec) : mMediaCodec(codec) {} 69 70 virtual bool reclaimResource() { 71 sp<MediaCodec> codec = mMediaCodec.promote(); 72 if (codec == NULL) { 73 // codec is already gone. 74 return true; 75 } 76 status_t err = codec->reclaim(); 77 if (err != OK) { 78 ALOGW("ResourceManagerClient failed to release codec with err %d", err); 79 } 80 return (err == OK); 81 } 82 83 virtual String8 getName() { 84 String8 ret; 85 sp<MediaCodec> codec = mMediaCodec.promote(); 86 if (codec == NULL) { 87 // codec is already gone. 88 return ret; 89 } 90 91 AString name; 92 if (codec->getName(&name) == OK) { 93 ret.setTo(name.c_str()); 94 } 95 return ret; 96 } 97 98 protected: 99 virtual ~ResourceManagerClient() {} 100 101 private: 102 wp<MediaCodec> mMediaCodec; 103 104 DISALLOW_EVIL_CONSTRUCTORS(ResourceManagerClient); 105 }; 106 107 MediaCodec::ResourceManagerServiceProxy::ResourceManagerServiceProxy(pid_t pid) 108 : mPid(pid) { 109 if (mPid == MediaCodec::kNoPid) { 110 mPid = IPCThreadState::self()->getCallingPid(); 111 } 112 } 113 114 MediaCodec::ResourceManagerServiceProxy::~ResourceManagerServiceProxy() { 115 if (mService != NULL) { 116 IInterface::asBinder(mService)->unlinkToDeath(this); 117 } 118 } 119 120 void MediaCodec::ResourceManagerServiceProxy::init() { 121 sp<IServiceManager> sm = defaultServiceManager(); 122 sp<IBinder> binder = sm->getService(String16("media.resource_manager")); 123 mService = interface_cast<IResourceManagerService>(binder); 124 if (mService == NULL) { 125 ALOGE("Failed to get ResourceManagerService"); 126 return; 127 } 128 IInterface::asBinder(mService)->linkToDeath(this); 129 } 130 131 void MediaCodec::ResourceManagerServiceProxy::binderDied(const wp<IBinder>& /*who*/) { 132 ALOGW("ResourceManagerService died."); 133 Mutex::Autolock _l(mLock); 134 mService.clear(); 135 } 136 137 void MediaCodec::ResourceManagerServiceProxy::addResource( 138 int64_t clientId, 139 const sp<IResourceManagerClient> client, 140 const Vector<MediaResource> &resources) { 141 Mutex::Autolock _l(mLock); 142 if (mService == NULL) { 143 return; 144 } 145 mService->addResource(mPid, clientId, client, resources); 146 } 147 148 void MediaCodec::ResourceManagerServiceProxy::removeResource(int64_t clientId) { 149 Mutex::Autolock _l(mLock); 150 if (mService == NULL) { 151 return; 152 } 153 mService->removeResource(mPid, clientId); 154 } 155 156 bool MediaCodec::ResourceManagerServiceProxy::reclaimResource( 157 const Vector<MediaResource> &resources) { 158 Mutex::Autolock _l(mLock); 159 if (mService == NULL) { 160 return false; 161 } 162 return mService->reclaimResource(mPid, resources); 163 } 164 165 // static 166 sp<MediaCodec> MediaCodec::CreateByType( 167 const sp<ALooper> &looper, const char *mime, bool encoder, status_t *err, pid_t pid) { 168 sp<MediaCodec> codec = new MediaCodec(looper, pid); 169 170 const status_t ret = codec->init(mime, true /* nameIsType */, encoder); 171 if (err != NULL) { 172 *err = ret; 173 } 174 return ret == OK ? codec : NULL; // NULL deallocates codec. 175 } 176 177 // static 178 sp<MediaCodec> MediaCodec::CreateByComponentName( 179 const sp<ALooper> &looper, const char *name, status_t *err, pid_t pid) { 180 sp<MediaCodec> codec = new MediaCodec(looper, pid); 181 182 const status_t ret = codec->init(name, false /* nameIsType */, false /* encoder */); 183 if (err != NULL) { 184 *err = ret; 185 } 186 return ret == OK ? codec : NULL; // NULL deallocates codec. 187 } 188 189 // static 190 sp<PersistentSurface> MediaCodec::CreatePersistentInputSurface() { 191 OMXClient client; 192 CHECK_EQ(client.connect(), (status_t)OK); 193 sp<IOMX> omx = client.interface(); 194 195 const sp<IMediaCodecList> mediaCodecList = MediaCodecList::getInstance(); 196 if (mediaCodecList == NULL) { 197 ALOGE("Failed to obtain MediaCodecList!"); 198 return NULL; // if called from Java should raise IOException 199 } 200 201 AString tmp; 202 sp<AMessage> globalSettings = mediaCodecList->getGlobalSettings(); 203 if (globalSettings == NULL || !globalSettings->findString( 204 kMaxEncoderInputBuffers, &tmp)) { 205 ALOGE("Failed to get encoder input buffer count!"); 206 return NULL; 207 } 208 209 int32_t bufferCount = strtol(tmp.c_str(), NULL, 10); 210 if (bufferCount <= 0 211 || bufferCount > BufferQueue::MAX_MAX_ACQUIRED_BUFFERS) { 212 ALOGE("Encoder input buffer count is invalid!"); 213 return NULL; 214 } 215 216 sp<IGraphicBufferProducer> bufferProducer; 217 sp<IGraphicBufferConsumer> bufferConsumer; 218 219 status_t err = omx->createPersistentInputSurface( 220 &bufferProducer, &bufferConsumer); 221 222 if (err != OK) { 223 ALOGE("Failed to create persistent input surface."); 224 return NULL; 225 } 226 227 err = bufferConsumer->setMaxAcquiredBufferCount(bufferCount); 228 229 if (err != NO_ERROR) { 230 ALOGE("Unable to set BQ max acquired buffer count to %u: %d", 231 bufferCount, err); 232 return NULL; 233 } 234 235 return new PersistentSurface(bufferProducer, bufferConsumer); 236 } 237 238 MediaCodec::MediaCodec(const sp<ALooper> &looper, pid_t pid) 239 : mState(UNINITIALIZED), 240 mReleasedByResourceManager(false), 241 mLooper(looper), 242 mCodec(NULL), 243 mReplyID(0), 244 mFlags(0), 245 mStickyError(OK), 246 mSoftRenderer(NULL), 247 mResourceManagerClient(new ResourceManagerClient(this)), 248 mResourceManagerService(new ResourceManagerServiceProxy(pid)), 249 mBatteryStatNotified(false), 250 mIsVideo(false), 251 mVideoWidth(0), 252 mVideoHeight(0), 253 mRotationDegrees(0), 254 mDequeueInputTimeoutGeneration(0), 255 mDequeueInputReplyID(0), 256 mDequeueOutputTimeoutGeneration(0), 257 mDequeueOutputReplyID(0), 258 mHaveInputSurface(false), 259 mHavePendingInputBuffers(false) { 260 } 261 262 MediaCodec::~MediaCodec() { 263 CHECK_EQ(mState, UNINITIALIZED); 264 mResourceManagerService->removeResource(getId(mResourceManagerClient)); 265 } 266 267 // static 268 status_t MediaCodec::PostAndAwaitResponse( 269 const sp<AMessage> &msg, sp<AMessage> *response) { 270 status_t err = msg->postAndAwaitResponse(response); 271 272 if (err != OK) { 273 return err; 274 } 275 276 if (!(*response)->findInt32("err", &err)) { 277 err = OK; 278 } 279 280 return err; 281 } 282 283 void MediaCodec::PostReplyWithError(const sp<AReplyToken> &replyID, int32_t err) { 284 int32_t finalErr = err; 285 if (mReleasedByResourceManager) { 286 // override the err code if MediaCodec has been released by ResourceManager. 287 finalErr = DEAD_OBJECT; 288 } 289 290 sp<AMessage> response = new AMessage; 291 response->setInt32("err", finalErr); 292 response->postReply(replyID); 293 } 294 295 status_t MediaCodec::init(const AString &name, bool nameIsType, bool encoder) { 296 mResourceManagerService->init(); 297 298 // save init parameters for reset 299 mInitName = name; 300 mInitNameIsType = nameIsType; 301 mInitIsEncoder = encoder; 302 303 // Current video decoders do not return from OMX_FillThisBuffer 304 // quickly, violating the OpenMAX specs, until that is remedied 305 // we need to invest in an extra looper to free the main event 306 // queue. 307 308 if (nameIsType || !strncasecmp(name.c_str(), "omx.", 4)) { 309 mCodec = new ACodec; 310 } else if (!nameIsType 311 && !strncasecmp(name.c_str(), "android.filter.", 15)) { 312 mCodec = new MediaFilter; 313 } else { 314 return NAME_NOT_FOUND; 315 } 316 317 bool secureCodec = false; 318 if (nameIsType && !strncasecmp(name.c_str(), "video/", 6)) { 319 mIsVideo = true; 320 } else { 321 AString tmp = name; 322 if (tmp.endsWith(".secure")) { 323 secureCodec = true; 324 tmp.erase(tmp.size() - 7, 7); 325 } 326 const sp<IMediaCodecList> mcl = MediaCodecList::getInstance(); 327 if (mcl == NULL) { 328 mCodec = NULL; // remove the codec. 329 return NO_INIT; // if called from Java should raise IOException 330 } 331 ssize_t codecIdx = mcl->findCodecByName(tmp.c_str()); 332 if (codecIdx >= 0) { 333 const sp<MediaCodecInfo> info = mcl->getCodecInfo(codecIdx); 334 Vector<AString> mimes; 335 info->getSupportedMimes(&mimes); 336 for (size_t i = 0; i < mimes.size(); i++) { 337 if (mimes[i].startsWith("video/")) { 338 mIsVideo = true; 339 break; 340 } 341 } 342 } 343 } 344 345 if (mIsVideo) { 346 // video codec needs dedicated looper 347 if (mCodecLooper == NULL) { 348 mCodecLooper = new ALooper; 349 mCodecLooper->setName("CodecLooper"); 350 mCodecLooper->start(false, false, ANDROID_PRIORITY_AUDIO); 351 } 352 353 mCodecLooper->registerHandler(mCodec); 354 } else { 355 mLooper->registerHandler(mCodec); 356 } 357 358 mLooper->registerHandler(this); 359 360 mCodec->setNotificationMessage(new AMessage(kWhatCodecNotify, this)); 361 362 sp<AMessage> msg = new AMessage(kWhatInit, this); 363 msg->setString("name", name); 364 msg->setInt32("nameIsType", nameIsType); 365 366 if (nameIsType) { 367 msg->setInt32("encoder", encoder); 368 } 369 370 status_t err; 371 Vector<MediaResource> resources; 372 const char *type = secureCodec ? kResourceSecureCodec : kResourceNonSecureCodec; 373 const char *subtype = mIsVideo ? kResourceVideoCodec : kResourceAudioCodec; 374 resources.push_back(MediaResource(String8(type), String8(subtype), 1)); 375 for (int i = 0; i <= kMaxRetry; ++i) { 376 if (i > 0) { 377 // Don't try to reclaim resource for the first time. 378 if (!mResourceManagerService->reclaimResource(resources)) { 379 break; 380 } 381 } 382 383 sp<AMessage> response; 384 err = PostAndAwaitResponse(msg, &response); 385 if (!isResourceError(err)) { 386 break; 387 } 388 } 389 return err; 390 } 391 392 status_t MediaCodec::setCallback(const sp<AMessage> &callback) { 393 sp<AMessage> msg = new AMessage(kWhatSetCallback, this); 394 msg->setMessage("callback", callback); 395 396 sp<AMessage> response; 397 return PostAndAwaitResponse(msg, &response); 398 } 399 400 status_t MediaCodec::setOnFrameRenderedNotification(const sp<AMessage> ¬ify) { 401 sp<AMessage> msg = new AMessage(kWhatSetNotification, this); 402 msg->setMessage("on-frame-rendered", notify); 403 return msg->post(); 404 } 405 406 status_t MediaCodec::configure( 407 const sp<AMessage> &format, 408 const sp<Surface> &surface, 409 const sp<ICrypto> &crypto, 410 uint32_t flags) { 411 sp<AMessage> msg = new AMessage(kWhatConfigure, this); 412 413 if (mIsVideo) { 414 format->findInt32("width", &mVideoWidth); 415 format->findInt32("height", &mVideoHeight); 416 if (!format->findInt32("rotation-degrees", &mRotationDegrees)) { 417 mRotationDegrees = 0; 418 } 419 } 420 421 msg->setMessage("format", format); 422 msg->setInt32("flags", flags); 423 msg->setObject("surface", surface); 424 425 if (crypto != NULL) { 426 msg->setPointer("crypto", crypto.get()); 427 } 428 429 // save msg for reset 430 mConfigureMsg = msg; 431 432 status_t err; 433 Vector<MediaResource> resources; 434 const char *type = (mFlags & kFlagIsSecure) ? 435 kResourceSecureCodec : kResourceNonSecureCodec; 436 const char *subtype = mIsVideo ? kResourceVideoCodec : kResourceAudioCodec; 437 resources.push_back(MediaResource(String8(type), String8(subtype), 1)); 438 // Don't know the buffer size at this point, but it's fine to use 1 because 439 // the reclaimResource call doesn't consider the requester's buffer size for now. 440 resources.push_back(MediaResource(String8(kResourceGraphicMemory), 1)); 441 for (int i = 0; i <= kMaxRetry; ++i) { 442 if (i > 0) { 443 // Don't try to reclaim resource for the first time. 444 if (!mResourceManagerService->reclaimResource(resources)) { 445 break; 446 } 447 } 448 449 sp<AMessage> response; 450 err = PostAndAwaitResponse(msg, &response); 451 if (err != OK && err != INVALID_OPERATION) { 452 // MediaCodec now set state to UNINITIALIZED upon any fatal error. 453 // To maintain backward-compatibility, do a reset() to put codec 454 // back into INITIALIZED state. 455 // But don't reset if the err is INVALID_OPERATION, which means 456 // the configure failure is due to wrong state. 457 458 ALOGE("configure failed with err 0x%08x, resetting...", err); 459 reset(); 460 } 461 if (!isResourceError(err)) { 462 break; 463 } 464 } 465 return err; 466 } 467 468 status_t MediaCodec::setInputSurface( 469 const sp<PersistentSurface> &surface) { 470 sp<AMessage> msg = new AMessage(kWhatSetInputSurface, this); 471 msg->setObject("input-surface", surface.get()); 472 473 sp<AMessage> response; 474 return PostAndAwaitResponse(msg, &response); 475 } 476 477 status_t MediaCodec::setSurface(const sp<Surface> &surface) { 478 sp<AMessage> msg = new AMessage(kWhatSetSurface, this); 479 msg->setObject("surface", surface); 480 481 sp<AMessage> response; 482 return PostAndAwaitResponse(msg, &response); 483 } 484 485 status_t MediaCodec::createInputSurface( 486 sp<IGraphicBufferProducer>* bufferProducer) { 487 sp<AMessage> msg = new AMessage(kWhatCreateInputSurface, this); 488 489 sp<AMessage> response; 490 status_t err = PostAndAwaitResponse(msg, &response); 491 if (err == NO_ERROR) { 492 // unwrap the sp<IGraphicBufferProducer> 493 sp<RefBase> obj; 494 bool found = response->findObject("input-surface", &obj); 495 CHECK(found); 496 sp<BufferProducerWrapper> wrapper( 497 static_cast<BufferProducerWrapper*>(obj.get())); 498 *bufferProducer = wrapper->getBufferProducer(); 499 } else { 500 ALOGW("createInputSurface failed, err=%d", err); 501 } 502 return err; 503 } 504 505 uint64_t MediaCodec::getGraphicBufferSize() { 506 if (!mIsVideo) { 507 return 0; 508 } 509 510 uint64_t size = 0; 511 size_t portNum = sizeof(mPortBuffers) / sizeof((mPortBuffers)[0]); 512 for (size_t i = 0; i < portNum; ++i) { 513 // TODO: this is just an estimation, we should get the real buffer size from ACodec. 514 size += mPortBuffers[i].size() * mVideoWidth * mVideoHeight * 3 / 2; 515 } 516 return size; 517 } 518 519 void MediaCodec::addResource(const String8 &type, const String8 &subtype, uint64_t value) { 520 Vector<MediaResource> resources; 521 resources.push_back(MediaResource(type, subtype, value)); 522 mResourceManagerService->addResource( 523 getId(mResourceManagerClient), mResourceManagerClient, resources); 524 } 525 526 status_t MediaCodec::start() { 527 sp<AMessage> msg = new AMessage(kWhatStart, this); 528 529 status_t err; 530 Vector<MediaResource> resources; 531 const char *type = (mFlags & kFlagIsSecure) ? 532 kResourceSecureCodec : kResourceNonSecureCodec; 533 const char *subtype = mIsVideo ? kResourceVideoCodec : kResourceAudioCodec; 534 resources.push_back(MediaResource(String8(type), String8(subtype), 1)); 535 // Don't know the buffer size at this point, but it's fine to use 1 because 536 // the reclaimResource call doesn't consider the requester's buffer size for now. 537 resources.push_back(MediaResource(String8(kResourceGraphicMemory), 1)); 538 for (int i = 0; i <= kMaxRetry; ++i) { 539 if (i > 0) { 540 // Don't try to reclaim resource for the first time. 541 if (!mResourceManagerService->reclaimResource(resources)) { 542 break; 543 } 544 // Recover codec from previous error before retry start. 545 err = reset(); 546 if (err != OK) { 547 ALOGE("retrying start: failed to reset codec"); 548 break; 549 } 550 sp<AMessage> response; 551 err = PostAndAwaitResponse(mConfigureMsg, &response); 552 if (err != OK) { 553 ALOGE("retrying start: failed to configure codec"); 554 break; 555 } 556 } 557 558 sp<AMessage> response; 559 err = PostAndAwaitResponse(msg, &response); 560 if (!isResourceError(err)) { 561 break; 562 } 563 } 564 return err; 565 } 566 567 status_t MediaCodec::stop() { 568 sp<AMessage> msg = new AMessage(kWhatStop, this); 569 570 sp<AMessage> response; 571 return PostAndAwaitResponse(msg, &response); 572 } 573 574 status_t MediaCodec::reclaim() { 575 sp<AMessage> msg = new AMessage(kWhatRelease, this); 576 msg->setInt32("reclaimed", 1); 577 578 sp<AMessage> response; 579 return PostAndAwaitResponse(msg, &response); 580 } 581 582 status_t MediaCodec::release() { 583 sp<AMessage> msg = new AMessage(kWhatRelease, this); 584 585 sp<AMessage> response; 586 return PostAndAwaitResponse(msg, &response); 587 } 588 589 status_t MediaCodec::reset() { 590 /* When external-facing MediaCodec object is created, 591 it is already initialized. Thus, reset is essentially 592 release() followed by init(), plus clearing the state */ 593 594 status_t err = release(); 595 596 // unregister handlers 597 if (mCodec != NULL) { 598 if (mCodecLooper != NULL) { 599 mCodecLooper->unregisterHandler(mCodec->id()); 600 } else { 601 mLooper->unregisterHandler(mCodec->id()); 602 } 603 mCodec = NULL; 604 } 605 mLooper->unregisterHandler(id()); 606 607 mFlags = 0; // clear all flags 608 mStickyError = OK; 609 610 // reset state not reset by setState(UNINITIALIZED) 611 mReplyID = 0; 612 mDequeueInputReplyID = 0; 613 mDequeueOutputReplyID = 0; 614 mDequeueInputTimeoutGeneration = 0; 615 mDequeueOutputTimeoutGeneration = 0; 616 mHaveInputSurface = false; 617 618 if (err == OK) { 619 err = init(mInitName, mInitNameIsType, mInitIsEncoder); 620 } 621 return err; 622 } 623 624 status_t MediaCodec::queueInputBuffer( 625 size_t index, 626 size_t offset, 627 size_t size, 628 int64_t presentationTimeUs, 629 uint32_t flags, 630 AString *errorDetailMsg) { 631 if (errorDetailMsg != NULL) { 632 errorDetailMsg->clear(); 633 } 634 635 sp<AMessage> msg = new AMessage(kWhatQueueInputBuffer, this); 636 msg->setSize("index", index); 637 msg->setSize("offset", offset); 638 msg->setSize("size", size); 639 msg->setInt64("timeUs", presentationTimeUs); 640 msg->setInt32("flags", flags); 641 msg->setPointer("errorDetailMsg", errorDetailMsg); 642 643 sp<AMessage> response; 644 return PostAndAwaitResponse(msg, &response); 645 } 646 647 status_t MediaCodec::queueSecureInputBuffer( 648 size_t index, 649 size_t offset, 650 const CryptoPlugin::SubSample *subSamples, 651 size_t numSubSamples, 652 const uint8_t key[16], 653 const uint8_t iv[16], 654 CryptoPlugin::Mode mode, 655 int64_t presentationTimeUs, 656 uint32_t flags, 657 AString *errorDetailMsg) { 658 if (errorDetailMsg != NULL) { 659 errorDetailMsg->clear(); 660 } 661 662 sp<AMessage> msg = new AMessage(kWhatQueueInputBuffer, this); 663 msg->setSize("index", index); 664 msg->setSize("offset", offset); 665 msg->setPointer("subSamples", (void *)subSamples); 666 msg->setSize("numSubSamples", numSubSamples); 667 msg->setPointer("key", (void *)key); 668 msg->setPointer("iv", (void *)iv); 669 msg->setInt32("mode", mode); 670 msg->setInt64("timeUs", presentationTimeUs); 671 msg->setInt32("flags", flags); 672 msg->setPointer("errorDetailMsg", errorDetailMsg); 673 674 sp<AMessage> response; 675 status_t err = PostAndAwaitResponse(msg, &response); 676 677 return err; 678 } 679 680 status_t MediaCodec::dequeueInputBuffer(size_t *index, int64_t timeoutUs) { 681 sp<AMessage> msg = new AMessage(kWhatDequeueInputBuffer, this); 682 msg->setInt64("timeoutUs", timeoutUs); 683 684 sp<AMessage> response; 685 status_t err; 686 if ((err = PostAndAwaitResponse(msg, &response)) != OK) { 687 return err; 688 } 689 690 CHECK(response->findSize("index", index)); 691 692 return OK; 693 } 694 695 status_t MediaCodec::dequeueOutputBuffer( 696 size_t *index, 697 size_t *offset, 698 size_t *size, 699 int64_t *presentationTimeUs, 700 uint32_t *flags, 701 int64_t timeoutUs) { 702 sp<AMessage> msg = new AMessage(kWhatDequeueOutputBuffer, this); 703 msg->setInt64("timeoutUs", timeoutUs); 704 705 sp<AMessage> response; 706 status_t err; 707 if ((err = PostAndAwaitResponse(msg, &response)) != OK) { 708 return err; 709 } 710 711 CHECK(response->findSize("index", index)); 712 CHECK(response->findSize("offset", offset)); 713 CHECK(response->findSize("size", size)); 714 CHECK(response->findInt64("timeUs", presentationTimeUs)); 715 CHECK(response->findInt32("flags", (int32_t *)flags)); 716 717 return OK; 718 } 719 720 status_t MediaCodec::renderOutputBufferAndRelease(size_t index) { 721 sp<AMessage> msg = new AMessage(kWhatReleaseOutputBuffer, this); 722 msg->setSize("index", index); 723 msg->setInt32("render", true); 724 725 sp<AMessage> response; 726 return PostAndAwaitResponse(msg, &response); 727 } 728 729 status_t MediaCodec::renderOutputBufferAndRelease(size_t index, int64_t timestampNs) { 730 sp<AMessage> msg = new AMessage(kWhatReleaseOutputBuffer, this); 731 msg->setSize("index", index); 732 msg->setInt32("render", true); 733 msg->setInt64("timestampNs", timestampNs); 734 735 sp<AMessage> response; 736 return PostAndAwaitResponse(msg, &response); 737 } 738 739 status_t MediaCodec::releaseOutputBuffer(size_t index) { 740 sp<AMessage> msg = new AMessage(kWhatReleaseOutputBuffer, this); 741 msg->setSize("index", index); 742 743 sp<AMessage> response; 744 return PostAndAwaitResponse(msg, &response); 745 } 746 747 status_t MediaCodec::signalEndOfInputStream() { 748 sp<AMessage> msg = new AMessage(kWhatSignalEndOfInputStream, this); 749 750 sp<AMessage> response; 751 return PostAndAwaitResponse(msg, &response); 752 } 753 754 status_t MediaCodec::getOutputFormat(sp<AMessage> *format) const { 755 sp<AMessage> msg = new AMessage(kWhatGetOutputFormat, this); 756 757 sp<AMessage> response; 758 status_t err; 759 if ((err = PostAndAwaitResponse(msg, &response)) != OK) { 760 return err; 761 } 762 763 CHECK(response->findMessage("format", format)); 764 765 return OK; 766 } 767 768 status_t MediaCodec::getInputFormat(sp<AMessage> *format) const { 769 sp<AMessage> msg = new AMessage(kWhatGetInputFormat, this); 770 771 sp<AMessage> response; 772 status_t err; 773 if ((err = PostAndAwaitResponse(msg, &response)) != OK) { 774 return err; 775 } 776 777 CHECK(response->findMessage("format", format)); 778 779 return OK; 780 } 781 782 status_t MediaCodec::getName(AString *name) const { 783 sp<AMessage> msg = new AMessage(kWhatGetName, this); 784 785 sp<AMessage> response; 786 status_t err; 787 if ((err = PostAndAwaitResponse(msg, &response)) != OK) { 788 return err; 789 } 790 791 CHECK(response->findString("name", name)); 792 793 return OK; 794 } 795 796 status_t MediaCodec::getWidevineLegacyBuffers(Vector<sp<ABuffer> > *buffers) const { 797 sp<AMessage> msg = new AMessage(kWhatGetBuffers, this); 798 msg->setInt32("portIndex", kPortIndexInput); 799 msg->setPointer("buffers", buffers); 800 msg->setInt32("widevine", true); 801 802 sp<AMessage> response; 803 return PostAndAwaitResponse(msg, &response); 804 } 805 806 status_t MediaCodec::getInputBuffers(Vector<sp<ABuffer> > *buffers) const { 807 sp<AMessage> msg = new AMessage(kWhatGetBuffers, this); 808 msg->setInt32("portIndex", kPortIndexInput); 809 msg->setPointer("buffers", buffers); 810 811 sp<AMessage> response; 812 return PostAndAwaitResponse(msg, &response); 813 } 814 815 status_t MediaCodec::getOutputBuffers(Vector<sp<ABuffer> > *buffers) const { 816 sp<AMessage> msg = new AMessage(kWhatGetBuffers, this); 817 msg->setInt32("portIndex", kPortIndexOutput); 818 msg->setPointer("buffers", buffers); 819 820 sp<AMessage> response; 821 return PostAndAwaitResponse(msg, &response); 822 } 823 824 status_t MediaCodec::getOutputBuffer(size_t index, sp<ABuffer> *buffer) { 825 sp<AMessage> format; 826 return getBufferAndFormat(kPortIndexOutput, index, buffer, &format); 827 } 828 829 status_t MediaCodec::getOutputFormat(size_t index, sp<AMessage> *format) { 830 sp<ABuffer> buffer; 831 return getBufferAndFormat(kPortIndexOutput, index, &buffer, format); 832 } 833 834 status_t MediaCodec::getInputBuffer(size_t index, sp<ABuffer> *buffer) { 835 sp<AMessage> format; 836 return getBufferAndFormat(kPortIndexInput, index, buffer, &format); 837 } 838 839 bool MediaCodec::isExecuting() const { 840 return mState == STARTED || mState == FLUSHED; 841 } 842 843 status_t MediaCodec::getBufferAndFormat( 844 size_t portIndex, size_t index, 845 sp<ABuffer> *buffer, sp<AMessage> *format) { 846 // use mutex instead of a context switch 847 848 if (mReleasedByResourceManager) { 849 return DEAD_OBJECT; 850 } 851 852 buffer->clear(); 853 format->clear(); 854 if (!isExecuting()) { 855 return INVALID_OPERATION; 856 } 857 858 // we do not want mPortBuffers to change during this section 859 // we also don't want mOwnedByClient to change during this 860 Mutex::Autolock al(mBufferLock); 861 Vector<BufferInfo> *buffers = &mPortBuffers[portIndex]; 862 if (index < buffers->size()) { 863 const BufferInfo &info = buffers->itemAt(index); 864 if (info.mOwnedByClient) { 865 // by the time buffers array is initialized, crypto is set 866 if (portIndex == kPortIndexInput && mCrypto != NULL) { 867 *buffer = info.mEncryptedData; 868 } else { 869 *buffer = info.mData; 870 } 871 *format = info.mFormat; 872 } 873 } 874 return OK; 875 } 876 877 status_t MediaCodec::flush() { 878 sp<AMessage> msg = new AMessage(kWhatFlush, this); 879 880 sp<AMessage> response; 881 return PostAndAwaitResponse(msg, &response); 882 } 883 884 status_t MediaCodec::requestIDRFrame() { 885 (new AMessage(kWhatRequestIDRFrame, this))->post(); 886 887 return OK; 888 } 889 890 void MediaCodec::requestActivityNotification(const sp<AMessage> ¬ify) { 891 sp<AMessage> msg = new AMessage(kWhatRequestActivityNotification, this); 892 msg->setMessage("notify", notify); 893 msg->post(); 894 } 895 896 //////////////////////////////////////////////////////////////////////////////// 897 898 void MediaCodec::cancelPendingDequeueOperations() { 899 if (mFlags & kFlagDequeueInputPending) { 900 PostReplyWithError(mDequeueInputReplyID, INVALID_OPERATION); 901 902 ++mDequeueInputTimeoutGeneration; 903 mDequeueInputReplyID = 0; 904 mFlags &= ~kFlagDequeueInputPending; 905 } 906 907 if (mFlags & kFlagDequeueOutputPending) { 908 PostReplyWithError(mDequeueOutputReplyID, INVALID_OPERATION); 909 910 ++mDequeueOutputTimeoutGeneration; 911 mDequeueOutputReplyID = 0; 912 mFlags &= ~kFlagDequeueOutputPending; 913 } 914 } 915 916 bool MediaCodec::handleDequeueInputBuffer(const sp<AReplyToken> &replyID, bool newRequest) { 917 if (!isExecuting() || (mFlags & kFlagIsAsync) 918 || (newRequest && (mFlags & kFlagDequeueInputPending))) { 919 PostReplyWithError(replyID, INVALID_OPERATION); 920 return true; 921 } else if (mFlags & kFlagStickyError) { 922 PostReplyWithError(replyID, getStickyError()); 923 return true; 924 } 925 926 ssize_t index = dequeuePortBuffer(kPortIndexInput); 927 928 if (index < 0) { 929 CHECK_EQ(index, -EAGAIN); 930 return false; 931 } 932 933 sp<AMessage> response = new AMessage; 934 response->setSize("index", index); 935 response->postReply(replyID); 936 937 return true; 938 } 939 940 bool MediaCodec::handleDequeueOutputBuffer(const sp<AReplyToken> &replyID, bool newRequest) { 941 if (!isExecuting() || (mFlags & kFlagIsAsync) 942 || (newRequest && (mFlags & kFlagDequeueOutputPending))) { 943 PostReplyWithError(replyID, INVALID_OPERATION); 944 } else if (mFlags & kFlagStickyError) { 945 PostReplyWithError(replyID, getStickyError()); 946 } else if (mFlags & kFlagOutputBuffersChanged) { 947 PostReplyWithError(replyID, INFO_OUTPUT_BUFFERS_CHANGED); 948 mFlags &= ~kFlagOutputBuffersChanged; 949 } else if (mFlags & kFlagOutputFormatChanged) { 950 PostReplyWithError(replyID, INFO_FORMAT_CHANGED); 951 mFlags &= ~kFlagOutputFormatChanged; 952 } else { 953 sp<AMessage> response = new AMessage; 954 ssize_t index = dequeuePortBuffer(kPortIndexOutput); 955 956 if (index < 0) { 957 CHECK_EQ(index, -EAGAIN); 958 return false; 959 } 960 961 const sp<ABuffer> &buffer = 962 mPortBuffers[kPortIndexOutput].itemAt(index).mData; 963 964 response->setSize("index", index); 965 response->setSize("offset", buffer->offset()); 966 response->setSize("size", buffer->size()); 967 968 int64_t timeUs; 969 CHECK(buffer->meta()->findInt64("timeUs", &timeUs)); 970 971 response->setInt64("timeUs", timeUs); 972 973 int32_t omxFlags; 974 CHECK(buffer->meta()->findInt32("omxFlags", &omxFlags)); 975 976 uint32_t flags = 0; 977 if (omxFlags & OMX_BUFFERFLAG_SYNCFRAME) { 978 flags |= BUFFER_FLAG_SYNCFRAME; 979 } 980 if (omxFlags & OMX_BUFFERFLAG_CODECCONFIG) { 981 flags |= BUFFER_FLAG_CODECCONFIG; 982 } 983 if (omxFlags & OMX_BUFFERFLAG_EOS) { 984 flags |= BUFFER_FLAG_EOS; 985 } 986 987 response->setInt32("flags", flags); 988 response->postReply(replyID); 989 } 990 991 return true; 992 } 993 994 void MediaCodec::onMessageReceived(const sp<AMessage> &msg) { 995 switch (msg->what()) { 996 case kWhatCodecNotify: 997 { 998 int32_t what; 999 CHECK(msg->findInt32("what", &what)); 1000 1001 switch (what) { 1002 case CodecBase::kWhatError: 1003 { 1004 int32_t err, actionCode; 1005 CHECK(msg->findInt32("err", &err)); 1006 CHECK(msg->findInt32("actionCode", &actionCode)); 1007 1008 ALOGE("Codec reported err %#x, actionCode %d, while in state %d", 1009 err, actionCode, mState); 1010 if (err == DEAD_OBJECT) { 1011 mFlags |= kFlagSawMediaServerDie; 1012 mFlags &= ~kFlagIsComponentAllocated; 1013 } 1014 1015 bool sendErrorResponse = true; 1016 1017 switch (mState) { 1018 case INITIALIZING: 1019 { 1020 setState(UNINITIALIZED); 1021 break; 1022 } 1023 1024 case CONFIGURING: 1025 { 1026 setState(actionCode == ACTION_CODE_FATAL ? 1027 UNINITIALIZED : INITIALIZED); 1028 break; 1029 } 1030 1031 case STARTING: 1032 { 1033 setState(actionCode == ACTION_CODE_FATAL ? 1034 UNINITIALIZED : CONFIGURED); 1035 break; 1036 } 1037 1038 case STOPPING: 1039 case RELEASING: 1040 { 1041 // Ignore the error, assuming we'll still get 1042 // the shutdown complete notification. 1043 1044 sendErrorResponse = false; 1045 1046 if (mFlags & kFlagSawMediaServerDie) { 1047 // MediaServer died, there definitely won't 1048 // be a shutdown complete notification after 1049 // all. 1050 1051 // note that we're directly going from 1052 // STOPPING->UNINITIALIZED, instead of the 1053 // usual STOPPING->INITIALIZED state. 1054 setState(UNINITIALIZED); 1055 if (mState == RELEASING) { 1056 mComponentName.clear(); 1057 } 1058 (new AMessage)->postReply(mReplyID); 1059 } 1060 break; 1061 } 1062 1063 case FLUSHING: 1064 { 1065 if (actionCode == ACTION_CODE_FATAL) { 1066 setState(UNINITIALIZED); 1067 } else { 1068 setState( 1069 (mFlags & kFlagIsAsync) ? FLUSHED : STARTED); 1070 } 1071 break; 1072 } 1073 1074 case FLUSHED: 1075 case STARTED: 1076 { 1077 sendErrorResponse = false; 1078 1079 setStickyError(err); 1080 postActivityNotificationIfPossible(); 1081 1082 cancelPendingDequeueOperations(); 1083 1084 if (mFlags & kFlagIsAsync) { 1085 onError(err, actionCode); 1086 } 1087 switch (actionCode) { 1088 case ACTION_CODE_TRANSIENT: 1089 break; 1090 case ACTION_CODE_RECOVERABLE: 1091 setState(INITIALIZED); 1092 break; 1093 default: 1094 setState(UNINITIALIZED); 1095 break; 1096 } 1097 break; 1098 } 1099 1100 default: 1101 { 1102 sendErrorResponse = false; 1103 1104 setStickyError(err); 1105 postActivityNotificationIfPossible(); 1106 1107 // actionCode in an uninitialized state is always fatal. 1108 if (mState == UNINITIALIZED) { 1109 actionCode = ACTION_CODE_FATAL; 1110 } 1111 if (mFlags & kFlagIsAsync) { 1112 onError(err, actionCode); 1113 } 1114 switch (actionCode) { 1115 case ACTION_CODE_TRANSIENT: 1116 break; 1117 case ACTION_CODE_RECOVERABLE: 1118 setState(INITIALIZED); 1119 break; 1120 default: 1121 setState(UNINITIALIZED); 1122 break; 1123 } 1124 break; 1125 } 1126 } 1127 1128 if (sendErrorResponse) { 1129 PostReplyWithError(mReplyID, err); 1130 } 1131 break; 1132 } 1133 1134 case CodecBase::kWhatComponentAllocated: 1135 { 1136 CHECK_EQ(mState, INITIALIZING); 1137 setState(INITIALIZED); 1138 mFlags |= kFlagIsComponentAllocated; 1139 1140 CHECK(msg->findString("componentName", &mComponentName)); 1141 1142 if (mComponentName.startsWith("OMX.google.")) { 1143 mFlags |= kFlagUsesSoftwareRenderer; 1144 } else { 1145 mFlags &= ~kFlagUsesSoftwareRenderer; 1146 } 1147 1148 String8 resourceType; 1149 if (mComponentName.endsWith(".secure")) { 1150 mFlags |= kFlagIsSecure; 1151 resourceType = String8(kResourceSecureCodec); 1152 } else { 1153 mFlags &= ~kFlagIsSecure; 1154 resourceType = String8(kResourceNonSecureCodec); 1155 } 1156 1157 const char *subtype = mIsVideo ? kResourceVideoCodec : kResourceAudioCodec; 1158 addResource(resourceType, String8(subtype), 1); 1159 1160 (new AMessage)->postReply(mReplyID); 1161 break; 1162 } 1163 1164 case CodecBase::kWhatComponentConfigured: 1165 { 1166 if (mState == UNINITIALIZED || mState == INITIALIZED) { 1167 // In case a kWhatError message came in and replied with error, 1168 // we log a warning and ignore. 1169 ALOGW("configure interrupted by error, current state %d", mState); 1170 break; 1171 } 1172 CHECK_EQ(mState, CONFIGURING); 1173 1174 // reset input surface flag 1175 mHaveInputSurface = false; 1176 1177 CHECK(msg->findMessage("input-format", &mInputFormat)); 1178 CHECK(msg->findMessage("output-format", &mOutputFormat)); 1179 1180 int32_t usingSwRenderer; 1181 if (mOutputFormat->findInt32("using-sw-renderer", &usingSwRenderer) 1182 && usingSwRenderer) { 1183 mFlags |= kFlagUsesSoftwareRenderer; 1184 } 1185 setState(CONFIGURED); 1186 (new AMessage)->postReply(mReplyID); 1187 break; 1188 } 1189 1190 case CodecBase::kWhatInputSurfaceCreated: 1191 { 1192 // response to initiateCreateInputSurface() 1193 status_t err = NO_ERROR; 1194 sp<AMessage> response = new AMessage; 1195 if (!msg->findInt32("err", &err)) { 1196 sp<RefBase> obj; 1197 msg->findObject("input-surface", &obj); 1198 CHECK(obj != NULL); 1199 response->setObject("input-surface", obj); 1200 mHaveInputSurface = true; 1201 } else { 1202 response->setInt32("err", err); 1203 } 1204 response->postReply(mReplyID); 1205 break; 1206 } 1207 1208 case CodecBase::kWhatInputSurfaceAccepted: 1209 { 1210 // response to initiateSetInputSurface() 1211 status_t err = NO_ERROR; 1212 sp<AMessage> response = new AMessage(); 1213 if (!msg->findInt32("err", &err)) { 1214 mHaveInputSurface = true; 1215 } else { 1216 response->setInt32("err", err); 1217 } 1218 response->postReply(mReplyID); 1219 break; 1220 } 1221 1222 case CodecBase::kWhatSignaledInputEOS: 1223 { 1224 // response to signalEndOfInputStream() 1225 sp<AMessage> response = new AMessage; 1226 status_t err; 1227 if (msg->findInt32("err", &err)) { 1228 response->setInt32("err", err); 1229 } 1230 response->postReply(mReplyID); 1231 break; 1232 } 1233 1234 1235 case CodecBase::kWhatBuffersAllocated: 1236 { 1237 Mutex::Autolock al(mBufferLock); 1238 int32_t portIndex; 1239 CHECK(msg->findInt32("portIndex", &portIndex)); 1240 1241 ALOGV("%s buffers allocated", 1242 portIndex == kPortIndexInput ? "input" : "output"); 1243 1244 CHECK(portIndex == kPortIndexInput 1245 || portIndex == kPortIndexOutput); 1246 1247 mPortBuffers[portIndex].clear(); 1248 1249 Vector<BufferInfo> *buffers = &mPortBuffers[portIndex]; 1250 1251 sp<RefBase> obj; 1252 CHECK(msg->findObject("portDesc", &obj)); 1253 1254 sp<CodecBase::PortDescription> portDesc = 1255 static_cast<CodecBase::PortDescription *>(obj.get()); 1256 1257 size_t numBuffers = portDesc->countBuffers(); 1258 1259 size_t totalSize = 0; 1260 for (size_t i = 0; i < numBuffers; ++i) { 1261 if (portIndex == kPortIndexInput && mCrypto != NULL) { 1262 totalSize += portDesc->bufferAt(i)->capacity(); 1263 } 1264 } 1265 1266 if (totalSize) { 1267 mDealer = new MemoryDealer(totalSize, "MediaCodec"); 1268 } 1269 1270 for (size_t i = 0; i < numBuffers; ++i) { 1271 BufferInfo info; 1272 info.mBufferID = portDesc->bufferIDAt(i); 1273 info.mOwnedByClient = false; 1274 info.mData = portDesc->bufferAt(i); 1275 1276 if (portIndex == kPortIndexInput && mCrypto != NULL) { 1277 sp<IMemory> mem = mDealer->allocate(info.mData->capacity()); 1278 info.mEncryptedData = 1279 new ABuffer(mem->pointer(), info.mData->capacity()); 1280 info.mSharedEncryptedBuffer = mem; 1281 } 1282 1283 buffers->push_back(info); 1284 } 1285 1286 if (portIndex == kPortIndexOutput) { 1287 if (mState == STARTING) { 1288 // We're always allocating output buffers after 1289 // allocating input buffers, so this is a good 1290 // indication that now all buffers are allocated. 1291 if (mIsVideo) { 1292 String8 subtype; 1293 addResource( 1294 String8(kResourceGraphicMemory), 1295 subtype, 1296 getGraphicBufferSize()); 1297 } 1298 setState(STARTED); 1299 (new AMessage)->postReply(mReplyID); 1300 } else { 1301 mFlags |= kFlagOutputBuffersChanged; 1302 postActivityNotificationIfPossible(); 1303 } 1304 } 1305 break; 1306 } 1307 1308 case CodecBase::kWhatOutputFormatChanged: 1309 { 1310 ALOGV("codec output format changed"); 1311 1312 if (mSoftRenderer == NULL && 1313 mSurface != NULL && 1314 (mFlags & kFlagUsesSoftwareRenderer)) { 1315 AString mime; 1316 CHECK(msg->findString("mime", &mime)); 1317 1318 if (mime.startsWithIgnoreCase("video/")) { 1319 mSoftRenderer = new SoftwareRenderer(mSurface, mRotationDegrees); 1320 } 1321 } 1322 1323 mOutputFormat = msg; 1324 1325 if (mFlags & kFlagIsEncoder) { 1326 // Before we announce the format change we should 1327 // collect codec specific data and amend the output 1328 // format as necessary. 1329 mFlags |= kFlagGatherCodecSpecificData; 1330 } else if (mFlags & kFlagIsAsync) { 1331 onOutputFormatChanged(); 1332 } else { 1333 mFlags |= kFlagOutputFormatChanged; 1334 postActivityNotificationIfPossible(); 1335 } 1336 1337 // Notify mCrypto of video resolution changes 1338 if (mCrypto != NULL) { 1339 int32_t left, top, right, bottom, width, height; 1340 if (mOutputFormat->findRect("crop", &left, &top, &right, &bottom)) { 1341 mCrypto->notifyResolution(right - left + 1, bottom - top + 1); 1342 } else if (mOutputFormat->findInt32("width", &width) 1343 && mOutputFormat->findInt32("height", &height)) { 1344 mCrypto->notifyResolution(width, height); 1345 } 1346 } 1347 1348 break; 1349 } 1350 1351 case CodecBase::kWhatOutputFramesRendered: 1352 { 1353 // ignore these in all states except running, and check that we have a 1354 // notification set 1355 if (mState == STARTED && mOnFrameRenderedNotification != NULL) { 1356 sp<AMessage> notify = mOnFrameRenderedNotification->dup(); 1357 notify->setMessage("data", msg); 1358 notify->post(); 1359 } 1360 break; 1361 } 1362 1363 case CodecBase::kWhatFillThisBuffer: 1364 { 1365 /* size_t index = */updateBuffers(kPortIndexInput, msg); 1366 1367 if (mState == FLUSHING 1368 || mState == STOPPING 1369 || mState == RELEASING) { 1370 returnBuffersToCodecOnPort(kPortIndexInput); 1371 break; 1372 } 1373 1374 if (!mCSD.empty()) { 1375 ssize_t index = dequeuePortBuffer(kPortIndexInput); 1376 CHECK_GE(index, 0); 1377 1378 // If codec specific data had been specified as 1379 // part of the format in the call to configure and 1380 // if there's more csd left, we submit it here 1381 // clients only get access to input buffers once 1382 // this data has been exhausted. 1383 1384 status_t err = queueCSDInputBuffer(index); 1385 1386 if (err != OK) { 1387 ALOGE("queueCSDInputBuffer failed w/ error %d", 1388 err); 1389 1390 setStickyError(err); 1391 postActivityNotificationIfPossible(); 1392 1393 cancelPendingDequeueOperations(); 1394 } 1395 break; 1396 } 1397 1398 if (mFlags & kFlagIsAsync) { 1399 if (!mHaveInputSurface) { 1400 if (mState == FLUSHED) { 1401 mHavePendingInputBuffers = true; 1402 } else { 1403 onInputBufferAvailable(); 1404 } 1405 } 1406 } else if (mFlags & kFlagDequeueInputPending) { 1407 CHECK(handleDequeueInputBuffer(mDequeueInputReplyID)); 1408 1409 ++mDequeueInputTimeoutGeneration; 1410 mFlags &= ~kFlagDequeueInputPending; 1411 mDequeueInputReplyID = 0; 1412 } else { 1413 postActivityNotificationIfPossible(); 1414 } 1415 break; 1416 } 1417 1418 case CodecBase::kWhatDrainThisBuffer: 1419 { 1420 /* size_t index = */updateBuffers(kPortIndexOutput, msg); 1421 1422 if (mState == FLUSHING 1423 || mState == STOPPING 1424 || mState == RELEASING) { 1425 returnBuffersToCodecOnPort(kPortIndexOutput); 1426 break; 1427 } 1428 1429 sp<ABuffer> buffer; 1430 CHECK(msg->findBuffer("buffer", &buffer)); 1431 1432 int32_t omxFlags; 1433 CHECK(msg->findInt32("flags", &omxFlags)); 1434 1435 buffer->meta()->setInt32("omxFlags", omxFlags); 1436 1437 if (mFlags & kFlagGatherCodecSpecificData) { 1438 // This is the very first output buffer after a 1439 // format change was signalled, it'll either contain 1440 // the one piece of codec specific data we can expect 1441 // or there won't be codec specific data. 1442 if (omxFlags & OMX_BUFFERFLAG_CODECCONFIG) { 1443 status_t err = 1444 amendOutputFormatWithCodecSpecificData(buffer); 1445 1446 if (err != OK) { 1447 ALOGE("Codec spit out malformed codec " 1448 "specific data!"); 1449 } 1450 } 1451 1452 mFlags &= ~kFlagGatherCodecSpecificData; 1453 if (mFlags & kFlagIsAsync) { 1454 onOutputFormatChanged(); 1455 } else { 1456 mFlags |= kFlagOutputFormatChanged; 1457 } 1458 } 1459 1460 if (mFlags & kFlagIsAsync) { 1461 onOutputBufferAvailable(); 1462 } else if (mFlags & kFlagDequeueOutputPending) { 1463 CHECK(handleDequeueOutputBuffer(mDequeueOutputReplyID)); 1464 1465 ++mDequeueOutputTimeoutGeneration; 1466 mFlags &= ~kFlagDequeueOutputPending; 1467 mDequeueOutputReplyID = 0; 1468 } else { 1469 postActivityNotificationIfPossible(); 1470 } 1471 1472 break; 1473 } 1474 1475 case CodecBase::kWhatEOS: 1476 { 1477 // We already notify the client of this by using the 1478 // corresponding flag in "onOutputBufferReady". 1479 break; 1480 } 1481 1482 case CodecBase::kWhatShutdownCompleted: 1483 { 1484 if (mState == STOPPING) { 1485 setState(INITIALIZED); 1486 } else { 1487 CHECK_EQ(mState, RELEASING); 1488 setState(UNINITIALIZED); 1489 mComponentName.clear(); 1490 } 1491 mFlags &= ~kFlagIsComponentAllocated; 1492 1493 mResourceManagerService->removeResource(getId(mResourceManagerClient)); 1494 1495 (new AMessage)->postReply(mReplyID); 1496 break; 1497 } 1498 1499 case CodecBase::kWhatFlushCompleted: 1500 { 1501 if (mState != FLUSHING) { 1502 ALOGW("received FlushCompleted message in state %d", 1503 mState); 1504 break; 1505 } 1506 1507 if (mFlags & kFlagIsAsync) { 1508 setState(FLUSHED); 1509 } else { 1510 setState(STARTED); 1511 mCodec->signalResume(); 1512 } 1513 1514 (new AMessage)->postReply(mReplyID); 1515 break; 1516 } 1517 1518 default: 1519 TRESPASS(); 1520 } 1521 break; 1522 } 1523 1524 case kWhatInit: 1525 { 1526 sp<AReplyToken> replyID; 1527 CHECK(msg->senderAwaitsResponse(&replyID)); 1528 1529 if (mState != UNINITIALIZED) { 1530 PostReplyWithError(replyID, INVALID_OPERATION); 1531 break; 1532 } 1533 1534 mReplyID = replyID; 1535 setState(INITIALIZING); 1536 1537 AString name; 1538 CHECK(msg->findString("name", &name)); 1539 1540 int32_t nameIsType; 1541 int32_t encoder = false; 1542 CHECK(msg->findInt32("nameIsType", &nameIsType)); 1543 if (nameIsType) { 1544 CHECK(msg->findInt32("encoder", &encoder)); 1545 } 1546 1547 sp<AMessage> format = new AMessage; 1548 1549 if (nameIsType) { 1550 format->setString("mime", name.c_str()); 1551 format->setInt32("encoder", encoder); 1552 } else { 1553 format->setString("componentName", name.c_str()); 1554 } 1555 1556 mCodec->initiateAllocateComponent(format); 1557 break; 1558 } 1559 1560 case kWhatSetNotification: 1561 { 1562 sp<AMessage> notify; 1563 if (msg->findMessage("on-frame-rendered", ¬ify)) { 1564 mOnFrameRenderedNotification = notify; 1565 } 1566 break; 1567 } 1568 1569 case kWhatSetCallback: 1570 { 1571 sp<AReplyToken> replyID; 1572 CHECK(msg->senderAwaitsResponse(&replyID)); 1573 1574 if (mState == UNINITIALIZED 1575 || mState == INITIALIZING 1576 || isExecuting()) { 1577 // callback can't be set after codec is executing, 1578 // or before it's initialized (as the callback 1579 // will be cleared when it goes to INITIALIZED) 1580 PostReplyWithError(replyID, INVALID_OPERATION); 1581 break; 1582 } 1583 1584 sp<AMessage> callback; 1585 CHECK(msg->findMessage("callback", &callback)); 1586 1587 mCallback = callback; 1588 1589 if (mCallback != NULL) { 1590 ALOGI("MediaCodec will operate in async mode"); 1591 mFlags |= kFlagIsAsync; 1592 } else { 1593 mFlags &= ~kFlagIsAsync; 1594 } 1595 1596 sp<AMessage> response = new AMessage; 1597 response->postReply(replyID); 1598 break; 1599 } 1600 1601 case kWhatConfigure: 1602 { 1603 sp<AReplyToken> replyID; 1604 CHECK(msg->senderAwaitsResponse(&replyID)); 1605 1606 if (mState != INITIALIZED) { 1607 PostReplyWithError(replyID, INVALID_OPERATION); 1608 break; 1609 } 1610 1611 sp<RefBase> obj; 1612 CHECK(msg->findObject("surface", &obj)); 1613 1614 sp<AMessage> format; 1615 CHECK(msg->findMessage("format", &format)); 1616 1617 int32_t push; 1618 if (msg->findInt32("push-blank-buffers-on-shutdown", &push) && push != 0) { 1619 mFlags |= kFlagPushBlankBuffersOnShutdown; 1620 } 1621 1622 if (obj != NULL) { 1623 format->setObject("native-window", obj); 1624 status_t err = handleSetSurface(static_cast<Surface *>(obj.get())); 1625 if (err != OK) { 1626 PostReplyWithError(replyID, err); 1627 break; 1628 } 1629 } else { 1630 handleSetSurface(NULL); 1631 } 1632 1633 mReplyID = replyID; 1634 setState(CONFIGURING); 1635 1636 void *crypto; 1637 if (!msg->findPointer("crypto", &crypto)) { 1638 crypto = NULL; 1639 } 1640 1641 mCrypto = static_cast<ICrypto *>(crypto); 1642 1643 uint32_t flags; 1644 CHECK(msg->findInt32("flags", (int32_t *)&flags)); 1645 1646 if (flags & CONFIGURE_FLAG_ENCODE) { 1647 format->setInt32("encoder", true); 1648 mFlags |= kFlagIsEncoder; 1649 } 1650 1651 extractCSD(format); 1652 1653 mCodec->initiateConfigureComponent(format); 1654 break; 1655 } 1656 1657 case kWhatSetSurface: 1658 { 1659 sp<AReplyToken> replyID; 1660 CHECK(msg->senderAwaitsResponse(&replyID)); 1661 1662 status_t err = OK; 1663 sp<Surface> surface; 1664 1665 switch (mState) { 1666 case CONFIGURED: 1667 case STARTED: 1668 case FLUSHED: 1669 { 1670 sp<RefBase> obj; 1671 (void)msg->findObject("surface", &obj); 1672 sp<Surface> surface = static_cast<Surface *>(obj.get()); 1673 if (mSurface == NULL) { 1674 // do not support setting surface if it was not set 1675 err = INVALID_OPERATION; 1676 } else if (obj == NULL) { 1677 // do not support unsetting surface 1678 err = BAD_VALUE; 1679 } else { 1680 err = connectToSurface(surface); 1681 if (err == BAD_VALUE) { 1682 // assuming reconnecting to same surface 1683 // TODO: check if it is the same surface 1684 err = OK; 1685 } else { 1686 if (err == OK) { 1687 if (mFlags & kFlagUsesSoftwareRenderer) { 1688 if (mSoftRenderer != NULL 1689 && (mFlags & kFlagPushBlankBuffersOnShutdown)) { 1690 pushBlankBuffersToNativeWindow(mSurface.get()); 1691 } 1692 mSoftRenderer = new SoftwareRenderer(surface); 1693 // TODO: check if this was successful 1694 } else { 1695 err = mCodec->setSurface(surface); 1696 } 1697 } 1698 if (err == OK) { 1699 (void)disconnectFromSurface(); 1700 mSurface = surface; 1701 } 1702 } 1703 } 1704 break; 1705 } 1706 1707 default: 1708 err = INVALID_OPERATION; 1709 break; 1710 } 1711 1712 PostReplyWithError(replyID, err); 1713 break; 1714 } 1715 1716 case kWhatCreateInputSurface: 1717 case kWhatSetInputSurface: 1718 { 1719 sp<AReplyToken> replyID; 1720 CHECK(msg->senderAwaitsResponse(&replyID)); 1721 1722 // Must be configured, but can't have been started yet. 1723 if (mState != CONFIGURED) { 1724 PostReplyWithError(replyID, INVALID_OPERATION); 1725 break; 1726 } 1727 1728 mReplyID = replyID; 1729 if (msg->what() == kWhatCreateInputSurface) { 1730 mCodec->initiateCreateInputSurface(); 1731 } else { 1732 sp<RefBase> obj; 1733 CHECK(msg->findObject("input-surface", &obj)); 1734 1735 mCodec->initiateSetInputSurface( 1736 static_cast<PersistentSurface *>(obj.get())); 1737 } 1738 break; 1739 } 1740 case kWhatStart: 1741 { 1742 sp<AReplyToken> replyID; 1743 CHECK(msg->senderAwaitsResponse(&replyID)); 1744 1745 if (mState == FLUSHED) { 1746 setState(STARTED); 1747 if (mHavePendingInputBuffers) { 1748 onInputBufferAvailable(); 1749 mHavePendingInputBuffers = false; 1750 } 1751 mCodec->signalResume(); 1752 PostReplyWithError(replyID, OK); 1753 break; 1754 } else if (mState != CONFIGURED) { 1755 PostReplyWithError(replyID, INVALID_OPERATION); 1756 break; 1757 } 1758 1759 mReplyID = replyID; 1760 setState(STARTING); 1761 1762 mCodec->initiateStart(); 1763 break; 1764 } 1765 1766 case kWhatStop: 1767 case kWhatRelease: 1768 { 1769 State targetState = 1770 (msg->what() == kWhatStop) ? INITIALIZED : UNINITIALIZED; 1771 1772 sp<AReplyToken> replyID; 1773 CHECK(msg->senderAwaitsResponse(&replyID)); 1774 1775 // already stopped/released 1776 if (mState == UNINITIALIZED && mReleasedByResourceManager) { 1777 sp<AMessage> response = new AMessage; 1778 response->setInt32("err", OK); 1779 response->postReply(replyID); 1780 break; 1781 } 1782 1783 int32_t reclaimed = 0; 1784 msg->findInt32("reclaimed", &reclaimed); 1785 if (reclaimed) { 1786 mReleasedByResourceManager = true; 1787 } 1788 1789 if (!((mFlags & kFlagIsComponentAllocated) && targetState == UNINITIALIZED) // See 1 1790 && mState != INITIALIZED 1791 && mState != CONFIGURED && !isExecuting()) { 1792 // 1) Permit release to shut down the component if allocated. 1793 // 1794 // 2) We may be in "UNINITIALIZED" state already and 1795 // also shutdown the encoder/decoder without the 1796 // client being aware of this if media server died while 1797 // we were being stopped. The client would assume that 1798 // after stop() returned, it would be safe to call release() 1799 // and it should be in this case, no harm to allow a release() 1800 // if we're already uninitialized. 1801 sp<AMessage> response = new AMessage; 1802 // TODO: we shouldn't throw an exception for stop/release. Change this to wait until 1803 // the previous stop/release completes and then reply with OK. 1804 status_t err = mState == targetState ? OK : INVALID_OPERATION; 1805 response->setInt32("err", err); 1806 if (err == OK && targetState == UNINITIALIZED) { 1807 mComponentName.clear(); 1808 } 1809 response->postReply(replyID); 1810 break; 1811 } 1812 1813 if (mFlags & kFlagSawMediaServerDie) { 1814 // It's dead, Jim. Don't expect initiateShutdown to yield 1815 // any useful results now... 1816 setState(UNINITIALIZED); 1817 if (targetState == UNINITIALIZED) { 1818 mComponentName.clear(); 1819 } 1820 (new AMessage)->postReply(replyID); 1821 break; 1822 } 1823 1824 mReplyID = replyID; 1825 setState(msg->what() == kWhatStop ? STOPPING : RELEASING); 1826 1827 mCodec->initiateShutdown( 1828 msg->what() == kWhatStop /* keepComponentAllocated */); 1829 1830 returnBuffersToCodec(); 1831 1832 if (mSoftRenderer != NULL && (mFlags & kFlagPushBlankBuffersOnShutdown)) { 1833 pushBlankBuffersToNativeWindow(mSurface.get()); 1834 } 1835 break; 1836 } 1837 1838 case kWhatDequeueInputBuffer: 1839 { 1840 sp<AReplyToken> replyID; 1841 CHECK(msg->senderAwaitsResponse(&replyID)); 1842 1843 if (mFlags & kFlagIsAsync) { 1844 ALOGE("dequeueOutputBuffer can't be used in async mode"); 1845 PostReplyWithError(replyID, INVALID_OPERATION); 1846 break; 1847 } 1848 1849 if (mHaveInputSurface) { 1850 ALOGE("dequeueInputBuffer can't be used with input surface"); 1851 PostReplyWithError(replyID, INVALID_OPERATION); 1852 break; 1853 } 1854 1855 if (handleDequeueInputBuffer(replyID, true /* new request */)) { 1856 break; 1857 } 1858 1859 int64_t timeoutUs; 1860 CHECK(msg->findInt64("timeoutUs", &timeoutUs)); 1861 1862 if (timeoutUs == 0ll) { 1863 PostReplyWithError(replyID, -EAGAIN); 1864 break; 1865 } 1866 1867 mFlags |= kFlagDequeueInputPending; 1868 mDequeueInputReplyID = replyID; 1869 1870 if (timeoutUs > 0ll) { 1871 sp<AMessage> timeoutMsg = 1872 new AMessage(kWhatDequeueInputTimedOut, this); 1873 timeoutMsg->setInt32( 1874 "generation", ++mDequeueInputTimeoutGeneration); 1875 timeoutMsg->post(timeoutUs); 1876 } 1877 break; 1878 } 1879 1880 case kWhatDequeueInputTimedOut: 1881 { 1882 int32_t generation; 1883 CHECK(msg->findInt32("generation", &generation)); 1884 1885 if (generation != mDequeueInputTimeoutGeneration) { 1886 // Obsolete 1887 break; 1888 } 1889 1890 CHECK(mFlags & kFlagDequeueInputPending); 1891 1892 PostReplyWithError(mDequeueInputReplyID, -EAGAIN); 1893 1894 mFlags &= ~kFlagDequeueInputPending; 1895 mDequeueInputReplyID = 0; 1896 break; 1897 } 1898 1899 case kWhatQueueInputBuffer: 1900 { 1901 sp<AReplyToken> replyID; 1902 CHECK(msg->senderAwaitsResponse(&replyID)); 1903 1904 if (!isExecuting()) { 1905 PostReplyWithError(replyID, INVALID_OPERATION); 1906 break; 1907 } else if (mFlags & kFlagStickyError) { 1908 PostReplyWithError(replyID, getStickyError()); 1909 break; 1910 } 1911 1912 status_t err = onQueueInputBuffer(msg); 1913 1914 PostReplyWithError(replyID, err); 1915 break; 1916 } 1917 1918 case kWhatDequeueOutputBuffer: 1919 { 1920 sp<AReplyToken> replyID; 1921 CHECK(msg->senderAwaitsResponse(&replyID)); 1922 1923 if (mFlags & kFlagIsAsync) { 1924 ALOGE("dequeueOutputBuffer can't be used in async mode"); 1925 PostReplyWithError(replyID, INVALID_OPERATION); 1926 break; 1927 } 1928 1929 if (handleDequeueOutputBuffer(replyID, true /* new request */)) { 1930 break; 1931 } 1932 1933 int64_t timeoutUs; 1934 CHECK(msg->findInt64("timeoutUs", &timeoutUs)); 1935 1936 if (timeoutUs == 0ll) { 1937 PostReplyWithError(replyID, -EAGAIN); 1938 break; 1939 } 1940 1941 mFlags |= kFlagDequeueOutputPending; 1942 mDequeueOutputReplyID = replyID; 1943 1944 if (timeoutUs > 0ll) { 1945 sp<AMessage> timeoutMsg = 1946 new AMessage(kWhatDequeueOutputTimedOut, this); 1947 timeoutMsg->setInt32( 1948 "generation", ++mDequeueOutputTimeoutGeneration); 1949 timeoutMsg->post(timeoutUs); 1950 } 1951 break; 1952 } 1953 1954 case kWhatDequeueOutputTimedOut: 1955 { 1956 int32_t generation; 1957 CHECK(msg->findInt32("generation", &generation)); 1958 1959 if (generation != mDequeueOutputTimeoutGeneration) { 1960 // Obsolete 1961 break; 1962 } 1963 1964 CHECK(mFlags & kFlagDequeueOutputPending); 1965 1966 PostReplyWithError(mDequeueOutputReplyID, -EAGAIN); 1967 1968 mFlags &= ~kFlagDequeueOutputPending; 1969 mDequeueOutputReplyID = 0; 1970 break; 1971 } 1972 1973 case kWhatReleaseOutputBuffer: 1974 { 1975 sp<AReplyToken> replyID; 1976 CHECK(msg->senderAwaitsResponse(&replyID)); 1977 1978 if (!isExecuting()) { 1979 PostReplyWithError(replyID, INVALID_OPERATION); 1980 break; 1981 } else if (mFlags & kFlagStickyError) { 1982 PostReplyWithError(replyID, getStickyError()); 1983 break; 1984 } 1985 1986 status_t err = onReleaseOutputBuffer(msg); 1987 1988 PostReplyWithError(replyID, err); 1989 break; 1990 } 1991 1992 case kWhatSignalEndOfInputStream: 1993 { 1994 sp<AReplyToken> replyID; 1995 CHECK(msg->senderAwaitsResponse(&replyID)); 1996 1997 if (!isExecuting()) { 1998 PostReplyWithError(replyID, INVALID_OPERATION); 1999 break; 2000 } else if (mFlags & kFlagStickyError) { 2001 PostReplyWithError(replyID, getStickyError()); 2002 break; 2003 } 2004 2005 mReplyID = replyID; 2006 mCodec->signalEndOfInputStream(); 2007 break; 2008 } 2009 2010 case kWhatGetBuffers: 2011 { 2012 sp<AReplyToken> replyID; 2013 CHECK(msg->senderAwaitsResponse(&replyID)); 2014 // Unfortunately widevine legacy source requires knowing all of the 2015 // codec input buffers, so we have to provide them even in async mode. 2016 int32_t widevine = 0; 2017 msg->findInt32("widevine", &widevine); 2018 2019 if (!isExecuting() || ((mFlags & kFlagIsAsync) && !widevine)) { 2020 PostReplyWithError(replyID, INVALID_OPERATION); 2021 break; 2022 } else if (mFlags & kFlagStickyError) { 2023 PostReplyWithError(replyID, getStickyError()); 2024 break; 2025 } 2026 2027 int32_t portIndex; 2028 CHECK(msg->findInt32("portIndex", &portIndex)); 2029 2030 Vector<sp<ABuffer> > *dstBuffers; 2031 CHECK(msg->findPointer("buffers", (void **)&dstBuffers)); 2032 2033 dstBuffers->clear(); 2034 const Vector<BufferInfo> &srcBuffers = mPortBuffers[portIndex]; 2035 2036 for (size_t i = 0; i < srcBuffers.size(); ++i) { 2037 const BufferInfo &info = srcBuffers.itemAt(i); 2038 2039 dstBuffers->push_back( 2040 (portIndex == kPortIndexInput && mCrypto != NULL) 2041 ? info.mEncryptedData : info.mData); 2042 } 2043 2044 (new AMessage)->postReply(replyID); 2045 break; 2046 } 2047 2048 case kWhatFlush: 2049 { 2050 sp<AReplyToken> replyID; 2051 CHECK(msg->senderAwaitsResponse(&replyID)); 2052 2053 if (!isExecuting()) { 2054 PostReplyWithError(replyID, INVALID_OPERATION); 2055 break; 2056 } else if (mFlags & kFlagStickyError) { 2057 PostReplyWithError(replyID, getStickyError()); 2058 break; 2059 } 2060 2061 mReplyID = replyID; 2062 // TODO: skip flushing if already FLUSHED 2063 setState(FLUSHING); 2064 2065 mCodec->signalFlush(); 2066 returnBuffersToCodec(); 2067 break; 2068 } 2069 2070 case kWhatGetInputFormat: 2071 case kWhatGetOutputFormat: 2072 { 2073 sp<AMessage> format = 2074 (msg->what() == kWhatGetOutputFormat ? mOutputFormat : mInputFormat); 2075 2076 sp<AReplyToken> replyID; 2077 CHECK(msg->senderAwaitsResponse(&replyID)); 2078 2079 if ((mState != CONFIGURED && mState != STARTING && 2080 mState != STARTED && mState != FLUSHING && 2081 mState != FLUSHED) 2082 || format == NULL) { 2083 PostReplyWithError(replyID, INVALID_OPERATION); 2084 break; 2085 } else if (mFlags & kFlagStickyError) { 2086 PostReplyWithError(replyID, getStickyError()); 2087 break; 2088 } 2089 2090 sp<AMessage> response = new AMessage; 2091 response->setMessage("format", format); 2092 response->postReply(replyID); 2093 break; 2094 } 2095 2096 case kWhatRequestIDRFrame: 2097 { 2098 mCodec->signalRequestIDRFrame(); 2099 break; 2100 } 2101 2102 case kWhatRequestActivityNotification: 2103 { 2104 CHECK(mActivityNotify == NULL); 2105 CHECK(msg->findMessage("notify", &mActivityNotify)); 2106 2107 postActivityNotificationIfPossible(); 2108 break; 2109 } 2110 2111 case kWhatGetName: 2112 { 2113 sp<AReplyToken> replyID; 2114 CHECK(msg->senderAwaitsResponse(&replyID)); 2115 2116 if (mComponentName.empty()) { 2117 PostReplyWithError(replyID, INVALID_OPERATION); 2118 break; 2119 } 2120 2121 sp<AMessage> response = new AMessage; 2122 response->setString("name", mComponentName.c_str()); 2123 response->postReply(replyID); 2124 break; 2125 } 2126 2127 case kWhatSetParameters: 2128 { 2129 sp<AReplyToken> replyID; 2130 CHECK(msg->senderAwaitsResponse(&replyID)); 2131 2132 sp<AMessage> params; 2133 CHECK(msg->findMessage("params", ¶ms)); 2134 2135 status_t err = onSetParameters(params); 2136 2137 PostReplyWithError(replyID, err); 2138 break; 2139 } 2140 2141 default: 2142 TRESPASS(); 2143 } 2144 } 2145 2146 void MediaCodec::extractCSD(const sp<AMessage> &format) { 2147 mCSD.clear(); 2148 2149 size_t i = 0; 2150 for (;;) { 2151 sp<ABuffer> csd; 2152 if (!format->findBuffer(AStringPrintf("csd-%u", i).c_str(), &csd)) { 2153 break; 2154 } 2155 2156 mCSD.push_back(csd); 2157 ++i; 2158 } 2159 2160 ALOGV("Found %zu pieces of codec specific data.", mCSD.size()); 2161 } 2162 2163 status_t MediaCodec::queueCSDInputBuffer(size_t bufferIndex) { 2164 CHECK(!mCSD.empty()); 2165 2166 const BufferInfo *info = 2167 &mPortBuffers[kPortIndexInput].itemAt(bufferIndex); 2168 2169 sp<ABuffer> csd = *mCSD.begin(); 2170 mCSD.erase(mCSD.begin()); 2171 2172 const sp<ABuffer> &codecInputData = 2173 (mCrypto != NULL) ? info->mEncryptedData : info->mData; 2174 2175 if (csd->size() > codecInputData->capacity()) { 2176 return -EINVAL; 2177 } 2178 2179 memcpy(codecInputData->data(), csd->data(), csd->size()); 2180 2181 AString errorDetailMsg; 2182 2183 sp<AMessage> msg = new AMessage(kWhatQueueInputBuffer, this); 2184 msg->setSize("index", bufferIndex); 2185 msg->setSize("offset", 0); 2186 msg->setSize("size", csd->size()); 2187 msg->setInt64("timeUs", 0ll); 2188 msg->setInt32("flags", BUFFER_FLAG_CODECCONFIG); 2189 msg->setPointer("errorDetailMsg", &errorDetailMsg); 2190 2191 return onQueueInputBuffer(msg); 2192 } 2193 2194 void MediaCodec::setState(State newState) { 2195 if (newState == INITIALIZED || newState == UNINITIALIZED) { 2196 delete mSoftRenderer; 2197 mSoftRenderer = NULL; 2198 2199 mCrypto.clear(); 2200 handleSetSurface(NULL); 2201 2202 mInputFormat.clear(); 2203 mOutputFormat.clear(); 2204 mFlags &= ~kFlagOutputFormatChanged; 2205 mFlags &= ~kFlagOutputBuffersChanged; 2206 mFlags &= ~kFlagStickyError; 2207 mFlags &= ~kFlagIsEncoder; 2208 mFlags &= ~kFlagGatherCodecSpecificData; 2209 mFlags &= ~kFlagIsAsync; 2210 mStickyError = OK; 2211 2212 mActivityNotify.clear(); 2213 mCallback.clear(); 2214 } 2215 2216 if (newState == UNINITIALIZED) { 2217 // return any straggling buffers, e.g. if we got here on an error 2218 returnBuffersToCodec(); 2219 2220 // The component is gone, mediaserver's probably back up already 2221 // but should definitely be back up should we try to instantiate 2222 // another component.. and the cycle continues. 2223 mFlags &= ~kFlagSawMediaServerDie; 2224 } 2225 2226 mState = newState; 2227 2228 cancelPendingDequeueOperations(); 2229 2230 updateBatteryStat(); 2231 } 2232 2233 void MediaCodec::returnBuffersToCodec() { 2234 returnBuffersToCodecOnPort(kPortIndexInput); 2235 returnBuffersToCodecOnPort(kPortIndexOutput); 2236 } 2237 2238 void MediaCodec::returnBuffersToCodecOnPort(int32_t portIndex) { 2239 CHECK(portIndex == kPortIndexInput || portIndex == kPortIndexOutput); 2240 Mutex::Autolock al(mBufferLock); 2241 2242 Vector<BufferInfo> *buffers = &mPortBuffers[portIndex]; 2243 2244 for (size_t i = 0; i < buffers->size(); ++i) { 2245 BufferInfo *info = &buffers->editItemAt(i); 2246 2247 if (info->mNotify != NULL) { 2248 sp<AMessage> msg = info->mNotify; 2249 info->mNotify = NULL; 2250 info->mOwnedByClient = false; 2251 2252 if (portIndex == kPortIndexInput) { 2253 /* no error, just returning buffers */ 2254 msg->setInt32("err", OK); 2255 } 2256 msg->post(); 2257 } 2258 } 2259 2260 mAvailPortBuffers[portIndex].clear(); 2261 } 2262 2263 size_t MediaCodec::updateBuffers( 2264 int32_t portIndex, const sp<AMessage> &msg) { 2265 CHECK(portIndex == kPortIndexInput || portIndex == kPortIndexOutput); 2266 2267 uint32_t bufferID; 2268 CHECK(msg->findInt32("buffer-id", (int32_t*)&bufferID)); 2269 2270 Vector<BufferInfo> *buffers = &mPortBuffers[portIndex]; 2271 2272 for (size_t i = 0; i < buffers->size(); ++i) { 2273 BufferInfo *info = &buffers->editItemAt(i); 2274 2275 if (info->mBufferID == bufferID) { 2276 CHECK(info->mNotify == NULL); 2277 CHECK(msg->findMessage("reply", &info->mNotify)); 2278 2279 info->mFormat = 2280 (portIndex == kPortIndexInput) ? mInputFormat : mOutputFormat; 2281 mAvailPortBuffers[portIndex].push_back(i); 2282 2283 return i; 2284 } 2285 } 2286 2287 TRESPASS(); 2288 2289 return 0; 2290 } 2291 2292 status_t MediaCodec::onQueueInputBuffer(const sp<AMessage> &msg) { 2293 size_t index; 2294 size_t offset; 2295 size_t size; 2296 int64_t timeUs; 2297 uint32_t flags; 2298 CHECK(msg->findSize("index", &index)); 2299 CHECK(msg->findSize("offset", &offset)); 2300 CHECK(msg->findInt64("timeUs", &timeUs)); 2301 CHECK(msg->findInt32("flags", (int32_t *)&flags)); 2302 2303 const CryptoPlugin::SubSample *subSamples; 2304 size_t numSubSamples; 2305 const uint8_t *key; 2306 const uint8_t *iv; 2307 CryptoPlugin::Mode mode = CryptoPlugin::kMode_Unencrypted; 2308 2309 // We allow the simpler queueInputBuffer API to be used even in 2310 // secure mode, by fabricating a single unencrypted subSample. 2311 CryptoPlugin::SubSample ss; 2312 2313 if (msg->findSize("size", &size)) { 2314 if (mCrypto != NULL) { 2315 ss.mNumBytesOfClearData = size; 2316 ss.mNumBytesOfEncryptedData = 0; 2317 2318 subSamples = &ss; 2319 numSubSamples = 1; 2320 key = NULL; 2321 iv = NULL; 2322 } 2323 } else { 2324 if (mCrypto == NULL) { 2325 return -EINVAL; 2326 } 2327 2328 CHECK(msg->findPointer("subSamples", (void **)&subSamples)); 2329 CHECK(msg->findSize("numSubSamples", &numSubSamples)); 2330 CHECK(msg->findPointer("key", (void **)&key)); 2331 CHECK(msg->findPointer("iv", (void **)&iv)); 2332 2333 int32_t tmp; 2334 CHECK(msg->findInt32("mode", &tmp)); 2335 2336 mode = (CryptoPlugin::Mode)tmp; 2337 2338 size = 0; 2339 for (size_t i = 0; i < numSubSamples; ++i) { 2340 size += subSamples[i].mNumBytesOfClearData; 2341 size += subSamples[i].mNumBytesOfEncryptedData; 2342 } 2343 } 2344 2345 if (index >= mPortBuffers[kPortIndexInput].size()) { 2346 return -ERANGE; 2347 } 2348 2349 BufferInfo *info = &mPortBuffers[kPortIndexInput].editItemAt(index); 2350 2351 if (info->mNotify == NULL || !info->mOwnedByClient) { 2352 return -EACCES; 2353 } 2354 2355 if (offset + size > info->mData->capacity()) { 2356 return -EINVAL; 2357 } 2358 2359 sp<AMessage> reply = info->mNotify; 2360 info->mData->setRange(offset, size); 2361 info->mData->meta()->setInt64("timeUs", timeUs); 2362 2363 if (flags & BUFFER_FLAG_EOS) { 2364 info->mData->meta()->setInt32("eos", true); 2365 } 2366 2367 if (flags & BUFFER_FLAG_CODECCONFIG) { 2368 info->mData->meta()->setInt32("csd", true); 2369 } 2370 2371 if (mCrypto != NULL) { 2372 if (size > info->mEncryptedData->capacity()) { 2373 return -ERANGE; 2374 } 2375 2376 AString *errorDetailMsg; 2377 CHECK(msg->findPointer("errorDetailMsg", (void **)&errorDetailMsg)); 2378 2379 ssize_t result = mCrypto->decrypt( 2380 (mFlags & kFlagIsSecure) != 0, 2381 key, 2382 iv, 2383 mode, 2384 info->mSharedEncryptedBuffer, 2385 offset, 2386 subSamples, 2387 numSubSamples, 2388 info->mData->base(), 2389 errorDetailMsg); 2390 2391 if (result < 0) { 2392 return result; 2393 } 2394 2395 info->mData->setRange(0, result); 2396 } 2397 2398 // synchronization boundary for getBufferAndFormat 2399 { 2400 Mutex::Autolock al(mBufferLock); 2401 info->mOwnedByClient = false; 2402 } 2403 reply->setBuffer("buffer", info->mData); 2404 reply->post(); 2405 2406 info->mNotify = NULL; 2407 2408 return OK; 2409 } 2410 2411 //static 2412 size_t MediaCodec::CreateFramesRenderedMessage( 2413 std::list<FrameRenderTracker::Info> done, sp<AMessage> &msg) { 2414 size_t index = 0; 2415 2416 for (std::list<FrameRenderTracker::Info>::const_iterator it = done.cbegin(); 2417 it != done.cend(); ++it) { 2418 if (it->getRenderTimeNs() < 0) { 2419 continue; // dropped frame from tracking 2420 } 2421 msg->setInt64(AStringPrintf("%zu-media-time-us", index).c_str(), it->getMediaTimeUs()); 2422 msg->setInt64(AStringPrintf("%zu-system-nano", index).c_str(), it->getRenderTimeNs()); 2423 ++index; 2424 } 2425 return index; 2426 } 2427 2428 status_t MediaCodec::onReleaseOutputBuffer(const sp<AMessage> &msg) { 2429 size_t index; 2430 CHECK(msg->findSize("index", &index)); 2431 2432 int32_t render; 2433 if (!msg->findInt32("render", &render)) { 2434 render = 0; 2435 } 2436 2437 if (!isExecuting()) { 2438 return -EINVAL; 2439 } 2440 2441 if (index >= mPortBuffers[kPortIndexOutput].size()) { 2442 return -ERANGE; 2443 } 2444 2445 BufferInfo *info = &mPortBuffers[kPortIndexOutput].editItemAt(index); 2446 2447 if (info->mNotify == NULL || !info->mOwnedByClient) { 2448 return -EACCES; 2449 } 2450 2451 // synchronization boundary for getBufferAndFormat 2452 { 2453 Mutex::Autolock al(mBufferLock); 2454 info->mOwnedByClient = false; 2455 } 2456 2457 if (render && info->mData != NULL && info->mData->size() != 0) { 2458 info->mNotify->setInt32("render", true); 2459 2460 int64_t mediaTimeUs = -1; 2461 info->mData->meta()->findInt64("timeUs", &mediaTimeUs); 2462 2463 int64_t renderTimeNs = 0; 2464 if (!msg->findInt64("timestampNs", &renderTimeNs)) { 2465 // use media timestamp if client did not request a specific render timestamp 2466 ALOGV("using buffer PTS of %lld", (long long)mediaTimeUs); 2467 renderTimeNs = mediaTimeUs * 1000; 2468 } 2469 info->mNotify->setInt64("timestampNs", renderTimeNs); 2470 2471 if (mSoftRenderer != NULL) { 2472 std::list<FrameRenderTracker::Info> doneFrames = mSoftRenderer->render( 2473 info->mData->data(), info->mData->size(), 2474 mediaTimeUs, renderTimeNs, NULL, info->mFormat); 2475 2476 // if we are running, notify rendered frames 2477 if (!doneFrames.empty() && mState == STARTED && mOnFrameRenderedNotification != NULL) { 2478 sp<AMessage> notify = mOnFrameRenderedNotification->dup(); 2479 sp<AMessage> data = new AMessage; 2480 if (CreateFramesRenderedMessage(doneFrames, data)) { 2481 notify->setMessage("data", data); 2482 notify->post(); 2483 } 2484 } 2485 } 2486 } 2487 2488 info->mNotify->post(); 2489 info->mNotify = NULL; 2490 2491 return OK; 2492 } 2493 2494 ssize_t MediaCodec::dequeuePortBuffer(int32_t portIndex) { 2495 CHECK(portIndex == kPortIndexInput || portIndex == kPortIndexOutput); 2496 2497 List<size_t> *availBuffers = &mAvailPortBuffers[portIndex]; 2498 2499 if (availBuffers->empty()) { 2500 return -EAGAIN; 2501 } 2502 2503 size_t index = *availBuffers->begin(); 2504 availBuffers->erase(availBuffers->begin()); 2505 2506 BufferInfo *info = &mPortBuffers[portIndex].editItemAt(index); 2507 CHECK(!info->mOwnedByClient); 2508 { 2509 Mutex::Autolock al(mBufferLock); 2510 info->mOwnedByClient = true; 2511 2512 // set image-data 2513 if (info->mFormat != NULL) { 2514 sp<ABuffer> imageData; 2515 if (info->mFormat->findBuffer("image-data", &imageData)) { 2516 info->mData->meta()->setBuffer("image-data", imageData); 2517 } 2518 int32_t left, top, right, bottom; 2519 if (info->mFormat->findRect("crop", &left, &top, &right, &bottom)) { 2520 info->mData->meta()->setRect("crop-rect", left, top, right, bottom); 2521 } 2522 } 2523 } 2524 2525 return index; 2526 } 2527 2528 status_t MediaCodec::connectToSurface(const sp<Surface> &surface) { 2529 status_t err = OK; 2530 if (surface != NULL) { 2531 err = native_window_api_connect(surface.get(), NATIVE_WINDOW_API_MEDIA); 2532 if (err == BAD_VALUE) { 2533 ALOGI("native window already connected. Assuming no change of surface"); 2534 return err; 2535 } else if (err == OK) { 2536 // Require a fresh set of buffers after each connect by using a unique generation 2537 // number. Rely on the fact that max supported process id by Linux is 2^22. 2538 // PID is never 0 so we don't have to worry that we use the default generation of 0. 2539 // TODO: come up with a unique scheme if other producers also set the generation number. 2540 static uint32_t mSurfaceGeneration = 0; 2541 uint32_t generation = (getpid() << 10) | (++mSurfaceGeneration & ((1 << 10) - 1)); 2542 surface->setGenerationNumber(generation); 2543 ALOGI("[%s] setting surface generation to %u", mComponentName.c_str(), generation); 2544 2545 // HACK: clear any free buffers. Remove when connect will automatically do this. 2546 // This is needed as the consumer may be holding onto stale frames that it can reattach 2547 // to this surface after disconnect/connect, and those free frames would inherit the new 2548 // generation number. Disconnecting after setting a unique generation prevents this. 2549 native_window_api_disconnect(surface.get(), NATIVE_WINDOW_API_MEDIA); 2550 err = native_window_api_connect(surface.get(), NATIVE_WINDOW_API_MEDIA); 2551 } 2552 2553 if (err != OK) { 2554 ALOGE("native_window_api_connect returned an error: %s (%d)", strerror(-err), err); 2555 } 2556 } 2557 return err; 2558 } 2559 2560 status_t MediaCodec::disconnectFromSurface() { 2561 status_t err = OK; 2562 if (mSurface != NULL) { 2563 // Resetting generation is not technically needed, but there is no need to keep it either 2564 mSurface->setGenerationNumber(0); 2565 err = native_window_api_disconnect(mSurface.get(), NATIVE_WINDOW_API_MEDIA); 2566 if (err != OK) { 2567 ALOGW("native_window_api_disconnect returned an error: %s (%d)", strerror(-err), err); 2568 } 2569 // assume disconnected even on error 2570 mSurface.clear(); 2571 } 2572 return err; 2573 } 2574 2575 status_t MediaCodec::handleSetSurface(const sp<Surface> &surface) { 2576 status_t err = OK; 2577 if (mSurface != NULL) { 2578 (void)disconnectFromSurface(); 2579 } 2580 if (surface != NULL) { 2581 err = connectToSurface(surface); 2582 if (err == OK) { 2583 mSurface = surface; 2584 } 2585 } 2586 return err; 2587 } 2588 2589 void MediaCodec::onInputBufferAvailable() { 2590 int32_t index; 2591 while ((index = dequeuePortBuffer(kPortIndexInput)) >= 0) { 2592 sp<AMessage> msg = mCallback->dup(); 2593 msg->setInt32("callbackID", CB_INPUT_AVAILABLE); 2594 msg->setInt32("index", index); 2595 msg->post(); 2596 } 2597 } 2598 2599 void MediaCodec::onOutputBufferAvailable() { 2600 int32_t index; 2601 while ((index = dequeuePortBuffer(kPortIndexOutput)) >= 0) { 2602 const sp<ABuffer> &buffer = 2603 mPortBuffers[kPortIndexOutput].itemAt(index).mData; 2604 sp<AMessage> msg = mCallback->dup(); 2605 msg->setInt32("callbackID", CB_OUTPUT_AVAILABLE); 2606 msg->setInt32("index", index); 2607 msg->setSize("offset", buffer->offset()); 2608 msg->setSize("size", buffer->size()); 2609 2610 int64_t timeUs; 2611 CHECK(buffer->meta()->findInt64("timeUs", &timeUs)); 2612 2613 msg->setInt64("timeUs", timeUs); 2614 2615 int32_t omxFlags; 2616 CHECK(buffer->meta()->findInt32("omxFlags", &omxFlags)); 2617 2618 uint32_t flags = 0; 2619 if (omxFlags & OMX_BUFFERFLAG_SYNCFRAME) { 2620 flags |= BUFFER_FLAG_SYNCFRAME; 2621 } 2622 if (omxFlags & OMX_BUFFERFLAG_CODECCONFIG) { 2623 flags |= BUFFER_FLAG_CODECCONFIG; 2624 } 2625 if (omxFlags & OMX_BUFFERFLAG_EOS) { 2626 flags |= BUFFER_FLAG_EOS; 2627 } 2628 2629 msg->setInt32("flags", flags); 2630 2631 msg->post(); 2632 } 2633 } 2634 2635 void MediaCodec::onError(status_t err, int32_t actionCode, const char *detail) { 2636 if (mCallback != NULL) { 2637 sp<AMessage> msg = mCallback->dup(); 2638 msg->setInt32("callbackID", CB_ERROR); 2639 msg->setInt32("err", err); 2640 msg->setInt32("actionCode", actionCode); 2641 2642 if (detail != NULL) { 2643 msg->setString("detail", detail); 2644 } 2645 2646 msg->post(); 2647 } 2648 } 2649 2650 void MediaCodec::onOutputFormatChanged() { 2651 if (mCallback != NULL) { 2652 sp<AMessage> msg = mCallback->dup(); 2653 msg->setInt32("callbackID", CB_OUTPUT_FORMAT_CHANGED); 2654 msg->setMessage("format", mOutputFormat); 2655 msg->post(); 2656 } 2657 } 2658 2659 2660 void MediaCodec::postActivityNotificationIfPossible() { 2661 if (mActivityNotify == NULL) { 2662 return; 2663 } 2664 2665 bool isErrorOrOutputChanged = 2666 (mFlags & (kFlagStickyError 2667 | kFlagOutputBuffersChanged 2668 | kFlagOutputFormatChanged)); 2669 2670 if (isErrorOrOutputChanged 2671 || !mAvailPortBuffers[kPortIndexInput].empty() 2672 || !mAvailPortBuffers[kPortIndexOutput].empty()) { 2673 mActivityNotify->setInt32("input-buffers", 2674 mAvailPortBuffers[kPortIndexInput].size()); 2675 2676 if (isErrorOrOutputChanged) { 2677 // we want consumer to dequeue as many times as it can 2678 mActivityNotify->setInt32("output-buffers", INT32_MAX); 2679 } else { 2680 mActivityNotify->setInt32("output-buffers", 2681 mAvailPortBuffers[kPortIndexOutput].size()); 2682 } 2683 mActivityNotify->post(); 2684 mActivityNotify.clear(); 2685 } 2686 } 2687 2688 status_t MediaCodec::setParameters(const sp<AMessage> ¶ms) { 2689 sp<AMessage> msg = new AMessage(kWhatSetParameters, this); 2690 msg->setMessage("params", params); 2691 2692 sp<AMessage> response; 2693 return PostAndAwaitResponse(msg, &response); 2694 } 2695 2696 status_t MediaCodec::onSetParameters(const sp<AMessage> ¶ms) { 2697 mCodec->signalSetParameters(params); 2698 2699 return OK; 2700 } 2701 2702 status_t MediaCodec::amendOutputFormatWithCodecSpecificData( 2703 const sp<ABuffer> &buffer) { 2704 AString mime; 2705 CHECK(mOutputFormat->findString("mime", &mime)); 2706 2707 if (!strcasecmp(mime.c_str(), MEDIA_MIMETYPE_VIDEO_AVC)) { 2708 // Codec specific data should be SPS and PPS in a single buffer, 2709 // each prefixed by a startcode (0x00 0x00 0x00 0x01). 2710 // We separate the two and put them into the output format 2711 // under the keys "csd-0" and "csd-1". 2712 2713 unsigned csdIndex = 0; 2714 2715 const uint8_t *data = buffer->data(); 2716 size_t size = buffer->size(); 2717 2718 const uint8_t *nalStart; 2719 size_t nalSize; 2720 while (getNextNALUnit(&data, &size, &nalStart, &nalSize, true) == OK) { 2721 sp<ABuffer> csd = new ABuffer(nalSize + 4); 2722 memcpy(csd->data(), "\x00\x00\x00\x01", 4); 2723 memcpy(csd->data() + 4, nalStart, nalSize); 2724 2725 mOutputFormat->setBuffer( 2726 AStringPrintf("csd-%u", csdIndex).c_str(), csd); 2727 2728 ++csdIndex; 2729 } 2730 2731 if (csdIndex != 2) { 2732 return ERROR_MALFORMED; 2733 } 2734 } else { 2735 // For everything else we just stash the codec specific data into 2736 // the output format as a single piece of csd under "csd-0". 2737 mOutputFormat->setBuffer("csd-0", buffer); 2738 } 2739 2740 return OK; 2741 } 2742 2743 void MediaCodec::updateBatteryStat() { 2744 if (mState == CONFIGURED && !mBatteryStatNotified) { 2745 BatteryNotifier& notifier(BatteryNotifier::getInstance()); 2746 2747 if (mIsVideo) { 2748 notifier.noteStartVideo(); 2749 } else { 2750 notifier.noteStartAudio(); 2751 } 2752 2753 mBatteryStatNotified = true; 2754 } else if (mState == UNINITIALIZED && mBatteryStatNotified) { 2755 BatteryNotifier& notifier(BatteryNotifier::getInstance()); 2756 2757 if (mIsVideo) { 2758 notifier.noteStopVideo(); 2759 } else { 2760 notifier.noteStopAudio(); 2761 } 2762 2763 mBatteryStatNotified = false; 2764 } 2765 } 2766 2767 } // namespace android 2768