1 /* 2 * Copyright 2012, The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17 //#define LOG_NDEBUG 0 18 #define LOG_TAG "MediaCodec" 19 #include <utils/Log.h> 20 21 #include <inttypes.h> 22 #include <stdlib.h> 23 24 #include "include/SecureBuffer.h" 25 #include "include/SharedMemoryBuffer.h" 26 #include "include/SoftwareRenderer.h" 27 #include "StagefrightPluginLoader.h" 28 29 #include <android/hardware/cas/native/1.0/IDescrambler.h> 30 31 #include <binder/IMemory.h> 32 #include <binder/IPCThreadState.h> 33 #include <binder/IServiceManager.h> 34 #include <binder/MemoryDealer.h> 35 #include <cutils/properties.h> 36 #include <gui/BufferQueue.h> 37 #include <gui/Surface.h> 38 #include <media/ICrypto.h> 39 #include <media/IOMX.h> 40 #include <media/IResourceManagerService.h> 41 #include <media/MediaCodecBuffer.h> 42 #include <media/MediaAnalyticsItem.h> 43 #include <media/stagefright/foundation/ABuffer.h> 44 #include <media/stagefright/foundation/ADebug.h> 45 #include <media/stagefright/foundation/AMessage.h> 46 #include <media/stagefright/foundation/AString.h> 47 #include <media/stagefright/foundation/AUtils.h> 48 #include <media/stagefright/foundation/avc_utils.h> 49 #include <media/stagefright/foundation/hexdump.h> 50 #include <media/stagefright/ACodec.h> 51 #include <media/stagefright/BufferProducerWrapper.h> 52 #include <media/stagefright/MediaCodec.h> 53 #include <media/stagefright/MediaCodecList.h> 54 #include <media/stagefright/MediaDefs.h> 55 #include <media/stagefright/MediaErrors.h> 56 #include <media/stagefright/MediaFilter.h> 57 #include <media/stagefright/OMXClient.h> 58 #include <media/stagefright/PersistentSurface.h> 59 #include <media/stagefright/SurfaceUtils.h> 60 #include <mediautils/BatteryNotifier.h> 61 #include <private/android_filesystem_config.h> 62 #include <utils/Singleton.h> 63 64 namespace android { 65 66 // key for media statistics 67 static const char *kCodecKeyName = "codec"; 68 // attrs for media statistics 69 // NB: these are matched with public Java API constants defined 70 // in frameworks/base/media/java/android/media/MediaCodec.java 71 // These must be kept synchronized with the constants there. 72 static const char *kCodecCodec = "android.media.mediacodec.codec"; /* e.g. OMX.google.aac.decoder */ 73 static const char *kCodecMime = "android.media.mediacodec.mime"; /* e.g. audio/mime */ 74 static const char *kCodecMode = "android.media.mediacodec.mode"; /* audio, video */ 75 static const char *kCodecModeVideo = "video"; /* values returned for kCodecMode */ 76 static const char *kCodecModeAudio = "audio"; 77 static const char *kCodecEncoder = "android.media.mediacodec.encoder"; /* 0,1 */ 78 static const char *kCodecSecure = "android.media.mediacodec.secure"; /* 0, 1 */ 79 static const char *kCodecWidth = "android.media.mediacodec.width"; /* 0..n */ 80 static const char *kCodecHeight = "android.media.mediacodec.height"; /* 0..n */ 81 static const char *kCodecRotation = "android.media.mediacodec.rotation-degrees"; /* 0/90/180/270 */ 82 83 // NB: These are not yet exposed as public Java API constants. 84 static const char *kCodecCrypto = "android.media.mediacodec.crypto"; /* 0,1 */ 85 static const char *kCodecProfile = "android.media.mediacodec.profile"; /* 0..n */ 86 static const char *kCodecLevel = "android.media.mediacodec.level"; /* 0..n */ 87 static const char *kCodecMaxWidth = "android.media.mediacodec.maxwidth"; /* 0..n */ 88 static const char *kCodecMaxHeight = "android.media.mediacodec.maxheight"; /* 0..n */ 89 static const char *kCodecError = "android.media.mediacodec.errcode"; 90 static const char *kCodecErrorState = "android.media.mediacodec.errstate"; 91 static const char *kCodecLatencyMax = "android.media.mediacodec.latency.max"; /* in us */ 92 static const char *kCodecLatencyMin = "android.media.mediacodec.latency.min"; /* in us */ 93 static const char *kCodecLatencyAvg = "android.media.mediacodec.latency.avg"; /* in us */ 94 static const char *kCodecLatencyCount = "android.media.mediacodec.latency.n"; 95 static const char *kCodecLatencyHist = "android.media.mediacodec.latency.hist"; /* in us */ 96 static const char *kCodecLatencyUnknown = "android.media.mediacodec.latency.unknown"; 97 98 // the kCodecRecent* fields appear only in getMetrics() results 99 static const char *kCodecRecentLatencyMax = "android.media.mediacodec.recent.max"; /* in us */ 100 static const char *kCodecRecentLatencyMin = "android.media.mediacodec.recent.min"; /* in us */ 101 static const char *kCodecRecentLatencyAvg = "android.media.mediacodec.recent.avg"; /* in us */ 102 static const char *kCodecRecentLatencyCount = "android.media.mediacodec.recent.n"; 103 static const char *kCodecRecentLatencyHist = "android.media.mediacodec.recent.hist"; /* in us */ 104 105 // XXX suppress until we get our representation right 106 static bool kEmitHistogram = false; 107 108 109 static int64_t getId(const sp<IResourceManagerClient> &client) { 110 return (int64_t) client.get(); 111 } 112 113 static bool isResourceError(status_t err) { 114 return (err == NO_MEMORY); 115 } 116 117 static const int kMaxRetry = 2; 118 static const int kMaxReclaimWaitTimeInUs = 500000; // 0.5s 119 static const int kNumBuffersAlign = 16; 120 121 //////////////////////////////////////////////////////////////////////////////// 122 123 struct ResourceManagerClient : public BnResourceManagerClient { 124 explicit ResourceManagerClient(MediaCodec* codec) : mMediaCodec(codec) {} 125 126 virtual bool reclaimResource() { 127 sp<MediaCodec> codec = mMediaCodec.promote(); 128 if (codec == NULL) { 129 // codec is already gone. 130 return true; 131 } 132 status_t err = codec->reclaim(); 133 if (err == WOULD_BLOCK) { 134 ALOGD("Wait for the client to release codec."); 135 usleep(kMaxReclaimWaitTimeInUs); 136 ALOGD("Try to reclaim again."); 137 err = codec->reclaim(true /* force */); 138 } 139 if (err != OK) { 140 ALOGW("ResourceManagerClient failed to release codec with err %d", err); 141 } 142 return (err == OK); 143 } 144 145 virtual String8 getName() { 146 String8 ret; 147 sp<MediaCodec> codec = mMediaCodec.promote(); 148 if (codec == NULL) { 149 // codec is already gone. 150 return ret; 151 } 152 153 AString name; 154 if (codec->getName(&name) == OK) { 155 ret.setTo(name.c_str()); 156 } 157 return ret; 158 } 159 160 protected: 161 virtual ~ResourceManagerClient() {} 162 163 private: 164 wp<MediaCodec> mMediaCodec; 165 166 DISALLOW_EVIL_CONSTRUCTORS(ResourceManagerClient); 167 }; 168 169 MediaCodec::ResourceManagerServiceProxy::ResourceManagerServiceProxy(pid_t pid) 170 : mPid(pid) { 171 if (mPid == MediaCodec::kNoPid) { 172 mPid = IPCThreadState::self()->getCallingPid(); 173 } 174 } 175 176 MediaCodec::ResourceManagerServiceProxy::~ResourceManagerServiceProxy() { 177 if (mService != NULL) { 178 IInterface::asBinder(mService)->unlinkToDeath(this); 179 } 180 } 181 182 void MediaCodec::ResourceManagerServiceProxy::init() { 183 sp<IServiceManager> sm = defaultServiceManager(); 184 sp<IBinder> binder = sm->getService(String16("media.resource_manager")); 185 mService = interface_cast<IResourceManagerService>(binder); 186 if (mService == NULL) { 187 ALOGE("Failed to get ResourceManagerService"); 188 return; 189 } 190 IInterface::asBinder(mService)->linkToDeath(this); 191 } 192 193 void MediaCodec::ResourceManagerServiceProxy::binderDied(const wp<IBinder>& /*who*/) { 194 ALOGW("ResourceManagerService died."); 195 Mutex::Autolock _l(mLock); 196 mService.clear(); 197 } 198 199 void MediaCodec::ResourceManagerServiceProxy::addResource( 200 int64_t clientId, 201 const sp<IResourceManagerClient> &client, 202 const Vector<MediaResource> &resources) { 203 Mutex::Autolock _l(mLock); 204 if (mService == NULL) { 205 return; 206 } 207 mService->addResource(mPid, clientId, client, resources); 208 } 209 210 void MediaCodec::ResourceManagerServiceProxy::removeResource(int64_t clientId) { 211 Mutex::Autolock _l(mLock); 212 if (mService == NULL) { 213 return; 214 } 215 mService->removeResource(mPid, clientId); 216 } 217 218 bool MediaCodec::ResourceManagerServiceProxy::reclaimResource( 219 const Vector<MediaResource> &resources) { 220 Mutex::Autolock _l(mLock); 221 if (mService == NULL) { 222 return false; 223 } 224 return mService->reclaimResource(mPid, resources); 225 } 226 227 //////////////////////////////////////////////////////////////////////////////// 228 229 MediaCodec::BufferInfo::BufferInfo() : mOwnedByClient(false) {} 230 231 //////////////////////////////////////////////////////////////////////////////// 232 233 namespace { 234 235 enum { 236 kWhatFillThisBuffer = 'fill', 237 kWhatDrainThisBuffer = 'drai', 238 kWhatEOS = 'eos ', 239 kWhatStartCompleted = 'Scom', 240 kWhatStopCompleted = 'scom', 241 kWhatReleaseCompleted = 'rcom', 242 kWhatFlushCompleted = 'fcom', 243 kWhatError = 'erro', 244 kWhatComponentAllocated = 'cAll', 245 kWhatComponentConfigured = 'cCon', 246 kWhatInputSurfaceCreated = 'isfc', 247 kWhatInputSurfaceAccepted = 'isfa', 248 kWhatSignaledInputEOS = 'seos', 249 kWhatOutputFramesRendered = 'outR', 250 kWhatOutputBuffersChanged = 'outC', 251 }; 252 253 class BufferCallback : public CodecBase::BufferCallback { 254 public: 255 explicit BufferCallback(const sp<AMessage> ¬ify); 256 virtual ~BufferCallback() = default; 257 258 virtual void onInputBufferAvailable( 259 size_t index, const sp<MediaCodecBuffer> &buffer) override; 260 virtual void onOutputBufferAvailable( 261 size_t index, const sp<MediaCodecBuffer> &buffer) override; 262 private: 263 const sp<AMessage> mNotify; 264 }; 265 266 BufferCallback::BufferCallback(const sp<AMessage> ¬ify) 267 : mNotify(notify) {} 268 269 void BufferCallback::onInputBufferAvailable( 270 size_t index, const sp<MediaCodecBuffer> &buffer) { 271 sp<AMessage> notify(mNotify->dup()); 272 notify->setInt32("what", kWhatFillThisBuffer); 273 notify->setSize("index", index); 274 notify->setObject("buffer", buffer); 275 notify->post(); 276 } 277 278 void BufferCallback::onOutputBufferAvailable( 279 size_t index, const sp<MediaCodecBuffer> &buffer) { 280 sp<AMessage> notify(mNotify->dup()); 281 notify->setInt32("what", kWhatDrainThisBuffer); 282 notify->setSize("index", index); 283 notify->setObject("buffer", buffer); 284 notify->post(); 285 } 286 287 class CodecCallback : public CodecBase::CodecCallback { 288 public: 289 explicit CodecCallback(const sp<AMessage> ¬ify); 290 virtual ~CodecCallback() = default; 291 292 virtual void onEos(status_t err) override; 293 virtual void onStartCompleted() override; 294 virtual void onStopCompleted() override; 295 virtual void onReleaseCompleted() override; 296 virtual void onFlushCompleted() override; 297 virtual void onError(status_t err, enum ActionCode actionCode) override; 298 virtual void onComponentAllocated(const char *componentName) override; 299 virtual void onComponentConfigured( 300 const sp<AMessage> &inputFormat, const sp<AMessage> &outputFormat) override; 301 virtual void onInputSurfaceCreated( 302 const sp<AMessage> &inputFormat, 303 const sp<AMessage> &outputFormat, 304 const sp<BufferProducerWrapper> &inputSurface) override; 305 virtual void onInputSurfaceCreationFailed(status_t err) override; 306 virtual void onInputSurfaceAccepted( 307 const sp<AMessage> &inputFormat, 308 const sp<AMessage> &outputFormat) override; 309 virtual void onInputSurfaceDeclined(status_t err) override; 310 virtual void onSignaledInputEOS(status_t err) override; 311 virtual void onOutputFramesRendered(const std::list<FrameRenderTracker::Info> &done) override; 312 virtual void onOutputBuffersChanged() override; 313 private: 314 const sp<AMessage> mNotify; 315 }; 316 317 CodecCallback::CodecCallback(const sp<AMessage> ¬ify) : mNotify(notify) {} 318 319 void CodecCallback::onEos(status_t err) { 320 sp<AMessage> notify(mNotify->dup()); 321 notify->setInt32("what", kWhatEOS); 322 notify->setInt32("err", err); 323 notify->post(); 324 } 325 326 void CodecCallback::onStartCompleted() { 327 sp<AMessage> notify(mNotify->dup()); 328 notify->setInt32("what", kWhatStartCompleted); 329 notify->post(); 330 } 331 332 void CodecCallback::onStopCompleted() { 333 sp<AMessage> notify(mNotify->dup()); 334 notify->setInt32("what", kWhatStopCompleted); 335 notify->post(); 336 } 337 338 void CodecCallback::onReleaseCompleted() { 339 sp<AMessage> notify(mNotify->dup()); 340 notify->setInt32("what", kWhatReleaseCompleted); 341 notify->post(); 342 } 343 344 void CodecCallback::onFlushCompleted() { 345 sp<AMessage> notify(mNotify->dup()); 346 notify->setInt32("what", kWhatFlushCompleted); 347 notify->post(); 348 } 349 350 void CodecCallback::onError(status_t err, enum ActionCode actionCode) { 351 sp<AMessage> notify(mNotify->dup()); 352 notify->setInt32("what", kWhatError); 353 notify->setInt32("err", err); 354 notify->setInt32("actionCode", actionCode); 355 notify->post(); 356 } 357 358 void CodecCallback::onComponentAllocated(const char *componentName) { 359 sp<AMessage> notify(mNotify->dup()); 360 notify->setInt32("what", kWhatComponentAllocated); 361 notify->setString("componentName", componentName); 362 notify->post(); 363 } 364 365 void CodecCallback::onComponentConfigured( 366 const sp<AMessage> &inputFormat, const sp<AMessage> &outputFormat) { 367 sp<AMessage> notify(mNotify->dup()); 368 notify->setInt32("what", kWhatComponentConfigured); 369 notify->setMessage("input-format", inputFormat); 370 notify->setMessage("output-format", outputFormat); 371 notify->post(); 372 } 373 374 void CodecCallback::onInputSurfaceCreated( 375 const sp<AMessage> &inputFormat, 376 const sp<AMessage> &outputFormat, 377 const sp<BufferProducerWrapper> &inputSurface) { 378 sp<AMessage> notify(mNotify->dup()); 379 notify->setInt32("what", kWhatInputSurfaceCreated); 380 notify->setMessage("input-format", inputFormat); 381 notify->setMessage("output-format", outputFormat); 382 notify->setObject("input-surface", inputSurface); 383 notify->post(); 384 } 385 386 void CodecCallback::onInputSurfaceCreationFailed(status_t err) { 387 sp<AMessage> notify(mNotify->dup()); 388 notify->setInt32("what", kWhatInputSurfaceCreated); 389 notify->setInt32("err", err); 390 notify->post(); 391 } 392 393 void CodecCallback::onInputSurfaceAccepted( 394 const sp<AMessage> &inputFormat, 395 const sp<AMessage> &outputFormat) { 396 sp<AMessage> notify(mNotify->dup()); 397 notify->setInt32("what", kWhatInputSurfaceAccepted); 398 notify->setMessage("input-format", inputFormat); 399 notify->setMessage("output-format", outputFormat); 400 notify->post(); 401 } 402 403 void CodecCallback::onInputSurfaceDeclined(status_t err) { 404 sp<AMessage> notify(mNotify->dup()); 405 notify->setInt32("what", kWhatInputSurfaceAccepted); 406 notify->setInt32("err", err); 407 notify->post(); 408 } 409 410 void CodecCallback::onSignaledInputEOS(status_t err) { 411 sp<AMessage> notify(mNotify->dup()); 412 notify->setInt32("what", kWhatSignaledInputEOS); 413 if (err != OK) { 414 notify->setInt32("err", err); 415 } 416 notify->post(); 417 } 418 419 void CodecCallback::onOutputFramesRendered(const std::list<FrameRenderTracker::Info> &done) { 420 sp<AMessage> notify(mNotify->dup()); 421 notify->setInt32("what", kWhatOutputFramesRendered); 422 if (MediaCodec::CreateFramesRenderedMessage(done, notify)) { 423 notify->post(); 424 } 425 } 426 427 void CodecCallback::onOutputBuffersChanged() { 428 sp<AMessage> notify(mNotify->dup()); 429 notify->setInt32("what", kWhatOutputBuffersChanged); 430 notify->post(); 431 } 432 433 } // namespace 434 435 //////////////////////////////////////////////////////////////////////////////// 436 437 // static 438 sp<MediaCodec> MediaCodec::CreateByType( 439 const sp<ALooper> &looper, const AString &mime, bool encoder, status_t *err, pid_t pid, 440 uid_t uid) { 441 Vector<AString> matchingCodecs; 442 443 MediaCodecList::findMatchingCodecs( 444 mime.c_str(), 445 encoder, 446 0, 447 &matchingCodecs); 448 449 if (err != NULL) { 450 *err = NAME_NOT_FOUND; 451 } 452 for (size_t i = 0; i < matchingCodecs.size(); ++i) { 453 sp<MediaCodec> codec = new MediaCodec(looper, pid, uid); 454 AString componentName = matchingCodecs[i]; 455 status_t ret = codec->init(componentName); 456 if (err != NULL) { 457 *err = ret; 458 } 459 if (ret == OK) { 460 return codec; 461 } 462 ALOGD("Allocating component '%s' failed (%d), try next one.", 463 componentName.c_str(), ret); 464 } 465 return NULL; 466 } 467 468 // static 469 sp<MediaCodec> MediaCodec::CreateByComponentName( 470 const sp<ALooper> &looper, const AString &name, status_t *err, pid_t pid, uid_t uid) { 471 sp<MediaCodec> codec = new MediaCodec(looper, pid, uid); 472 473 const status_t ret = codec->init(name); 474 if (err != NULL) { 475 *err = ret; 476 } 477 return ret == OK ? codec : NULL; // NULL deallocates codec. 478 } 479 480 // static 481 sp<PersistentSurface> MediaCodec::CreatePersistentInputSurface() { 482 // allow plugin to create surface 483 sp<PersistentSurface> pluginSurface = 484 StagefrightPluginLoader::GetCCodecInstance()->createInputSurface(); 485 if (pluginSurface != nullptr) { 486 return pluginSurface; 487 } 488 489 OMXClient client; 490 if (client.connect() != OK) { 491 ALOGE("Failed to connect to OMX to create persistent input surface."); 492 return NULL; 493 } 494 495 sp<IOMX> omx = client.interface(); 496 497 sp<IGraphicBufferProducer> bufferProducer; 498 sp<IGraphicBufferSource> bufferSource; 499 500 status_t err = omx->createInputSurface(&bufferProducer, &bufferSource); 501 502 if (err != OK) { 503 ALOGE("Failed to create persistent input surface."); 504 return NULL; 505 } 506 507 return new PersistentSurface(bufferProducer, bufferSource); 508 } 509 510 MediaCodec::MediaCodec(const sp<ALooper> &looper, pid_t pid, uid_t uid) 511 : mState(UNINITIALIZED), 512 mReleasedByResourceManager(false), 513 mLooper(looper), 514 mCodec(NULL), 515 mReplyID(0), 516 mFlags(0), 517 mStickyError(OK), 518 mSoftRenderer(NULL), 519 mAnalyticsItem(NULL), 520 mResourceManagerClient(new ResourceManagerClient(this)), 521 mResourceManagerService(new ResourceManagerServiceProxy(pid)), 522 mBatteryStatNotified(false), 523 mIsVideo(false), 524 mVideoWidth(0), 525 mVideoHeight(0), 526 mRotationDegrees(0), 527 mDequeueInputTimeoutGeneration(0), 528 mDequeueInputReplyID(0), 529 mDequeueOutputTimeoutGeneration(0), 530 mDequeueOutputReplyID(0), 531 mHaveInputSurface(false), 532 mHavePendingInputBuffers(false), 533 mCpuBoostRequested(false), 534 mLatencyUnknown(0) { 535 if (uid == kNoUid) { 536 mUid = IPCThreadState::self()->getCallingUid(); 537 } else { 538 mUid = uid; 539 } 540 541 initAnalyticsItem(); 542 } 543 544 MediaCodec::~MediaCodec() { 545 CHECK_EQ(mState, UNINITIALIZED); 546 mResourceManagerService->removeResource(getId(mResourceManagerClient)); 547 548 flushAnalyticsItem(); 549 } 550 551 void MediaCodec::initAnalyticsItem() { 552 if (mAnalyticsItem == NULL) { 553 mAnalyticsItem = MediaAnalyticsItem::create(kCodecKeyName); 554 } 555 556 mLatencyHist.setup(kLatencyHistBuckets, kLatencyHistWidth, kLatencyHistFloor); 557 558 { 559 Mutex::Autolock al(mRecentLock); 560 for (int i = 0; i<kRecentLatencyFrames; i++) { 561 mRecentSamples[i] = kRecentSampleInvalid; 562 } 563 mRecentHead = 0; 564 } 565 } 566 567 void MediaCodec::updateAnalyticsItem() { 568 ALOGV("MediaCodec::updateAnalyticsItem"); 569 if (mAnalyticsItem == NULL) { 570 return; 571 } 572 573 if (mLatencyHist.getCount() != 0 ) { 574 mAnalyticsItem->setInt64(kCodecLatencyMax, mLatencyHist.getMax()); 575 mAnalyticsItem->setInt64(kCodecLatencyMin, mLatencyHist.getMin()); 576 mAnalyticsItem->setInt64(kCodecLatencyAvg, mLatencyHist.getAvg()); 577 mAnalyticsItem->setInt64(kCodecLatencyCount, mLatencyHist.getCount()); 578 579 if (kEmitHistogram) { 580 // and the histogram itself 581 std::string hist = mLatencyHist.emit(); 582 mAnalyticsItem->setCString(kCodecLatencyHist, hist.c_str()); 583 } 584 } 585 if (mLatencyUnknown > 0) { 586 mAnalyticsItem->setInt64(kCodecLatencyUnknown, mLatencyUnknown); 587 } 588 589 #if 0 590 // enable for short term, only while debugging 591 updateEphemeralAnalytics(mAnalyticsItem); 592 #endif 593 } 594 595 void MediaCodec::updateEphemeralAnalytics(MediaAnalyticsItem *item) { 596 ALOGD("MediaCodec::updateEphemeralAnalytics()"); 597 598 if (item == NULL) { 599 return; 600 } 601 602 Histogram recentHist; 603 604 // build an empty histogram 605 recentHist.setup(kLatencyHistBuckets, kLatencyHistWidth, kLatencyHistFloor); 606 607 // stuff it with the samples in the ring buffer 608 { 609 Mutex::Autolock al(mRecentLock); 610 611 for (int i=0; i<kRecentLatencyFrames; i++) { 612 if (mRecentSamples[i] != kRecentSampleInvalid) { 613 recentHist.insert(mRecentSamples[i]); 614 } 615 } 616 } 617 618 619 // spit the data (if any) into the supplied analytics record 620 if (recentHist.getCount()!= 0 ) { 621 item->setInt64(kCodecRecentLatencyMax, recentHist.getMax()); 622 item->setInt64(kCodecRecentLatencyMin, recentHist.getMin()); 623 item->setInt64(kCodecRecentLatencyAvg, recentHist.getAvg()); 624 item->setInt64(kCodecRecentLatencyCount, recentHist.getCount()); 625 626 if (kEmitHistogram) { 627 // and the histogram itself 628 std::string hist = recentHist.emit(); 629 item->setCString(kCodecRecentLatencyHist, hist.c_str()); 630 } 631 } 632 } 633 634 void MediaCodec::flushAnalyticsItem() { 635 updateAnalyticsItem(); 636 if (mAnalyticsItem != NULL) { 637 // don't log empty records 638 if (mAnalyticsItem->count() > 0) { 639 mAnalyticsItem->selfrecord(); 640 } 641 delete mAnalyticsItem; 642 mAnalyticsItem = NULL; 643 } 644 } 645 646 bool MediaCodec::Histogram::setup(int nbuckets, int64_t width, int64_t floor) 647 { 648 if (nbuckets <= 0 || width <= 0) { 649 return false; 650 } 651 652 // get histogram buckets 653 if (nbuckets == mBucketCount && mBuckets != NULL) { 654 // reuse our existing buffer 655 memset(mBuckets, 0, sizeof(*mBuckets) * mBucketCount); 656 } else { 657 // get a new pre-zeroed buffer 658 int64_t *newbuckets = (int64_t *)calloc(nbuckets, sizeof (*mBuckets)); 659 if (newbuckets == NULL) { 660 goto bad; 661 } 662 if (mBuckets != NULL) 663 free(mBuckets); 664 mBuckets = newbuckets; 665 } 666 667 mWidth = width; 668 mFloor = floor; 669 mCeiling = floor + nbuckets * width; 670 mBucketCount = nbuckets; 671 672 mMin = INT64_MAX; 673 mMax = INT64_MIN; 674 mSum = 0; 675 mCount = 0; 676 mBelow = mAbove = 0; 677 678 return true; 679 680 bad: 681 if (mBuckets != NULL) { 682 free(mBuckets); 683 mBuckets = NULL; 684 } 685 686 return false; 687 } 688 689 void MediaCodec::Histogram::insert(int64_t sample) 690 { 691 // histogram is not set up 692 if (mBuckets == NULL) { 693 return; 694 } 695 696 mCount++; 697 mSum += sample; 698 if (mMin > sample) mMin = sample; 699 if (mMax < sample) mMax = sample; 700 701 if (sample < mFloor) { 702 mBelow++; 703 } else if (sample >= mCeiling) { 704 mAbove++; 705 } else { 706 int64_t slot = (sample - mFloor) / mWidth; 707 CHECK(slot < mBucketCount); 708 mBuckets[slot]++; 709 } 710 return; 711 } 712 713 std::string MediaCodec::Histogram::emit() 714 { 715 std::string value; 716 char buffer[64]; 717 718 // emits: width,Below{bucket0,bucket1,...., bucketN}above 719 // unconfigured will emit: 0,0{}0 720 // XXX: is this best representation? 721 snprintf(buffer, sizeof(buffer), "%" PRId64 ",%" PRId64 ",%" PRId64 "{", 722 mFloor, mWidth, mBelow); 723 value = buffer; 724 for (int i = 0; i < mBucketCount; i++) { 725 if (i != 0) { 726 value = value + ","; 727 } 728 snprintf(buffer, sizeof(buffer), "%" PRId64, mBuckets[i]); 729 value = value + buffer; 730 } 731 snprintf(buffer, sizeof(buffer), "}%" PRId64 , mAbove); 732 value = value + buffer; 733 return value; 734 } 735 736 // when we send a buffer to the codec; 737 void MediaCodec::statsBufferSent(int64_t presentationUs) { 738 739 // only enqueue if we have a legitimate time 740 if (presentationUs <= 0) { 741 ALOGV("presentation time: %" PRId64, presentationUs); 742 return; 743 } 744 745 const int64_t nowNs = systemTime(SYSTEM_TIME_MONOTONIC); 746 BufferFlightTiming_t startdata = { presentationUs, nowNs }; 747 748 { 749 // mutex access to mBuffersInFlight and other stats 750 Mutex::Autolock al(mLatencyLock); 751 752 753 // XXX: we *could* make sure that the time is later than the end of queue 754 // as part of a consistency check... 755 mBuffersInFlight.push_back(startdata); 756 } 757 } 758 759 // when we get a buffer back from the codec 760 void MediaCodec::statsBufferReceived(int64_t presentationUs) { 761 762 CHECK_NE(mState, UNINITIALIZED); 763 764 // mutex access to mBuffersInFlight and other stats 765 Mutex::Autolock al(mLatencyLock); 766 767 // how long this buffer took for the round trip through the codec 768 // NB: pipelining can/will make these times larger. e.g., if each packet 769 // is always 2 msec and we have 3 in flight at any given time, we're going to 770 // see "6 msec" as an answer. 771 772 // ignore stuff with no presentation time 773 if (presentationUs <= 0) { 774 ALOGV("-- returned buffer timestamp %" PRId64 " <= 0, ignore it", presentationUs); 775 mLatencyUnknown++; 776 return; 777 } 778 779 BufferFlightTiming_t startdata; 780 bool valid = false; 781 while (mBuffersInFlight.size() > 0) { 782 startdata = *mBuffersInFlight.begin(); 783 ALOGV("-- Looking at startdata. presentation %" PRId64 ", start %" PRId64, 784 startdata.presentationUs, startdata.startedNs); 785 if (startdata.presentationUs == presentationUs) { 786 // a match 787 ALOGV("-- match entry for %" PRId64 ", hits our frame of %" PRId64, 788 startdata.presentationUs, presentationUs); 789 mBuffersInFlight.pop_front(); 790 valid = true; 791 break; 792 } else if (startdata.presentationUs < presentationUs) { 793 // we must have missed the match for this, drop it and keep looking 794 ALOGV("-- drop entry for %" PRId64 ", before our frame of %" PRId64, 795 startdata.presentationUs, presentationUs); 796 mBuffersInFlight.pop_front(); 797 continue; 798 } else { 799 // head is after, so we don't have a frame for ourselves 800 ALOGV("-- found entry for %" PRId64 ", AFTER our frame of %" PRId64 801 " we have nothing to pair with", 802 startdata.presentationUs, presentationUs); 803 mLatencyUnknown++; 804 return; 805 } 806 } 807 if (!valid) { 808 ALOGV("-- empty queue, so ignore that."); 809 mLatencyUnknown++; 810 return; 811 } 812 813 // nowNs start our calculations 814 const int64_t nowNs = systemTime(SYSTEM_TIME_MONOTONIC); 815 int64_t latencyUs = (nowNs - startdata.startedNs + 500) / 1000; 816 817 mLatencyHist.insert(latencyUs); 818 819 // push into the recent samples 820 { 821 Mutex::Autolock al(mRecentLock); 822 823 if (mRecentHead >= kRecentLatencyFrames) { 824 mRecentHead = 0; 825 } 826 mRecentSamples[mRecentHead++] = latencyUs; 827 } 828 } 829 830 // static 831 status_t MediaCodec::PostAndAwaitResponse( 832 const sp<AMessage> &msg, sp<AMessage> *response) { 833 status_t err = msg->postAndAwaitResponse(response); 834 835 if (err != OK) { 836 return err; 837 } 838 839 if (!(*response)->findInt32("err", &err)) { 840 err = OK; 841 } 842 843 return err; 844 } 845 846 void MediaCodec::PostReplyWithError(const sp<AReplyToken> &replyID, int32_t err) { 847 int32_t finalErr = err; 848 if (mReleasedByResourceManager) { 849 // override the err code if MediaCodec has been released by ResourceManager. 850 finalErr = DEAD_OBJECT; 851 } 852 853 sp<AMessage> response = new AMessage; 854 response->setInt32("err", finalErr); 855 response->postReply(replyID); 856 } 857 858 static CodecBase *CreateCCodec() { 859 return StagefrightPluginLoader::GetCCodecInstance()->createCodec(); 860 } 861 862 //static 863 sp<CodecBase> MediaCodec::GetCodecBase(const AString &name, const char *owner) { 864 if (owner) { 865 if (strcmp(owner, "default") == 0) { 866 return new ACodec; 867 } else if (strncmp(owner, "codec2", 6) == 0) { 868 return CreateCCodec(); 869 } 870 } 871 872 if (name.startsWithIgnoreCase("c2.")) { 873 return CreateCCodec(); 874 } else if (name.startsWithIgnoreCase("omx.")) { 875 // at this time only ACodec specifies a mime type. 876 return new ACodec; 877 } else if (name.startsWithIgnoreCase("android.filter.")) { 878 return new MediaFilter; 879 } else { 880 return NULL; 881 } 882 } 883 884 status_t MediaCodec::init(const AString &name) { 885 mResourceManagerService->init(); 886 887 // save init parameters for reset 888 mInitName = name; 889 890 // Current video decoders do not return from OMX_FillThisBuffer 891 // quickly, violating the OpenMAX specs, until that is remedied 892 // we need to invest in an extra looper to free the main event 893 // queue. 894 895 mCodecInfo.clear(); 896 897 bool secureCodec = false; 898 AString tmp = name; 899 if (tmp.endsWith(".secure")) { 900 secureCodec = true; 901 tmp.erase(tmp.size() - 7, 7); 902 } 903 const sp<IMediaCodecList> mcl = MediaCodecList::getInstance(); 904 if (mcl == NULL) { 905 mCodec = NULL; // remove the codec. 906 return NO_INIT; // if called from Java should raise IOException 907 } 908 for (const AString &codecName : { name, tmp }) { 909 ssize_t codecIdx = mcl->findCodecByName(codecName.c_str()); 910 if (codecIdx < 0) { 911 continue; 912 } 913 mCodecInfo = mcl->getCodecInfo(codecIdx); 914 Vector<AString> mediaTypes; 915 mCodecInfo->getSupportedMediaTypes(&mediaTypes); 916 for (size_t i = 0; i < mediaTypes.size(); i++) { 917 if (mediaTypes[i].startsWith("video/")) { 918 mIsVideo = true; 919 break; 920 } 921 } 922 break; 923 } 924 if (mCodecInfo == nullptr) { 925 return NAME_NOT_FOUND; 926 } 927 928 mCodec = GetCodecBase(name, mCodecInfo->getOwnerName()); 929 if (mCodec == NULL) { 930 return NAME_NOT_FOUND; 931 } 932 933 if (mIsVideo) { 934 // video codec needs dedicated looper 935 if (mCodecLooper == NULL) { 936 mCodecLooper = new ALooper; 937 mCodecLooper->setName("CodecLooper"); 938 mCodecLooper->start(false, false, ANDROID_PRIORITY_AUDIO); 939 } 940 941 mCodecLooper->registerHandler(mCodec); 942 } else { 943 mLooper->registerHandler(mCodec); 944 } 945 946 mLooper->registerHandler(this); 947 948 mCodec->setCallback( 949 std::unique_ptr<CodecBase::CodecCallback>( 950 new CodecCallback(new AMessage(kWhatCodecNotify, this)))); 951 mBufferChannel = mCodec->getBufferChannel(); 952 mBufferChannel->setCallback( 953 std::unique_ptr<CodecBase::BufferCallback>( 954 new BufferCallback(new AMessage(kWhatCodecNotify, this)))); 955 956 sp<AMessage> msg = new AMessage(kWhatInit, this); 957 msg->setObject("codecInfo", mCodecInfo); 958 // name may be different from mCodecInfo->getCodecName() if we stripped 959 // ".secure" 960 msg->setString("name", name); 961 962 if (mAnalyticsItem != NULL) { 963 mAnalyticsItem->setCString(kCodecCodec, name.c_str()); 964 mAnalyticsItem->setCString(kCodecMode, mIsVideo ? kCodecModeVideo : kCodecModeAudio); 965 } 966 967 status_t err; 968 Vector<MediaResource> resources; 969 MediaResource::Type type = 970 secureCodec ? MediaResource::kSecureCodec : MediaResource::kNonSecureCodec; 971 MediaResource::SubType subtype = 972 mIsVideo ? MediaResource::kVideoCodec : MediaResource::kAudioCodec; 973 resources.push_back(MediaResource(type, subtype, 1)); 974 for (int i = 0; i <= kMaxRetry; ++i) { 975 if (i > 0) { 976 // Don't try to reclaim resource for the first time. 977 if (!mResourceManagerService->reclaimResource(resources)) { 978 break; 979 } 980 } 981 982 sp<AMessage> response; 983 err = PostAndAwaitResponse(msg, &response); 984 if (!isResourceError(err)) { 985 break; 986 } 987 } 988 return err; 989 } 990 991 status_t MediaCodec::setCallback(const sp<AMessage> &callback) { 992 sp<AMessage> msg = new AMessage(kWhatSetCallback, this); 993 msg->setMessage("callback", callback); 994 995 sp<AMessage> response; 996 return PostAndAwaitResponse(msg, &response); 997 } 998 999 status_t MediaCodec::setOnFrameRenderedNotification(const sp<AMessage> ¬ify) { 1000 sp<AMessage> msg = new AMessage(kWhatSetNotification, this); 1001 msg->setMessage("on-frame-rendered", notify); 1002 return msg->post(); 1003 } 1004 1005 status_t MediaCodec::configure( 1006 const sp<AMessage> &format, 1007 const sp<Surface> &nativeWindow, 1008 const sp<ICrypto> &crypto, 1009 uint32_t flags) { 1010 return configure(format, nativeWindow, crypto, NULL, flags); 1011 } 1012 1013 status_t MediaCodec::configure( 1014 const sp<AMessage> &format, 1015 const sp<Surface> &surface, 1016 const sp<ICrypto> &crypto, 1017 const sp<IDescrambler> &descrambler, 1018 uint32_t flags) { 1019 sp<AMessage> msg = new AMessage(kWhatConfigure, this); 1020 1021 if (mAnalyticsItem != NULL) { 1022 int32_t profile = 0; 1023 if (format->findInt32("profile", &profile)) { 1024 mAnalyticsItem->setInt32(kCodecProfile, profile); 1025 } 1026 int32_t level = 0; 1027 if (format->findInt32("level", &level)) { 1028 mAnalyticsItem->setInt32(kCodecLevel, level); 1029 } 1030 mAnalyticsItem->setInt32(kCodecEncoder, (flags & CONFIGURE_FLAG_ENCODE) ? 1 : 0); 1031 } 1032 1033 if (mIsVideo) { 1034 format->findInt32("width", &mVideoWidth); 1035 format->findInt32("height", &mVideoHeight); 1036 if (!format->findInt32("rotation-degrees", &mRotationDegrees)) { 1037 mRotationDegrees = 0; 1038 } 1039 1040 if (mAnalyticsItem != NULL) { 1041 mAnalyticsItem->setInt32(kCodecWidth, mVideoWidth); 1042 mAnalyticsItem->setInt32(kCodecHeight, mVideoHeight); 1043 mAnalyticsItem->setInt32(kCodecRotation, mRotationDegrees); 1044 int32_t maxWidth = 0; 1045 if (format->findInt32("max-width", &maxWidth)) { 1046 mAnalyticsItem->setInt32(kCodecMaxWidth, maxWidth); 1047 } 1048 int32_t maxHeight = 0; 1049 if (format->findInt32("max-height", &maxHeight)) { 1050 mAnalyticsItem->setInt32(kCodecMaxHeight, maxHeight); 1051 } 1052 } 1053 1054 // Prevent possible integer overflow in downstream code. 1055 if (mVideoWidth < 0 || mVideoHeight < 0 || 1056 (uint64_t)mVideoWidth * mVideoHeight > (uint64_t)INT32_MAX / 4) { 1057 ALOGE("Invalid size(s), width=%d, height=%d", mVideoWidth, mVideoHeight); 1058 return BAD_VALUE; 1059 } 1060 } 1061 1062 msg->setMessage("format", format); 1063 msg->setInt32("flags", flags); 1064 msg->setObject("surface", surface); 1065 1066 if (crypto != NULL || descrambler != NULL) { 1067 if (crypto != NULL) { 1068 msg->setPointer("crypto", crypto.get()); 1069 } else { 1070 msg->setPointer("descrambler", descrambler.get()); 1071 } 1072 if (mAnalyticsItem != NULL) { 1073 mAnalyticsItem->setInt32(kCodecCrypto, 1); 1074 } 1075 } else if (mFlags & kFlagIsSecure) { 1076 ALOGW("Crypto or descrambler should be given for secure codec"); 1077 } 1078 1079 // save msg for reset 1080 mConfigureMsg = msg; 1081 1082 status_t err; 1083 Vector<MediaResource> resources; 1084 MediaResource::Type type = (mFlags & kFlagIsSecure) ? 1085 MediaResource::kSecureCodec : MediaResource::kNonSecureCodec; 1086 MediaResource::SubType subtype = 1087 mIsVideo ? MediaResource::kVideoCodec : MediaResource::kAudioCodec; 1088 resources.push_back(MediaResource(type, subtype, 1)); 1089 // Don't know the buffer size at this point, but it's fine to use 1 because 1090 // the reclaimResource call doesn't consider the requester's buffer size for now. 1091 resources.push_back(MediaResource(MediaResource::kGraphicMemory, 1)); 1092 for (int i = 0; i <= kMaxRetry; ++i) { 1093 if (i > 0) { 1094 // Don't try to reclaim resource for the first time. 1095 if (!mResourceManagerService->reclaimResource(resources)) { 1096 break; 1097 } 1098 } 1099 1100 sp<AMessage> response; 1101 err = PostAndAwaitResponse(msg, &response); 1102 if (err != OK && err != INVALID_OPERATION) { 1103 // MediaCodec now set state to UNINITIALIZED upon any fatal error. 1104 // To maintain backward-compatibility, do a reset() to put codec 1105 // back into INITIALIZED state. 1106 // But don't reset if the err is INVALID_OPERATION, which means 1107 // the configure failure is due to wrong state. 1108 1109 ALOGE("configure failed with err 0x%08x, resetting...", err); 1110 reset(); 1111 } 1112 if (!isResourceError(err)) { 1113 break; 1114 } 1115 } 1116 1117 return err; 1118 } 1119 1120 status_t MediaCodec::releaseCrypto() 1121 { 1122 ALOGV("releaseCrypto"); 1123 1124 sp<AMessage> msg = new AMessage(kWhatDrmReleaseCrypto, this); 1125 1126 sp<AMessage> response; 1127 status_t status = msg->postAndAwaitResponse(&response); 1128 1129 if (status == OK && response != NULL) { 1130 CHECK(response->findInt32("status", &status)); 1131 ALOGV("releaseCrypto ret: %d ", status); 1132 } 1133 else { 1134 ALOGE("releaseCrypto err: %d", status); 1135 } 1136 1137 return status; 1138 } 1139 1140 void MediaCodec::onReleaseCrypto(const sp<AMessage>& msg) 1141 { 1142 status_t status = INVALID_OPERATION; 1143 if (mCrypto != NULL) { 1144 ALOGV("onReleaseCrypto: mCrypto: %p (%d)", mCrypto.get(), mCrypto->getStrongCount()); 1145 mBufferChannel->setCrypto(NULL); 1146 // TODO change to ALOGV 1147 ALOGD("onReleaseCrypto: [before clear] mCrypto: %p (%d)", 1148 mCrypto.get(), mCrypto->getStrongCount()); 1149 mCrypto.clear(); 1150 1151 status = OK; 1152 } 1153 else { 1154 ALOGW("onReleaseCrypto: No mCrypto. err: %d", status); 1155 } 1156 1157 sp<AMessage> response = new AMessage; 1158 response->setInt32("status", status); 1159 1160 sp<AReplyToken> replyID; 1161 CHECK(msg->senderAwaitsResponse(&replyID)); 1162 response->postReply(replyID); 1163 } 1164 1165 status_t MediaCodec::setInputSurface( 1166 const sp<PersistentSurface> &surface) { 1167 sp<AMessage> msg = new AMessage(kWhatSetInputSurface, this); 1168 msg->setObject("input-surface", surface.get()); 1169 1170 sp<AMessage> response; 1171 return PostAndAwaitResponse(msg, &response); 1172 } 1173 1174 status_t MediaCodec::setSurface(const sp<Surface> &surface) { 1175 sp<AMessage> msg = new AMessage(kWhatSetSurface, this); 1176 msg->setObject("surface", surface); 1177 1178 sp<AMessage> response; 1179 return PostAndAwaitResponse(msg, &response); 1180 } 1181 1182 status_t MediaCodec::createInputSurface( 1183 sp<IGraphicBufferProducer>* bufferProducer) { 1184 sp<AMessage> msg = new AMessage(kWhatCreateInputSurface, this); 1185 1186 sp<AMessage> response; 1187 status_t err = PostAndAwaitResponse(msg, &response); 1188 if (err == NO_ERROR) { 1189 // unwrap the sp<IGraphicBufferProducer> 1190 sp<RefBase> obj; 1191 bool found = response->findObject("input-surface", &obj); 1192 CHECK(found); 1193 sp<BufferProducerWrapper> wrapper( 1194 static_cast<BufferProducerWrapper*>(obj.get())); 1195 *bufferProducer = wrapper->getBufferProducer(); 1196 } else { 1197 ALOGW("createInputSurface failed, err=%d", err); 1198 } 1199 return err; 1200 } 1201 1202 uint64_t MediaCodec::getGraphicBufferSize() { 1203 if (!mIsVideo) { 1204 return 0; 1205 } 1206 1207 uint64_t size = 0; 1208 size_t portNum = sizeof(mPortBuffers) / sizeof((mPortBuffers)[0]); 1209 for (size_t i = 0; i < portNum; ++i) { 1210 // TODO: this is just an estimation, we should get the real buffer size from ACodec. 1211 size += mPortBuffers[i].size() * mVideoWidth * mVideoHeight * 3 / 2; 1212 } 1213 return size; 1214 } 1215 1216 void MediaCodec::addResource( 1217 MediaResource::Type type, MediaResource::SubType subtype, uint64_t value) { 1218 Vector<MediaResource> resources; 1219 resources.push_back(MediaResource(type, subtype, value)); 1220 mResourceManagerService->addResource( 1221 getId(mResourceManagerClient), mResourceManagerClient, resources); 1222 } 1223 1224 status_t MediaCodec::start() { 1225 sp<AMessage> msg = new AMessage(kWhatStart, this); 1226 1227 status_t err; 1228 Vector<MediaResource> resources; 1229 MediaResource::Type type = (mFlags & kFlagIsSecure) ? 1230 MediaResource::kSecureCodec : MediaResource::kNonSecureCodec; 1231 MediaResource::SubType subtype = 1232 mIsVideo ? MediaResource::kVideoCodec : MediaResource::kAudioCodec; 1233 resources.push_back(MediaResource(type, subtype, 1)); 1234 // Don't know the buffer size at this point, but it's fine to use 1 because 1235 // the reclaimResource call doesn't consider the requester's buffer size for now. 1236 resources.push_back(MediaResource(MediaResource::kGraphicMemory, 1)); 1237 for (int i = 0; i <= kMaxRetry; ++i) { 1238 if (i > 0) { 1239 // Don't try to reclaim resource for the first time. 1240 if (!mResourceManagerService->reclaimResource(resources)) { 1241 break; 1242 } 1243 // Recover codec from previous error before retry start. 1244 err = reset(); 1245 if (err != OK) { 1246 ALOGE("retrying start: failed to reset codec"); 1247 break; 1248 } 1249 sp<AMessage> response; 1250 err = PostAndAwaitResponse(mConfigureMsg, &response); 1251 if (err != OK) { 1252 ALOGE("retrying start: failed to configure codec"); 1253 break; 1254 } 1255 } 1256 1257 sp<AMessage> response; 1258 err = PostAndAwaitResponse(msg, &response); 1259 if (!isResourceError(err)) { 1260 break; 1261 } 1262 } 1263 return err; 1264 } 1265 1266 status_t MediaCodec::stop() { 1267 sp<AMessage> msg = new AMessage(kWhatStop, this); 1268 1269 sp<AMessage> response; 1270 return PostAndAwaitResponse(msg, &response); 1271 } 1272 1273 bool MediaCodec::hasPendingBuffer(int portIndex) { 1274 return std::any_of( 1275 mPortBuffers[portIndex].begin(), mPortBuffers[portIndex].end(), 1276 [](const BufferInfo &info) { return info.mOwnedByClient; }); 1277 } 1278 1279 bool MediaCodec::hasPendingBuffer() { 1280 return hasPendingBuffer(kPortIndexInput) || hasPendingBuffer(kPortIndexOutput); 1281 } 1282 1283 status_t MediaCodec::reclaim(bool force) { 1284 ALOGD("MediaCodec::reclaim(%p) %s", this, mInitName.c_str()); 1285 sp<AMessage> msg = new AMessage(kWhatRelease, this); 1286 msg->setInt32("reclaimed", 1); 1287 msg->setInt32("force", force ? 1 : 0); 1288 1289 sp<AMessage> response; 1290 status_t ret = PostAndAwaitResponse(msg, &response); 1291 if (ret == -ENOENT) { 1292 ALOGD("MediaCodec looper is gone, skip reclaim"); 1293 ret = OK; 1294 } 1295 return ret; 1296 } 1297 1298 status_t MediaCodec::release() { 1299 sp<AMessage> msg = new AMessage(kWhatRelease, this); 1300 1301 sp<AMessage> response; 1302 return PostAndAwaitResponse(msg, &response); 1303 } 1304 1305 status_t MediaCodec::reset() { 1306 /* When external-facing MediaCodec object is created, 1307 it is already initialized. Thus, reset is essentially 1308 release() followed by init(), plus clearing the state */ 1309 1310 status_t err = release(); 1311 1312 // unregister handlers 1313 if (mCodec != NULL) { 1314 if (mCodecLooper != NULL) { 1315 mCodecLooper->unregisterHandler(mCodec->id()); 1316 } else { 1317 mLooper->unregisterHandler(mCodec->id()); 1318 } 1319 mCodec = NULL; 1320 } 1321 mLooper->unregisterHandler(id()); 1322 1323 mFlags = 0; // clear all flags 1324 mStickyError = OK; 1325 1326 // reset state not reset by setState(UNINITIALIZED) 1327 mReplyID = 0; 1328 mDequeueInputReplyID = 0; 1329 mDequeueOutputReplyID = 0; 1330 mDequeueInputTimeoutGeneration = 0; 1331 mDequeueOutputTimeoutGeneration = 0; 1332 mHaveInputSurface = false; 1333 1334 if (err == OK) { 1335 err = init(mInitName); 1336 } 1337 return err; 1338 } 1339 1340 status_t MediaCodec::queueInputBuffer( 1341 size_t index, 1342 size_t offset, 1343 size_t size, 1344 int64_t presentationTimeUs, 1345 uint32_t flags, 1346 AString *errorDetailMsg) { 1347 if (errorDetailMsg != NULL) { 1348 errorDetailMsg->clear(); 1349 } 1350 1351 sp<AMessage> msg = new AMessage(kWhatQueueInputBuffer, this); 1352 msg->setSize("index", index); 1353 msg->setSize("offset", offset); 1354 msg->setSize("size", size); 1355 msg->setInt64("timeUs", presentationTimeUs); 1356 msg->setInt32("flags", flags); 1357 msg->setPointer("errorDetailMsg", errorDetailMsg); 1358 1359 sp<AMessage> response; 1360 return PostAndAwaitResponse(msg, &response); 1361 } 1362 1363 status_t MediaCodec::queueSecureInputBuffer( 1364 size_t index, 1365 size_t offset, 1366 const CryptoPlugin::SubSample *subSamples, 1367 size_t numSubSamples, 1368 const uint8_t key[16], 1369 const uint8_t iv[16], 1370 CryptoPlugin::Mode mode, 1371 const CryptoPlugin::Pattern &pattern, 1372 int64_t presentationTimeUs, 1373 uint32_t flags, 1374 AString *errorDetailMsg) { 1375 if (errorDetailMsg != NULL) { 1376 errorDetailMsg->clear(); 1377 } 1378 1379 sp<AMessage> msg = new AMessage(kWhatQueueInputBuffer, this); 1380 msg->setSize("index", index); 1381 msg->setSize("offset", offset); 1382 msg->setPointer("subSamples", (void *)subSamples); 1383 msg->setSize("numSubSamples", numSubSamples); 1384 msg->setPointer("key", (void *)key); 1385 msg->setPointer("iv", (void *)iv); 1386 msg->setInt32("mode", mode); 1387 msg->setInt32("encryptBlocks", pattern.mEncryptBlocks); 1388 msg->setInt32("skipBlocks", pattern.mSkipBlocks); 1389 msg->setInt64("timeUs", presentationTimeUs); 1390 msg->setInt32("flags", flags); 1391 msg->setPointer("errorDetailMsg", errorDetailMsg); 1392 1393 sp<AMessage> response; 1394 status_t err = PostAndAwaitResponse(msg, &response); 1395 1396 return err; 1397 } 1398 1399 status_t MediaCodec::dequeueInputBuffer(size_t *index, int64_t timeoutUs) { 1400 sp<AMessage> msg = new AMessage(kWhatDequeueInputBuffer, this); 1401 msg->setInt64("timeoutUs", timeoutUs); 1402 1403 sp<AMessage> response; 1404 status_t err; 1405 if ((err = PostAndAwaitResponse(msg, &response)) != OK) { 1406 return err; 1407 } 1408 1409 CHECK(response->findSize("index", index)); 1410 1411 return OK; 1412 } 1413 1414 status_t MediaCodec::dequeueOutputBuffer( 1415 size_t *index, 1416 size_t *offset, 1417 size_t *size, 1418 int64_t *presentationTimeUs, 1419 uint32_t *flags, 1420 int64_t timeoutUs) { 1421 sp<AMessage> msg = new AMessage(kWhatDequeueOutputBuffer, this); 1422 msg->setInt64("timeoutUs", timeoutUs); 1423 1424 sp<AMessage> response; 1425 status_t err; 1426 if ((err = PostAndAwaitResponse(msg, &response)) != OK) { 1427 return err; 1428 } 1429 1430 CHECK(response->findSize("index", index)); 1431 CHECK(response->findSize("offset", offset)); 1432 CHECK(response->findSize("size", size)); 1433 CHECK(response->findInt64("timeUs", presentationTimeUs)); 1434 CHECK(response->findInt32("flags", (int32_t *)flags)); 1435 1436 return OK; 1437 } 1438 1439 status_t MediaCodec::renderOutputBufferAndRelease(size_t index) { 1440 sp<AMessage> msg = new AMessage(kWhatReleaseOutputBuffer, this); 1441 msg->setSize("index", index); 1442 msg->setInt32("render", true); 1443 1444 sp<AMessage> response; 1445 return PostAndAwaitResponse(msg, &response); 1446 } 1447 1448 status_t MediaCodec::renderOutputBufferAndRelease(size_t index, int64_t timestampNs) { 1449 sp<AMessage> msg = new AMessage(kWhatReleaseOutputBuffer, this); 1450 msg->setSize("index", index); 1451 msg->setInt32("render", true); 1452 msg->setInt64("timestampNs", timestampNs); 1453 1454 sp<AMessage> response; 1455 return PostAndAwaitResponse(msg, &response); 1456 } 1457 1458 status_t MediaCodec::releaseOutputBuffer(size_t index) { 1459 sp<AMessage> msg = new AMessage(kWhatReleaseOutputBuffer, this); 1460 msg->setSize("index", index); 1461 1462 sp<AMessage> response; 1463 return PostAndAwaitResponse(msg, &response); 1464 } 1465 1466 status_t MediaCodec::signalEndOfInputStream() { 1467 sp<AMessage> msg = new AMessage(kWhatSignalEndOfInputStream, this); 1468 1469 sp<AMessage> response; 1470 return PostAndAwaitResponse(msg, &response); 1471 } 1472 1473 status_t MediaCodec::getOutputFormat(sp<AMessage> *format) const { 1474 sp<AMessage> msg = new AMessage(kWhatGetOutputFormat, this); 1475 1476 sp<AMessage> response; 1477 status_t err; 1478 if ((err = PostAndAwaitResponse(msg, &response)) != OK) { 1479 return err; 1480 } 1481 1482 CHECK(response->findMessage("format", format)); 1483 1484 return OK; 1485 } 1486 1487 status_t MediaCodec::getInputFormat(sp<AMessage> *format) const { 1488 sp<AMessage> msg = new AMessage(kWhatGetInputFormat, this); 1489 1490 sp<AMessage> response; 1491 status_t err; 1492 if ((err = PostAndAwaitResponse(msg, &response)) != OK) { 1493 return err; 1494 } 1495 1496 CHECK(response->findMessage("format", format)); 1497 1498 return OK; 1499 } 1500 1501 status_t MediaCodec::getName(AString *name) const { 1502 sp<AMessage> msg = new AMessage(kWhatGetName, this); 1503 1504 sp<AMessage> response; 1505 status_t err; 1506 if ((err = PostAndAwaitResponse(msg, &response)) != OK) { 1507 return err; 1508 } 1509 1510 CHECK(response->findString("name", name)); 1511 1512 return OK; 1513 } 1514 1515 status_t MediaCodec::getCodecInfo(sp<MediaCodecInfo> *codecInfo) const { 1516 sp<AMessage> msg = new AMessage(kWhatGetCodecInfo, this); 1517 1518 sp<AMessage> response; 1519 status_t err; 1520 if ((err = PostAndAwaitResponse(msg, &response)) != OK) { 1521 return err; 1522 } 1523 1524 sp<RefBase> obj; 1525 CHECK(response->findObject("codecInfo", &obj)); 1526 *codecInfo = static_cast<MediaCodecInfo *>(obj.get()); 1527 1528 return OK; 1529 } 1530 1531 status_t MediaCodec::getMetrics(MediaAnalyticsItem * &reply) { 1532 1533 reply = NULL; 1534 1535 // shouldn't happen, but be safe 1536 if (mAnalyticsItem == NULL) { 1537 return UNKNOWN_ERROR; 1538 } 1539 1540 // update any in-flight data that's not carried within the record 1541 updateAnalyticsItem(); 1542 1543 // send it back to the caller. 1544 reply = mAnalyticsItem->dup(); 1545 1546 updateEphemeralAnalytics(reply); 1547 1548 return OK; 1549 } 1550 1551 status_t MediaCodec::getInputBuffers(Vector<sp<MediaCodecBuffer> > *buffers) const { 1552 sp<AMessage> msg = new AMessage(kWhatGetBuffers, this); 1553 msg->setInt32("portIndex", kPortIndexInput); 1554 msg->setPointer("buffers", buffers); 1555 1556 sp<AMessage> response; 1557 return PostAndAwaitResponse(msg, &response); 1558 } 1559 1560 status_t MediaCodec::getOutputBuffers(Vector<sp<MediaCodecBuffer> > *buffers) const { 1561 sp<AMessage> msg = new AMessage(kWhatGetBuffers, this); 1562 msg->setInt32("portIndex", kPortIndexOutput); 1563 msg->setPointer("buffers", buffers); 1564 1565 sp<AMessage> response; 1566 return PostAndAwaitResponse(msg, &response); 1567 } 1568 1569 status_t MediaCodec::getOutputBuffer(size_t index, sp<MediaCodecBuffer> *buffer) { 1570 sp<AMessage> format; 1571 return getBufferAndFormat(kPortIndexOutput, index, buffer, &format); 1572 } 1573 1574 status_t MediaCodec::getOutputFormat(size_t index, sp<AMessage> *format) { 1575 sp<MediaCodecBuffer> buffer; 1576 return getBufferAndFormat(kPortIndexOutput, index, &buffer, format); 1577 } 1578 1579 status_t MediaCodec::getInputBuffer(size_t index, sp<MediaCodecBuffer> *buffer) { 1580 sp<AMessage> format; 1581 return getBufferAndFormat(kPortIndexInput, index, buffer, &format); 1582 } 1583 1584 bool MediaCodec::isExecuting() const { 1585 return mState == STARTED || mState == FLUSHED; 1586 } 1587 1588 status_t MediaCodec::getBufferAndFormat( 1589 size_t portIndex, size_t index, 1590 sp<MediaCodecBuffer> *buffer, sp<AMessage> *format) { 1591 // use mutex instead of a context switch 1592 if (mReleasedByResourceManager) { 1593 ALOGE("getBufferAndFormat - resource already released"); 1594 return DEAD_OBJECT; 1595 } 1596 1597 if (buffer == NULL) { 1598 ALOGE("getBufferAndFormat - null MediaCodecBuffer"); 1599 return INVALID_OPERATION; 1600 } 1601 1602 if (format == NULL) { 1603 ALOGE("getBufferAndFormat - null AMessage"); 1604 return INVALID_OPERATION; 1605 } 1606 1607 buffer->clear(); 1608 format->clear(); 1609 1610 if (!isExecuting()) { 1611 ALOGE("getBufferAndFormat - not executing"); 1612 return INVALID_OPERATION; 1613 } 1614 1615 // we do not want mPortBuffers to change during this section 1616 // we also don't want mOwnedByClient to change during this 1617 Mutex::Autolock al(mBufferLock); 1618 1619 std::vector<BufferInfo> &buffers = mPortBuffers[portIndex]; 1620 if (index >= buffers.size()) { 1621 ALOGE("getBufferAndFormat - trying to get buffer with " 1622 "bad index (index=%zu buffer_size=%zu)", index, buffers.size()); 1623 return INVALID_OPERATION; 1624 } 1625 1626 const BufferInfo &info = buffers[index]; 1627 if (!info.mOwnedByClient) { 1628 ALOGE("getBufferAndFormat - invalid operation " 1629 "(the index %zu is not owned by client)", index); 1630 return INVALID_OPERATION; 1631 } 1632 1633 *buffer = info.mData; 1634 *format = info.mData->format(); 1635 1636 return OK; 1637 } 1638 1639 status_t MediaCodec::flush() { 1640 sp<AMessage> msg = new AMessage(kWhatFlush, this); 1641 1642 sp<AMessage> response; 1643 return PostAndAwaitResponse(msg, &response); 1644 } 1645 1646 status_t MediaCodec::requestIDRFrame() { 1647 (new AMessage(kWhatRequestIDRFrame, this))->post(); 1648 1649 return OK; 1650 } 1651 1652 void MediaCodec::requestActivityNotification(const sp<AMessage> ¬ify) { 1653 sp<AMessage> msg = new AMessage(kWhatRequestActivityNotification, this); 1654 msg->setMessage("notify", notify); 1655 msg->post(); 1656 } 1657 1658 void MediaCodec::requestCpuBoostIfNeeded() { 1659 if (mCpuBoostRequested) { 1660 return; 1661 } 1662 int32_t colorFormat; 1663 if (mOutputFormat->contains("hdr-static-info") 1664 && mOutputFormat->findInt32("color-format", &colorFormat) 1665 // check format for OMX only, for C2 the format is always opaque since the 1666 // software rendering doesn't go through client 1667 && ((mSoftRenderer != NULL && colorFormat == OMX_COLOR_FormatYUV420Planar16) 1668 || mOwnerName.equalsIgnoreCase("codec2::software"))) { 1669 int32_t left, top, right, bottom, width, height; 1670 int64_t totalPixel = 0; 1671 if (mOutputFormat->findRect("crop", &left, &top, &right, &bottom)) { 1672 totalPixel = (right - left + 1) * (bottom - top + 1); 1673 } else if (mOutputFormat->findInt32("width", &width) 1674 && mOutputFormat->findInt32("height", &height)) { 1675 totalPixel = width * height; 1676 } 1677 if (totalPixel >= 1920 * 1080) { 1678 addResource(MediaResource::kCpuBoost, 1679 MediaResource::kUnspecifiedSubType, 1); 1680 mCpuBoostRequested = true; 1681 } 1682 } 1683 } 1684 1685 //////////////////////////////////////////////////////////////////////////////// 1686 1687 void MediaCodec::cancelPendingDequeueOperations() { 1688 if (mFlags & kFlagDequeueInputPending) { 1689 PostReplyWithError(mDequeueInputReplyID, INVALID_OPERATION); 1690 1691 ++mDequeueInputTimeoutGeneration; 1692 mDequeueInputReplyID = 0; 1693 mFlags &= ~kFlagDequeueInputPending; 1694 } 1695 1696 if (mFlags & kFlagDequeueOutputPending) { 1697 PostReplyWithError(mDequeueOutputReplyID, INVALID_OPERATION); 1698 1699 ++mDequeueOutputTimeoutGeneration; 1700 mDequeueOutputReplyID = 0; 1701 mFlags &= ~kFlagDequeueOutputPending; 1702 } 1703 } 1704 1705 bool MediaCodec::handleDequeueInputBuffer(const sp<AReplyToken> &replyID, bool newRequest) { 1706 if (!isExecuting() || (mFlags & kFlagIsAsync) 1707 || (newRequest && (mFlags & kFlagDequeueInputPending))) { 1708 PostReplyWithError(replyID, INVALID_OPERATION); 1709 return true; 1710 } else if (mFlags & kFlagStickyError) { 1711 PostReplyWithError(replyID, getStickyError()); 1712 return true; 1713 } 1714 1715 ssize_t index = dequeuePortBuffer(kPortIndexInput); 1716 1717 if (index < 0) { 1718 CHECK_EQ(index, -EAGAIN); 1719 return false; 1720 } 1721 1722 sp<AMessage> response = new AMessage; 1723 response->setSize("index", index); 1724 response->postReply(replyID); 1725 1726 return true; 1727 } 1728 1729 bool MediaCodec::handleDequeueOutputBuffer(const sp<AReplyToken> &replyID, bool newRequest) { 1730 if (!isExecuting() || (mFlags & kFlagIsAsync) 1731 || (newRequest && (mFlags & kFlagDequeueOutputPending))) { 1732 PostReplyWithError(replyID, INVALID_OPERATION); 1733 } else if (mFlags & kFlagStickyError) { 1734 PostReplyWithError(replyID, getStickyError()); 1735 } else if (mFlags & kFlagOutputBuffersChanged) { 1736 PostReplyWithError(replyID, INFO_OUTPUT_BUFFERS_CHANGED); 1737 mFlags &= ~kFlagOutputBuffersChanged; 1738 } else if (mFlags & kFlagOutputFormatChanged) { 1739 PostReplyWithError(replyID, INFO_FORMAT_CHANGED); 1740 mFlags &= ~kFlagOutputFormatChanged; 1741 } else { 1742 sp<AMessage> response = new AMessage; 1743 ssize_t index = dequeuePortBuffer(kPortIndexOutput); 1744 1745 if (index < 0) { 1746 CHECK_EQ(index, -EAGAIN); 1747 return false; 1748 } 1749 1750 const sp<MediaCodecBuffer> &buffer = 1751 mPortBuffers[kPortIndexOutput][index].mData; 1752 1753 response->setSize("index", index); 1754 response->setSize("offset", buffer->offset()); 1755 response->setSize("size", buffer->size()); 1756 1757 int64_t timeUs; 1758 CHECK(buffer->meta()->findInt64("timeUs", &timeUs)); 1759 1760 statsBufferReceived(timeUs); 1761 1762 response->setInt64("timeUs", timeUs); 1763 1764 int32_t flags; 1765 CHECK(buffer->meta()->findInt32("flags", &flags)); 1766 1767 response->setInt32("flags", flags); 1768 response->postReply(replyID); 1769 } 1770 1771 return true; 1772 } 1773 1774 void MediaCodec::onMessageReceived(const sp<AMessage> &msg) { 1775 switch (msg->what()) { 1776 case kWhatCodecNotify: 1777 { 1778 int32_t what; 1779 CHECK(msg->findInt32("what", &what)); 1780 1781 switch (what) { 1782 case kWhatError: 1783 { 1784 int32_t err, actionCode; 1785 CHECK(msg->findInt32("err", &err)); 1786 CHECK(msg->findInt32("actionCode", &actionCode)); 1787 1788 ALOGE("Codec reported err %#x, actionCode %d, while in state %d", 1789 err, actionCode, mState); 1790 if (err == DEAD_OBJECT) { 1791 mFlags |= kFlagSawMediaServerDie; 1792 mFlags &= ~kFlagIsComponentAllocated; 1793 } 1794 1795 bool sendErrorResponse = true; 1796 1797 switch (mState) { 1798 case INITIALIZING: 1799 { 1800 setState(UNINITIALIZED); 1801 break; 1802 } 1803 1804 case CONFIGURING: 1805 { 1806 if (actionCode == ACTION_CODE_FATAL) { 1807 mAnalyticsItem->setInt32(kCodecError, err); 1808 mAnalyticsItem->setCString(kCodecErrorState, stateString(mState).c_str()); 1809 flushAnalyticsItem(); 1810 initAnalyticsItem(); 1811 } 1812 setState(actionCode == ACTION_CODE_FATAL ? 1813 UNINITIALIZED : INITIALIZED); 1814 break; 1815 } 1816 1817 case STARTING: 1818 { 1819 if (actionCode == ACTION_CODE_FATAL) { 1820 mAnalyticsItem->setInt32(kCodecError, err); 1821 mAnalyticsItem->setCString(kCodecErrorState, stateString(mState).c_str()); 1822 flushAnalyticsItem(); 1823 initAnalyticsItem(); 1824 } 1825 setState(actionCode == ACTION_CODE_FATAL ? 1826 UNINITIALIZED : CONFIGURED); 1827 break; 1828 } 1829 1830 case RELEASING: 1831 { 1832 // Ignore the error, assuming we'll still get 1833 // the shutdown complete notification. If we 1834 // don't, we'll timeout and force release. 1835 sendErrorResponse = false; 1836 FALLTHROUGH_INTENDED; 1837 } 1838 case STOPPING: 1839 { 1840 if (mFlags & kFlagSawMediaServerDie) { 1841 // MediaServer died, there definitely won't 1842 // be a shutdown complete notification after 1843 // all. 1844 1845 // note that we're directly going from 1846 // STOPPING->UNINITIALIZED, instead of the 1847 // usual STOPPING->INITIALIZED state. 1848 setState(UNINITIALIZED); 1849 if (mState == RELEASING) { 1850 mComponentName.clear(); 1851 } 1852 (new AMessage)->postReply(mReplyID); 1853 sendErrorResponse = false; 1854 } 1855 break; 1856 } 1857 1858 case FLUSHING: 1859 { 1860 if (actionCode == ACTION_CODE_FATAL) { 1861 mAnalyticsItem->setInt32(kCodecError, err); 1862 mAnalyticsItem->setCString(kCodecErrorState, stateString(mState).c_str()); 1863 flushAnalyticsItem(); 1864 initAnalyticsItem(); 1865 1866 setState(UNINITIALIZED); 1867 } else { 1868 setState( 1869 (mFlags & kFlagIsAsync) ? FLUSHED : STARTED); 1870 } 1871 break; 1872 } 1873 1874 case FLUSHED: 1875 case STARTED: 1876 { 1877 sendErrorResponse = false; 1878 1879 setStickyError(err); 1880 postActivityNotificationIfPossible(); 1881 1882 cancelPendingDequeueOperations(); 1883 1884 if (mFlags & kFlagIsAsync) { 1885 onError(err, actionCode); 1886 } 1887 switch (actionCode) { 1888 case ACTION_CODE_TRANSIENT: 1889 break; 1890 case ACTION_CODE_RECOVERABLE: 1891 setState(INITIALIZED); 1892 break; 1893 default: 1894 mAnalyticsItem->setInt32(kCodecError, err); 1895 mAnalyticsItem->setCString(kCodecErrorState, stateString(mState).c_str()); 1896 flushAnalyticsItem(); 1897 initAnalyticsItem(); 1898 setState(UNINITIALIZED); 1899 break; 1900 } 1901 break; 1902 } 1903 1904 default: 1905 { 1906 sendErrorResponse = false; 1907 1908 setStickyError(err); 1909 postActivityNotificationIfPossible(); 1910 1911 // actionCode in an uninitialized state is always fatal. 1912 if (mState == UNINITIALIZED) { 1913 actionCode = ACTION_CODE_FATAL; 1914 } 1915 if (mFlags & kFlagIsAsync) { 1916 onError(err, actionCode); 1917 } 1918 switch (actionCode) { 1919 case ACTION_CODE_TRANSIENT: 1920 break; 1921 case ACTION_CODE_RECOVERABLE: 1922 setState(INITIALIZED); 1923 break; 1924 default: 1925 setState(UNINITIALIZED); 1926 break; 1927 } 1928 break; 1929 } 1930 } 1931 1932 if (sendErrorResponse) { 1933 PostReplyWithError(mReplyID, err); 1934 } 1935 break; 1936 } 1937 1938 case kWhatComponentAllocated: 1939 { 1940 if (mState == RELEASING || mState == UNINITIALIZED) { 1941 // In case a kWhatError or kWhatRelease message came in and replied, 1942 // we log a warning and ignore. 1943 ALOGW("allocate interrupted by error or release, current state %d", 1944 mState); 1945 break; 1946 } 1947 CHECK_EQ(mState, INITIALIZING); 1948 setState(INITIALIZED); 1949 mFlags |= kFlagIsComponentAllocated; 1950 1951 CHECK(msg->findString("componentName", &mComponentName)); 1952 1953 if (mComponentName.c_str()) { 1954 mAnalyticsItem->setCString(kCodecCodec, mComponentName.c_str()); 1955 } 1956 1957 const char *owner = mCodecInfo->getOwnerName(); 1958 if (mComponentName.startsWith("OMX.google.") 1959 && (owner == nullptr || strncmp(owner, "default", 8) == 0)) { 1960 mFlags |= kFlagUsesSoftwareRenderer; 1961 } else { 1962 mFlags &= ~kFlagUsesSoftwareRenderer; 1963 } 1964 mOwnerName = owner; 1965 1966 MediaResource::Type resourceType; 1967 if (mComponentName.endsWith(".secure")) { 1968 mFlags |= kFlagIsSecure; 1969 resourceType = MediaResource::kSecureCodec; 1970 mAnalyticsItem->setInt32(kCodecSecure, 1); 1971 } else { 1972 mFlags &= ~kFlagIsSecure; 1973 resourceType = MediaResource::kNonSecureCodec; 1974 mAnalyticsItem->setInt32(kCodecSecure, 0); 1975 } 1976 1977 if (mIsVideo) { 1978 // audio codec is currently ignored. 1979 addResource(resourceType, MediaResource::kVideoCodec, 1); 1980 } 1981 1982 (new AMessage)->postReply(mReplyID); 1983 break; 1984 } 1985 1986 case kWhatComponentConfigured: 1987 { 1988 if (mState == RELEASING || mState == UNINITIALIZED || mState == INITIALIZED) { 1989 // In case a kWhatError or kWhatRelease message came in and replied, 1990 // we log a warning and ignore. 1991 ALOGW("configure interrupted by error or release, current state %d", 1992 mState); 1993 break; 1994 } 1995 CHECK_EQ(mState, CONFIGURING); 1996 1997 // reset input surface flag 1998 mHaveInputSurface = false; 1999 2000 CHECK(msg->findMessage("input-format", &mInputFormat)); 2001 CHECK(msg->findMessage("output-format", &mOutputFormat)); 2002 2003 // limit to confirming the opt-in behavior to minimize any behavioral change 2004 if (mSurface != nullptr && !mAllowFrameDroppingBySurface) { 2005 // signal frame dropping mode in the input format as this may also be 2006 // meaningful and confusing for an encoder in a transcoder scenario 2007 mInputFormat->setInt32("allow-frame-drop", mAllowFrameDroppingBySurface); 2008 } 2009 ALOGV("[%s] configured as input format: %s, output format: %s", 2010 mComponentName.c_str(), 2011 mInputFormat->debugString(4).c_str(), 2012 mOutputFormat->debugString(4).c_str()); 2013 int32_t usingSwRenderer; 2014 if (mOutputFormat->findInt32("using-sw-renderer", &usingSwRenderer) 2015 && usingSwRenderer) { 2016 mFlags |= kFlagUsesSoftwareRenderer; 2017 } 2018 setState(CONFIGURED); 2019 (new AMessage)->postReply(mReplyID); 2020 2021 // augment our media metrics info, now that we know more things 2022 if (mAnalyticsItem != NULL) { 2023 sp<AMessage> format; 2024 if (mConfigureMsg != NULL && 2025 mConfigureMsg->findMessage("format", &format)) { 2026 // format includes: mime 2027 AString mime; 2028 if (format->findString("mime", &mime)) { 2029 mAnalyticsItem->setCString(kCodecMime, mime.c_str()); 2030 } 2031 } 2032 } 2033 break; 2034 } 2035 2036 case kWhatInputSurfaceCreated: 2037 { 2038 // response to initiateCreateInputSurface() 2039 status_t err = NO_ERROR; 2040 sp<AMessage> response = new AMessage; 2041 if (!msg->findInt32("err", &err)) { 2042 sp<RefBase> obj; 2043 msg->findObject("input-surface", &obj); 2044 CHECK(msg->findMessage("input-format", &mInputFormat)); 2045 CHECK(msg->findMessage("output-format", &mOutputFormat)); 2046 ALOGV("[%s] input surface created as input format: %s, output format: %s", 2047 mComponentName.c_str(), 2048 mInputFormat->debugString(4).c_str(), 2049 mOutputFormat->debugString(4).c_str()); 2050 CHECK(obj != NULL); 2051 response->setObject("input-surface", obj); 2052 mHaveInputSurface = true; 2053 } else { 2054 response->setInt32("err", err); 2055 } 2056 response->postReply(mReplyID); 2057 break; 2058 } 2059 2060 case kWhatInputSurfaceAccepted: 2061 { 2062 // response to initiateSetInputSurface() 2063 status_t err = NO_ERROR; 2064 sp<AMessage> response = new AMessage(); 2065 if (!msg->findInt32("err", &err)) { 2066 CHECK(msg->findMessage("input-format", &mInputFormat)); 2067 CHECK(msg->findMessage("output-format", &mOutputFormat)); 2068 mHaveInputSurface = true; 2069 } else { 2070 response->setInt32("err", err); 2071 } 2072 response->postReply(mReplyID); 2073 break; 2074 } 2075 2076 case kWhatSignaledInputEOS: 2077 { 2078 // response to signalEndOfInputStream() 2079 sp<AMessage> response = new AMessage; 2080 status_t err; 2081 if (msg->findInt32("err", &err)) { 2082 response->setInt32("err", err); 2083 } 2084 response->postReply(mReplyID); 2085 break; 2086 } 2087 2088 case kWhatStartCompleted: 2089 { 2090 if (mState == RELEASING || mState == UNINITIALIZED) { 2091 // In case a kWhatRelease message came in and replied, 2092 // we log a warning and ignore. 2093 ALOGW("start interrupted by release, current state %d", mState); 2094 break; 2095 } 2096 2097 CHECK_EQ(mState, STARTING); 2098 if (mIsVideo) { 2099 addResource( 2100 MediaResource::kGraphicMemory, 2101 MediaResource::kUnspecifiedSubType, 2102 getGraphicBufferSize()); 2103 } 2104 setState(STARTED); 2105 (new AMessage)->postReply(mReplyID); 2106 break; 2107 } 2108 2109 case kWhatOutputBuffersChanged: 2110 { 2111 mFlags |= kFlagOutputBuffersChanged; 2112 postActivityNotificationIfPossible(); 2113 break; 2114 } 2115 2116 case kWhatOutputFramesRendered: 2117 { 2118 // ignore these in all states except running, and check that we have a 2119 // notification set 2120 if (mState == STARTED && mOnFrameRenderedNotification != NULL) { 2121 sp<AMessage> notify = mOnFrameRenderedNotification->dup(); 2122 notify->setMessage("data", msg); 2123 notify->post(); 2124 } 2125 break; 2126 } 2127 2128 case kWhatFillThisBuffer: 2129 { 2130 /* size_t index = */updateBuffers(kPortIndexInput, msg); 2131 2132 if (mState == FLUSHING 2133 || mState == STOPPING 2134 || mState == RELEASING) { 2135 returnBuffersToCodecOnPort(kPortIndexInput); 2136 break; 2137 } 2138 2139 if (!mCSD.empty()) { 2140 ssize_t index = dequeuePortBuffer(kPortIndexInput); 2141 CHECK_GE(index, 0); 2142 2143 // If codec specific data had been specified as 2144 // part of the format in the call to configure and 2145 // if there's more csd left, we submit it here 2146 // clients only get access to input buffers once 2147 // this data has been exhausted. 2148 2149 status_t err = queueCSDInputBuffer(index); 2150 2151 if (err != OK) { 2152 ALOGE("queueCSDInputBuffer failed w/ error %d", 2153 err); 2154 2155 setStickyError(err); 2156 postActivityNotificationIfPossible(); 2157 2158 cancelPendingDequeueOperations(); 2159 } 2160 break; 2161 } 2162 2163 if (mFlags & kFlagIsAsync) { 2164 if (!mHaveInputSurface) { 2165 if (mState == FLUSHED) { 2166 mHavePendingInputBuffers = true; 2167 } else { 2168 onInputBufferAvailable(); 2169 } 2170 } 2171 } else if (mFlags & kFlagDequeueInputPending) { 2172 CHECK(handleDequeueInputBuffer(mDequeueInputReplyID)); 2173 2174 ++mDequeueInputTimeoutGeneration; 2175 mFlags &= ~kFlagDequeueInputPending; 2176 mDequeueInputReplyID = 0; 2177 } else { 2178 postActivityNotificationIfPossible(); 2179 } 2180 break; 2181 } 2182 2183 case kWhatDrainThisBuffer: 2184 { 2185 /* size_t index = */updateBuffers(kPortIndexOutput, msg); 2186 2187 if (mState == FLUSHING 2188 || mState == STOPPING 2189 || mState == RELEASING) { 2190 returnBuffersToCodecOnPort(kPortIndexOutput); 2191 break; 2192 } 2193 2194 sp<RefBase> obj; 2195 CHECK(msg->findObject("buffer", &obj)); 2196 sp<MediaCodecBuffer> buffer = static_cast<MediaCodecBuffer *>(obj.get()); 2197 2198 if (mOutputFormat != buffer->format()) { 2199 mOutputFormat = buffer->format(); 2200 ALOGV("[%s] output format changed to: %s", 2201 mComponentName.c_str(), mOutputFormat->debugString(4).c_str()); 2202 2203 if (mSoftRenderer == NULL && 2204 mSurface != NULL && 2205 (mFlags & kFlagUsesSoftwareRenderer)) { 2206 AString mime; 2207 CHECK(mOutputFormat->findString("mime", &mime)); 2208 2209 // TODO: propagate color aspects to software renderer to allow better 2210 // color conversion to RGB. For now, just mark dataspace for YUV 2211 // rendering. 2212 int32_t dataSpace; 2213 if (mOutputFormat->findInt32("android._dataspace", &dataSpace)) { 2214 ALOGD("[%s] setting dataspace on output surface to #%x", 2215 mComponentName.c_str(), dataSpace); 2216 int err = native_window_set_buffers_data_space( 2217 mSurface.get(), (android_dataspace)dataSpace); 2218 ALOGW_IF(err != 0, "failed to set dataspace on surface (%d)", err); 2219 } 2220 if (mOutputFormat->contains("hdr-static-info")) { 2221 HDRStaticInfo info; 2222 if (ColorUtils::getHDRStaticInfoFromFormat(mOutputFormat, &info)) { 2223 setNativeWindowHdrMetadata(mSurface.get(), &info); 2224 } 2225 } 2226 2227 sp<ABuffer> hdr10PlusInfo; 2228 if (mOutputFormat->findBuffer("hdr10-plus-info", &hdr10PlusInfo) 2229 && hdr10PlusInfo != nullptr && hdr10PlusInfo->size() > 0) { 2230 native_window_set_buffers_hdr10_plus_metadata(mSurface.get(), 2231 hdr10PlusInfo->size(), hdr10PlusInfo->data()); 2232 } 2233 2234 if (mime.startsWithIgnoreCase("video/")) { 2235 mSurface->setDequeueTimeout(-1); 2236 mSoftRenderer = new SoftwareRenderer(mSurface, mRotationDegrees); 2237 } 2238 } 2239 2240 requestCpuBoostIfNeeded(); 2241 2242 if (mFlags & kFlagIsEncoder) { 2243 // Before we announce the format change we should 2244 // collect codec specific data and amend the output 2245 // format as necessary. 2246 int32_t flags = 0; 2247 (void) buffer->meta()->findInt32("flags", &flags); 2248 if (flags & BUFFER_FLAG_CODECCONFIG) { 2249 status_t err = 2250 amendOutputFormatWithCodecSpecificData(buffer); 2251 2252 if (err != OK) { 2253 ALOGE("Codec spit out malformed codec " 2254 "specific data!"); 2255 } 2256 } 2257 } 2258 if (mFlags & kFlagIsAsync) { 2259 onOutputFormatChanged(); 2260 } else { 2261 mFlags |= kFlagOutputFormatChanged; 2262 postActivityNotificationIfPossible(); 2263 } 2264 2265 // Notify mCrypto of video resolution changes 2266 if (mCrypto != NULL) { 2267 int32_t left, top, right, bottom, width, height; 2268 if (mOutputFormat->findRect("crop", &left, &top, &right, &bottom)) { 2269 mCrypto->notifyResolution(right - left + 1, bottom - top + 1); 2270 } else if (mOutputFormat->findInt32("width", &width) 2271 && mOutputFormat->findInt32("height", &height)) { 2272 mCrypto->notifyResolution(width, height); 2273 } 2274 } 2275 } 2276 2277 if (mFlags & kFlagIsAsync) { 2278 onOutputBufferAvailable(); 2279 } else if (mFlags & kFlagDequeueOutputPending) { 2280 CHECK(handleDequeueOutputBuffer(mDequeueOutputReplyID)); 2281 2282 ++mDequeueOutputTimeoutGeneration; 2283 mFlags &= ~kFlagDequeueOutputPending; 2284 mDequeueOutputReplyID = 0; 2285 } else { 2286 postActivityNotificationIfPossible(); 2287 } 2288 2289 break; 2290 } 2291 2292 case kWhatEOS: 2293 { 2294 // We already notify the client of this by using the 2295 // corresponding flag in "onOutputBufferReady". 2296 break; 2297 } 2298 2299 case kWhatStopCompleted: 2300 { 2301 if (mState != STOPPING) { 2302 ALOGW("Received kWhatStopCompleted in state %d", mState); 2303 break; 2304 } 2305 setState(INITIALIZED); 2306 (new AMessage)->postReply(mReplyID); 2307 break; 2308 } 2309 2310 case kWhatReleaseCompleted: 2311 { 2312 if (mState != RELEASING) { 2313 ALOGW("Received kWhatReleaseCompleted in state %d", mState); 2314 break; 2315 } 2316 setState(UNINITIALIZED); 2317 mComponentName.clear(); 2318 2319 mFlags &= ~kFlagIsComponentAllocated; 2320 2321 mResourceManagerService->removeResource(getId(mResourceManagerClient)); 2322 2323 (new AMessage)->postReply(mReplyID); 2324 break; 2325 } 2326 2327 case kWhatFlushCompleted: 2328 { 2329 if (mState != FLUSHING) { 2330 ALOGW("received FlushCompleted message in state %d", 2331 mState); 2332 break; 2333 } 2334 2335 if (mFlags & kFlagIsAsync) { 2336 setState(FLUSHED); 2337 } else { 2338 setState(STARTED); 2339 mCodec->signalResume(); 2340 } 2341 2342 (new AMessage)->postReply(mReplyID); 2343 break; 2344 } 2345 2346 default: 2347 TRESPASS(); 2348 } 2349 break; 2350 } 2351 2352 case kWhatInit: 2353 { 2354 sp<AReplyToken> replyID; 2355 CHECK(msg->senderAwaitsResponse(&replyID)); 2356 2357 if (mState != UNINITIALIZED) { 2358 PostReplyWithError(replyID, INVALID_OPERATION); 2359 break; 2360 } 2361 2362 mReplyID = replyID; 2363 setState(INITIALIZING); 2364 2365 sp<RefBase> codecInfo; 2366 CHECK(msg->findObject("codecInfo", &codecInfo)); 2367 AString name; 2368 CHECK(msg->findString("name", &name)); 2369 2370 sp<AMessage> format = new AMessage; 2371 format->setObject("codecInfo", codecInfo); 2372 format->setString("componentName", name); 2373 2374 mCodec->initiateAllocateComponent(format); 2375 break; 2376 } 2377 2378 case kWhatSetNotification: 2379 { 2380 sp<AMessage> notify; 2381 if (msg->findMessage("on-frame-rendered", ¬ify)) { 2382 mOnFrameRenderedNotification = notify; 2383 } 2384 break; 2385 } 2386 2387 case kWhatSetCallback: 2388 { 2389 sp<AReplyToken> replyID; 2390 CHECK(msg->senderAwaitsResponse(&replyID)); 2391 2392 if (mState == UNINITIALIZED 2393 || mState == INITIALIZING 2394 || isExecuting()) { 2395 // callback can't be set after codec is executing, 2396 // or before it's initialized (as the callback 2397 // will be cleared when it goes to INITIALIZED) 2398 PostReplyWithError(replyID, INVALID_OPERATION); 2399 break; 2400 } 2401 2402 sp<AMessage> callback; 2403 CHECK(msg->findMessage("callback", &callback)); 2404 2405 mCallback = callback; 2406 2407 if (mCallback != NULL) { 2408 ALOGI("MediaCodec will operate in async mode"); 2409 mFlags |= kFlagIsAsync; 2410 } else { 2411 mFlags &= ~kFlagIsAsync; 2412 } 2413 2414 sp<AMessage> response = new AMessage; 2415 response->postReply(replyID); 2416 break; 2417 } 2418 2419 case kWhatConfigure: 2420 { 2421 sp<AReplyToken> replyID; 2422 CHECK(msg->senderAwaitsResponse(&replyID)); 2423 2424 if (mState != INITIALIZED) { 2425 PostReplyWithError(replyID, INVALID_OPERATION); 2426 break; 2427 } 2428 2429 sp<RefBase> obj; 2430 CHECK(msg->findObject("surface", &obj)); 2431 2432 sp<AMessage> format; 2433 CHECK(msg->findMessage("format", &format)); 2434 2435 int32_t push; 2436 if (msg->findInt32("push-blank-buffers-on-shutdown", &push) && push != 0) { 2437 mFlags |= kFlagPushBlankBuffersOnShutdown; 2438 } 2439 2440 if (obj != NULL) { 2441 if (!format->findInt32("allow-frame-drop", &mAllowFrameDroppingBySurface)) { 2442 // allow frame dropping by surface by default 2443 mAllowFrameDroppingBySurface = true; 2444 } 2445 2446 format->setObject("native-window", obj); 2447 status_t err = handleSetSurface(static_cast<Surface *>(obj.get())); 2448 if (err != OK) { 2449 PostReplyWithError(replyID, err); 2450 break; 2451 } 2452 } else { 2453 // we are not using surface so this variable is not used, but initialize sensibly anyway 2454 mAllowFrameDroppingBySurface = false; 2455 2456 handleSetSurface(NULL); 2457 } 2458 2459 mReplyID = replyID; 2460 setState(CONFIGURING); 2461 2462 void *crypto; 2463 if (!msg->findPointer("crypto", &crypto)) { 2464 crypto = NULL; 2465 } 2466 2467 ALOGV("kWhatConfigure: Old mCrypto: %p (%d)", 2468 mCrypto.get(), (mCrypto != NULL ? mCrypto->getStrongCount() : 0)); 2469 2470 mCrypto = static_cast<ICrypto *>(crypto); 2471 mBufferChannel->setCrypto(mCrypto); 2472 2473 ALOGV("kWhatConfigure: New mCrypto: %p (%d)", 2474 mCrypto.get(), (mCrypto != NULL ? mCrypto->getStrongCount() : 0)); 2475 2476 void *descrambler; 2477 if (!msg->findPointer("descrambler", &descrambler)) { 2478 descrambler = NULL; 2479 } 2480 2481 mDescrambler = static_cast<IDescrambler *>(descrambler); 2482 mBufferChannel->setDescrambler(mDescrambler); 2483 2484 uint32_t flags; 2485 CHECK(msg->findInt32("flags", (int32_t *)&flags)); 2486 2487 if (flags & CONFIGURE_FLAG_ENCODE) { 2488 format->setInt32("encoder", true); 2489 mFlags |= kFlagIsEncoder; 2490 } 2491 2492 extractCSD(format); 2493 2494 mCodec->initiateConfigureComponent(format); 2495 break; 2496 } 2497 2498 case kWhatSetSurface: 2499 { 2500 sp<AReplyToken> replyID; 2501 CHECK(msg->senderAwaitsResponse(&replyID)); 2502 2503 status_t err = OK; 2504 2505 switch (mState) { 2506 case CONFIGURED: 2507 case STARTED: 2508 case FLUSHED: 2509 { 2510 sp<RefBase> obj; 2511 (void)msg->findObject("surface", &obj); 2512 sp<Surface> surface = static_cast<Surface *>(obj.get()); 2513 if (mSurface == NULL) { 2514 // do not support setting surface if it was not set 2515 err = INVALID_OPERATION; 2516 } else if (obj == NULL) { 2517 // do not support unsetting surface 2518 err = BAD_VALUE; 2519 } else { 2520 err = connectToSurface(surface); 2521 if (err == ALREADY_EXISTS) { 2522 // reconnecting to same surface 2523 err = OK; 2524 } else { 2525 if (err == OK) { 2526 if (mFlags & kFlagUsesSoftwareRenderer) { 2527 if (mSoftRenderer != NULL 2528 && (mFlags & kFlagPushBlankBuffersOnShutdown)) { 2529 pushBlankBuffersToNativeWindow(mSurface.get()); 2530 } 2531 surface->setDequeueTimeout(-1); 2532 mSoftRenderer = new SoftwareRenderer(surface); 2533 // TODO: check if this was successful 2534 } else { 2535 err = mCodec->setSurface(surface); 2536 } 2537 } 2538 if (err == OK) { 2539 (void)disconnectFromSurface(); 2540 mSurface = surface; 2541 } 2542 } 2543 } 2544 break; 2545 } 2546 2547 default: 2548 err = INVALID_OPERATION; 2549 break; 2550 } 2551 2552 PostReplyWithError(replyID, err); 2553 break; 2554 } 2555 2556 case kWhatCreateInputSurface: 2557 case kWhatSetInputSurface: 2558 { 2559 sp<AReplyToken> replyID; 2560 CHECK(msg->senderAwaitsResponse(&replyID)); 2561 2562 // Must be configured, but can't have been started yet. 2563 if (mState != CONFIGURED) { 2564 PostReplyWithError(replyID, INVALID_OPERATION); 2565 break; 2566 } 2567 2568 mReplyID = replyID; 2569 if (msg->what() == kWhatCreateInputSurface) { 2570 mCodec->initiateCreateInputSurface(); 2571 } else { 2572 sp<RefBase> obj; 2573 CHECK(msg->findObject("input-surface", &obj)); 2574 2575 mCodec->initiateSetInputSurface( 2576 static_cast<PersistentSurface *>(obj.get())); 2577 } 2578 break; 2579 } 2580 case kWhatStart: 2581 { 2582 sp<AReplyToken> replyID; 2583 CHECK(msg->senderAwaitsResponse(&replyID)); 2584 2585 if (mState == FLUSHED) { 2586 setState(STARTED); 2587 if (mHavePendingInputBuffers) { 2588 onInputBufferAvailable(); 2589 mHavePendingInputBuffers = false; 2590 } 2591 mCodec->signalResume(); 2592 PostReplyWithError(replyID, OK); 2593 break; 2594 } else if (mState != CONFIGURED) { 2595 PostReplyWithError(replyID, INVALID_OPERATION); 2596 break; 2597 } 2598 2599 mReplyID = replyID; 2600 setState(STARTING); 2601 2602 mCodec->initiateStart(); 2603 break; 2604 } 2605 2606 case kWhatStop: 2607 case kWhatRelease: 2608 { 2609 State targetState = 2610 (msg->what() == kWhatStop) ? INITIALIZED : UNINITIALIZED; 2611 2612 sp<AReplyToken> replyID; 2613 CHECK(msg->senderAwaitsResponse(&replyID)); 2614 2615 // already stopped/released 2616 if (mState == UNINITIALIZED && mReleasedByResourceManager) { 2617 sp<AMessage> response = new AMessage; 2618 response->setInt32("err", OK); 2619 response->postReply(replyID); 2620 break; 2621 } 2622 2623 int32_t reclaimed = 0; 2624 msg->findInt32("reclaimed", &reclaimed); 2625 if (reclaimed) { 2626 mReleasedByResourceManager = true; 2627 2628 int32_t force = 0; 2629 msg->findInt32("force", &force); 2630 if (!force && hasPendingBuffer()) { 2631 ALOGW("Can't reclaim codec right now due to pending buffers."); 2632 2633 // return WOULD_BLOCK to ask resource manager to retry later. 2634 sp<AMessage> response = new AMessage; 2635 response->setInt32("err", WOULD_BLOCK); 2636 response->postReply(replyID); 2637 2638 // notify the async client 2639 if (mFlags & kFlagIsAsync) { 2640 onError(DEAD_OBJECT, ACTION_CODE_FATAL); 2641 } 2642 break; 2643 } 2644 } 2645 2646 bool isReleasingAllocatedComponent = 2647 (mFlags & kFlagIsComponentAllocated) && targetState == UNINITIALIZED; 2648 if (!isReleasingAllocatedComponent // See 1 2649 && mState != INITIALIZED 2650 && mState != CONFIGURED && !isExecuting()) { 2651 // 1) Permit release to shut down the component if allocated. 2652 // 2653 // 2) We may be in "UNINITIALIZED" state already and 2654 // also shutdown the encoder/decoder without the 2655 // client being aware of this if media server died while 2656 // we were being stopped. The client would assume that 2657 // after stop() returned, it would be safe to call release() 2658 // and it should be in this case, no harm to allow a release() 2659 // if we're already uninitialized. 2660 sp<AMessage> response = new AMessage; 2661 // TODO: we shouldn't throw an exception for stop/release. Change this to wait until 2662 // the previous stop/release completes and then reply with OK. 2663 status_t err = mState == targetState ? OK : INVALID_OPERATION; 2664 response->setInt32("err", err); 2665 if (err == OK && targetState == UNINITIALIZED) { 2666 mComponentName.clear(); 2667 } 2668 response->postReply(replyID); 2669 break; 2670 } 2671 2672 // If we're flushing, stopping, configuring or starting but 2673 // received a release request, post the reply for the pending call 2674 // first, and consider it done. The reply token will be replaced 2675 // after this, and we'll no longer be able to reply. 2676 if (mState == FLUSHING || mState == STOPPING 2677 || mState == CONFIGURING || mState == STARTING) { 2678 (new AMessage)->postReply(mReplyID); 2679 } 2680 2681 if (mFlags & kFlagSawMediaServerDie) { 2682 // It's dead, Jim. Don't expect initiateShutdown to yield 2683 // any useful results now... 2684 setState(UNINITIALIZED); 2685 if (targetState == UNINITIALIZED) { 2686 mComponentName.clear(); 2687 } 2688 (new AMessage)->postReply(replyID); 2689 break; 2690 } 2691 2692 // If we already have an error, component may not be able to 2693 // complete the shutdown properly. If we're stopping, post the 2694 // reply now with an error to unblock the client, client can 2695 // release after the failure (instead of ANR). 2696 if (msg->what() == kWhatStop && (mFlags & kFlagStickyError)) { 2697 PostReplyWithError(replyID, getStickyError()); 2698 break; 2699 } 2700 2701 mReplyID = replyID; 2702 setState(msg->what() == kWhatStop ? STOPPING : RELEASING); 2703 2704 mCodec->initiateShutdown( 2705 msg->what() == kWhatStop /* keepComponentAllocated */); 2706 2707 returnBuffersToCodec(reclaimed); 2708 2709 if (mSoftRenderer != NULL && (mFlags & kFlagPushBlankBuffersOnShutdown)) { 2710 pushBlankBuffersToNativeWindow(mSurface.get()); 2711 } 2712 2713 break; 2714 } 2715 2716 case kWhatDequeueInputBuffer: 2717 { 2718 sp<AReplyToken> replyID; 2719 CHECK(msg->senderAwaitsResponse(&replyID)); 2720 2721 if (mFlags & kFlagIsAsync) { 2722 ALOGE("dequeueInputBuffer can't be used in async mode"); 2723 PostReplyWithError(replyID, INVALID_OPERATION); 2724 break; 2725 } 2726 2727 if (mHaveInputSurface) { 2728 ALOGE("dequeueInputBuffer can't be used with input surface"); 2729 PostReplyWithError(replyID, INVALID_OPERATION); 2730 break; 2731 } 2732 2733 if (handleDequeueInputBuffer(replyID, true /* new request */)) { 2734 break; 2735 } 2736 2737 int64_t timeoutUs; 2738 CHECK(msg->findInt64("timeoutUs", &timeoutUs)); 2739 2740 if (timeoutUs == 0LL) { 2741 PostReplyWithError(replyID, -EAGAIN); 2742 break; 2743 } 2744 2745 mFlags |= kFlagDequeueInputPending; 2746 mDequeueInputReplyID = replyID; 2747 2748 if (timeoutUs > 0LL) { 2749 sp<AMessage> timeoutMsg = 2750 new AMessage(kWhatDequeueInputTimedOut, this); 2751 timeoutMsg->setInt32( 2752 "generation", ++mDequeueInputTimeoutGeneration); 2753 timeoutMsg->post(timeoutUs); 2754 } 2755 break; 2756 } 2757 2758 case kWhatDequeueInputTimedOut: 2759 { 2760 int32_t generation; 2761 CHECK(msg->findInt32("generation", &generation)); 2762 2763 if (generation != mDequeueInputTimeoutGeneration) { 2764 // Obsolete 2765 break; 2766 } 2767 2768 CHECK(mFlags & kFlagDequeueInputPending); 2769 2770 PostReplyWithError(mDequeueInputReplyID, -EAGAIN); 2771 2772 mFlags &= ~kFlagDequeueInputPending; 2773 mDequeueInputReplyID = 0; 2774 break; 2775 } 2776 2777 case kWhatQueueInputBuffer: 2778 { 2779 sp<AReplyToken> replyID; 2780 CHECK(msg->senderAwaitsResponse(&replyID)); 2781 2782 if (!isExecuting()) { 2783 PostReplyWithError(replyID, INVALID_OPERATION); 2784 break; 2785 } else if (mFlags & kFlagStickyError) { 2786 PostReplyWithError(replyID, getStickyError()); 2787 break; 2788 } 2789 2790 status_t err = onQueueInputBuffer(msg); 2791 2792 PostReplyWithError(replyID, err); 2793 break; 2794 } 2795 2796 case kWhatDequeueOutputBuffer: 2797 { 2798 sp<AReplyToken> replyID; 2799 CHECK(msg->senderAwaitsResponse(&replyID)); 2800 2801 if (mFlags & kFlagIsAsync) { 2802 ALOGE("dequeueOutputBuffer can't be used in async mode"); 2803 PostReplyWithError(replyID, INVALID_OPERATION); 2804 break; 2805 } 2806 2807 if (handleDequeueOutputBuffer(replyID, true /* new request */)) { 2808 break; 2809 } 2810 2811 int64_t timeoutUs; 2812 CHECK(msg->findInt64("timeoutUs", &timeoutUs)); 2813 2814 if (timeoutUs == 0LL) { 2815 PostReplyWithError(replyID, -EAGAIN); 2816 break; 2817 } 2818 2819 mFlags |= kFlagDequeueOutputPending; 2820 mDequeueOutputReplyID = replyID; 2821 2822 if (timeoutUs > 0LL) { 2823 sp<AMessage> timeoutMsg = 2824 new AMessage(kWhatDequeueOutputTimedOut, this); 2825 timeoutMsg->setInt32( 2826 "generation", ++mDequeueOutputTimeoutGeneration); 2827 timeoutMsg->post(timeoutUs); 2828 } 2829 break; 2830 } 2831 2832 case kWhatDequeueOutputTimedOut: 2833 { 2834 int32_t generation; 2835 CHECK(msg->findInt32("generation", &generation)); 2836 2837 if (generation != mDequeueOutputTimeoutGeneration) { 2838 // Obsolete 2839 break; 2840 } 2841 2842 CHECK(mFlags & kFlagDequeueOutputPending); 2843 2844 PostReplyWithError(mDequeueOutputReplyID, -EAGAIN); 2845 2846 mFlags &= ~kFlagDequeueOutputPending; 2847 mDequeueOutputReplyID = 0; 2848 break; 2849 } 2850 2851 case kWhatReleaseOutputBuffer: 2852 { 2853 sp<AReplyToken> replyID; 2854 CHECK(msg->senderAwaitsResponse(&replyID)); 2855 2856 if (!isExecuting()) { 2857 PostReplyWithError(replyID, INVALID_OPERATION); 2858 break; 2859 } else if (mFlags & kFlagStickyError) { 2860 PostReplyWithError(replyID, getStickyError()); 2861 break; 2862 } 2863 2864 status_t err = onReleaseOutputBuffer(msg); 2865 2866 PostReplyWithError(replyID, err); 2867 break; 2868 } 2869 2870 case kWhatSignalEndOfInputStream: 2871 { 2872 sp<AReplyToken> replyID; 2873 CHECK(msg->senderAwaitsResponse(&replyID)); 2874 2875 if (!isExecuting() || !mHaveInputSurface) { 2876 PostReplyWithError(replyID, INVALID_OPERATION); 2877 break; 2878 } else if (mFlags & kFlagStickyError) { 2879 PostReplyWithError(replyID, getStickyError()); 2880 break; 2881 } 2882 2883 mReplyID = replyID; 2884 mCodec->signalEndOfInputStream(); 2885 break; 2886 } 2887 2888 case kWhatGetBuffers: 2889 { 2890 sp<AReplyToken> replyID; 2891 CHECK(msg->senderAwaitsResponse(&replyID)); 2892 if (!isExecuting() || (mFlags & kFlagIsAsync)) { 2893 PostReplyWithError(replyID, INVALID_OPERATION); 2894 break; 2895 } else if (mFlags & kFlagStickyError) { 2896 PostReplyWithError(replyID, getStickyError()); 2897 break; 2898 } 2899 2900 int32_t portIndex; 2901 CHECK(msg->findInt32("portIndex", &portIndex)); 2902 2903 Vector<sp<MediaCodecBuffer> > *dstBuffers; 2904 CHECK(msg->findPointer("buffers", (void **)&dstBuffers)); 2905 2906 dstBuffers->clear(); 2907 // If we're using input surface (either non-persistent created by 2908 // createInputSurface(), or persistent set by setInputSurface()), 2909 // give the client an empty input buffers array. 2910 if (portIndex != kPortIndexInput || !mHaveInputSurface) { 2911 if (portIndex == kPortIndexInput) { 2912 mBufferChannel->getInputBufferArray(dstBuffers); 2913 } else { 2914 mBufferChannel->getOutputBufferArray(dstBuffers); 2915 } 2916 } 2917 2918 (new AMessage)->postReply(replyID); 2919 break; 2920 } 2921 2922 case kWhatFlush: 2923 { 2924 sp<AReplyToken> replyID; 2925 CHECK(msg->senderAwaitsResponse(&replyID)); 2926 2927 if (!isExecuting()) { 2928 PostReplyWithError(replyID, INVALID_OPERATION); 2929 break; 2930 } else if (mFlags & kFlagStickyError) { 2931 PostReplyWithError(replyID, getStickyError()); 2932 break; 2933 } 2934 2935 mReplyID = replyID; 2936 // TODO: skip flushing if already FLUSHED 2937 setState(FLUSHING); 2938 2939 mCodec->signalFlush(); 2940 returnBuffersToCodec(); 2941 break; 2942 } 2943 2944 case kWhatGetInputFormat: 2945 case kWhatGetOutputFormat: 2946 { 2947 sp<AMessage> format = 2948 (msg->what() == kWhatGetOutputFormat ? mOutputFormat : mInputFormat); 2949 2950 sp<AReplyToken> replyID; 2951 CHECK(msg->senderAwaitsResponse(&replyID)); 2952 2953 if ((mState != CONFIGURED && mState != STARTING && 2954 mState != STARTED && mState != FLUSHING && 2955 mState != FLUSHED) 2956 || format == NULL) { 2957 PostReplyWithError(replyID, INVALID_OPERATION); 2958 break; 2959 } else if (mFlags & kFlagStickyError) { 2960 PostReplyWithError(replyID, getStickyError()); 2961 break; 2962 } 2963 2964 sp<AMessage> response = new AMessage; 2965 response->setMessage("format", format); 2966 response->postReply(replyID); 2967 break; 2968 } 2969 2970 case kWhatRequestIDRFrame: 2971 { 2972 mCodec->signalRequestIDRFrame(); 2973 break; 2974 } 2975 2976 case kWhatRequestActivityNotification: 2977 { 2978 CHECK(mActivityNotify == NULL); 2979 CHECK(msg->findMessage("notify", &mActivityNotify)); 2980 2981 postActivityNotificationIfPossible(); 2982 break; 2983 } 2984 2985 case kWhatGetName: 2986 { 2987 sp<AReplyToken> replyID; 2988 CHECK(msg->senderAwaitsResponse(&replyID)); 2989 2990 if (mComponentName.empty()) { 2991 PostReplyWithError(replyID, INVALID_OPERATION); 2992 break; 2993 } 2994 2995 sp<AMessage> response = new AMessage; 2996 response->setString("name", mComponentName.c_str()); 2997 response->postReply(replyID); 2998 break; 2999 } 3000 3001 case kWhatGetCodecInfo: 3002 { 3003 sp<AReplyToken> replyID; 3004 CHECK(msg->senderAwaitsResponse(&replyID)); 3005 3006 sp<AMessage> response = new AMessage; 3007 response->setObject("codecInfo", mCodecInfo); 3008 response->postReply(replyID); 3009 break; 3010 } 3011 3012 case kWhatSetParameters: 3013 { 3014 sp<AReplyToken> replyID; 3015 CHECK(msg->senderAwaitsResponse(&replyID)); 3016 3017 sp<AMessage> params; 3018 CHECK(msg->findMessage("params", ¶ms)); 3019 3020 status_t err = onSetParameters(params); 3021 3022 PostReplyWithError(replyID, err); 3023 break; 3024 } 3025 3026 case kWhatDrmReleaseCrypto: 3027 { 3028 onReleaseCrypto(msg); 3029 break; 3030 } 3031 3032 default: 3033 TRESPASS(); 3034 } 3035 } 3036 3037 void MediaCodec::extractCSD(const sp<AMessage> &format) { 3038 mCSD.clear(); 3039 3040 size_t i = 0; 3041 for (;;) { 3042 sp<ABuffer> csd; 3043 if (!format->findBuffer(AStringPrintf("csd-%u", i).c_str(), &csd)) { 3044 break; 3045 } 3046 if (csd->size() == 0) { 3047 ALOGW("csd-%zu size is 0", i); 3048 } 3049 3050 mCSD.push_back(csd); 3051 ++i; 3052 } 3053 3054 ALOGV("Found %zu pieces of codec specific data.", mCSD.size()); 3055 } 3056 3057 status_t MediaCodec::queueCSDInputBuffer(size_t bufferIndex) { 3058 CHECK(!mCSD.empty()); 3059 3060 const BufferInfo &info = mPortBuffers[kPortIndexInput][bufferIndex]; 3061 3062 sp<ABuffer> csd = *mCSD.begin(); 3063 mCSD.erase(mCSD.begin()); 3064 3065 const sp<MediaCodecBuffer> &codecInputData = info.mData; 3066 3067 if (csd->size() > codecInputData->capacity()) { 3068 return -EINVAL; 3069 } 3070 if (codecInputData->data() == NULL) { 3071 ALOGV("Input buffer %zu is not properly allocated", bufferIndex); 3072 return -EINVAL; 3073 } 3074 3075 memcpy(codecInputData->data(), csd->data(), csd->size()); 3076 3077 AString errorDetailMsg; 3078 3079 sp<AMessage> msg = new AMessage(kWhatQueueInputBuffer, this); 3080 msg->setSize("index", bufferIndex); 3081 msg->setSize("offset", 0); 3082 msg->setSize("size", csd->size()); 3083 msg->setInt64("timeUs", 0LL); 3084 msg->setInt32("flags", BUFFER_FLAG_CODECCONFIG); 3085 msg->setPointer("errorDetailMsg", &errorDetailMsg); 3086 3087 return onQueueInputBuffer(msg); 3088 } 3089 3090 void MediaCodec::setState(State newState) { 3091 if (newState == INITIALIZED || newState == UNINITIALIZED) { 3092 delete mSoftRenderer; 3093 mSoftRenderer = NULL; 3094 3095 if ( mCrypto != NULL ) { 3096 ALOGV("setState: ~mCrypto: %p (%d)", 3097 mCrypto.get(), (mCrypto != NULL ? mCrypto->getStrongCount() : 0)); 3098 } 3099 mCrypto.clear(); 3100 mDescrambler.clear(); 3101 handleSetSurface(NULL); 3102 3103 mInputFormat.clear(); 3104 mOutputFormat.clear(); 3105 mFlags &= ~kFlagOutputFormatChanged; 3106 mFlags &= ~kFlagOutputBuffersChanged; 3107 mFlags &= ~kFlagStickyError; 3108 mFlags &= ~kFlagIsEncoder; 3109 mFlags &= ~kFlagIsAsync; 3110 mStickyError = OK; 3111 3112 mActivityNotify.clear(); 3113 mCallback.clear(); 3114 } 3115 3116 if (newState == UNINITIALIZED) { 3117 // return any straggling buffers, e.g. if we got here on an error 3118 returnBuffersToCodec(); 3119 3120 // The component is gone, mediaserver's probably back up already 3121 // but should definitely be back up should we try to instantiate 3122 // another component.. and the cycle continues. 3123 mFlags &= ~kFlagSawMediaServerDie; 3124 } 3125 3126 mState = newState; 3127 3128 cancelPendingDequeueOperations(); 3129 3130 updateBatteryStat(); 3131 } 3132 3133 void MediaCodec::returnBuffersToCodec(bool isReclaim) { 3134 returnBuffersToCodecOnPort(kPortIndexInput, isReclaim); 3135 returnBuffersToCodecOnPort(kPortIndexOutput, isReclaim); 3136 } 3137 3138 void MediaCodec::returnBuffersToCodecOnPort(int32_t portIndex, bool isReclaim) { 3139 CHECK(portIndex == kPortIndexInput || portIndex == kPortIndexOutput); 3140 Mutex::Autolock al(mBufferLock); 3141 3142 for (size_t i = 0; i < mPortBuffers[portIndex].size(); ++i) { 3143 BufferInfo *info = &mPortBuffers[portIndex][i]; 3144 3145 if (info->mData != nullptr) { 3146 sp<MediaCodecBuffer> buffer = info->mData; 3147 if (isReclaim && info->mOwnedByClient) { 3148 ALOGD("port %d buffer %zu still owned by client when codec is reclaimed", 3149 portIndex, i); 3150 } else { 3151 info->mOwnedByClient = false; 3152 info->mData.clear(); 3153 } 3154 mBufferChannel->discardBuffer(buffer); 3155 } 3156 } 3157 3158 mAvailPortBuffers[portIndex].clear(); 3159 } 3160 3161 size_t MediaCodec::updateBuffers( 3162 int32_t portIndex, const sp<AMessage> &msg) { 3163 CHECK(portIndex == kPortIndexInput || portIndex == kPortIndexOutput); 3164 size_t index; 3165 CHECK(msg->findSize("index", &index)); 3166 sp<RefBase> obj; 3167 CHECK(msg->findObject("buffer", &obj)); 3168 sp<MediaCodecBuffer> buffer = static_cast<MediaCodecBuffer *>(obj.get()); 3169 3170 { 3171 Mutex::Autolock al(mBufferLock); 3172 if (mPortBuffers[portIndex].size() <= index) { 3173 mPortBuffers[portIndex].resize(align(index + 1, kNumBuffersAlign)); 3174 } 3175 mPortBuffers[portIndex][index].mData = buffer; 3176 } 3177 mAvailPortBuffers[portIndex].push_back(index); 3178 3179 return index; 3180 } 3181 3182 status_t MediaCodec::onQueueInputBuffer(const sp<AMessage> &msg) { 3183 size_t index; 3184 size_t offset; 3185 size_t size; 3186 int64_t timeUs; 3187 uint32_t flags; 3188 CHECK(msg->findSize("index", &index)); 3189 CHECK(msg->findSize("offset", &offset)); 3190 CHECK(msg->findInt64("timeUs", &timeUs)); 3191 CHECK(msg->findInt32("flags", (int32_t *)&flags)); 3192 3193 const CryptoPlugin::SubSample *subSamples; 3194 size_t numSubSamples; 3195 const uint8_t *key; 3196 const uint8_t *iv; 3197 CryptoPlugin::Mode mode = CryptoPlugin::kMode_Unencrypted; 3198 3199 // We allow the simpler queueInputBuffer API to be used even in 3200 // secure mode, by fabricating a single unencrypted subSample. 3201 CryptoPlugin::SubSample ss; 3202 CryptoPlugin::Pattern pattern; 3203 3204 if (msg->findSize("size", &size)) { 3205 if (hasCryptoOrDescrambler()) { 3206 ss.mNumBytesOfClearData = size; 3207 ss.mNumBytesOfEncryptedData = 0; 3208 3209 subSamples = &ss; 3210 numSubSamples = 1; 3211 key = NULL; 3212 iv = NULL; 3213 pattern.mEncryptBlocks = 0; 3214 pattern.mSkipBlocks = 0; 3215 } 3216 } else { 3217 if (!hasCryptoOrDescrambler()) { 3218 ALOGE("[%s] queuing secure buffer without mCrypto or mDescrambler!", 3219 mComponentName.c_str()); 3220 return -EINVAL; 3221 } 3222 3223 CHECK(msg->findPointer("subSamples", (void **)&subSamples)); 3224 CHECK(msg->findSize("numSubSamples", &numSubSamples)); 3225 CHECK(msg->findPointer("key", (void **)&key)); 3226 CHECK(msg->findPointer("iv", (void **)&iv)); 3227 CHECK(msg->findInt32("encryptBlocks", (int32_t *)&pattern.mEncryptBlocks)); 3228 CHECK(msg->findInt32("skipBlocks", (int32_t *)&pattern.mSkipBlocks)); 3229 3230 int32_t tmp; 3231 CHECK(msg->findInt32("mode", &tmp)); 3232 3233 mode = (CryptoPlugin::Mode)tmp; 3234 3235 size = 0; 3236 for (size_t i = 0; i < numSubSamples; ++i) { 3237 size += subSamples[i].mNumBytesOfClearData; 3238 size += subSamples[i].mNumBytesOfEncryptedData; 3239 } 3240 } 3241 3242 if (index >= mPortBuffers[kPortIndexInput].size()) { 3243 return -ERANGE; 3244 } 3245 3246 BufferInfo *info = &mPortBuffers[kPortIndexInput][index]; 3247 3248 if (info->mData == nullptr || !info->mOwnedByClient) { 3249 return -EACCES; 3250 } 3251 3252 if (offset + size > info->mData->capacity()) { 3253 return -EINVAL; 3254 } 3255 3256 info->mData->setRange(offset, size); 3257 info->mData->meta()->setInt64("timeUs", timeUs); 3258 if (flags & BUFFER_FLAG_EOS) { 3259 info->mData->meta()->setInt32("eos", true); 3260 } 3261 3262 if (flags & BUFFER_FLAG_CODECCONFIG) { 3263 info->mData->meta()->setInt32("csd", true); 3264 } 3265 3266 sp<MediaCodecBuffer> buffer = info->mData; 3267 status_t err = OK; 3268 if (hasCryptoOrDescrambler()) { 3269 AString *errorDetailMsg; 3270 CHECK(msg->findPointer("errorDetailMsg", (void **)&errorDetailMsg)); 3271 3272 err = mBufferChannel->queueSecureInputBuffer( 3273 buffer, 3274 (mFlags & kFlagIsSecure), 3275 key, 3276 iv, 3277 mode, 3278 pattern, 3279 subSamples, 3280 numSubSamples, 3281 errorDetailMsg); 3282 } else { 3283 err = mBufferChannel->queueInputBuffer(buffer); 3284 } 3285 3286 if (err == OK) { 3287 // synchronization boundary for getBufferAndFormat 3288 Mutex::Autolock al(mBufferLock); 3289 info->mOwnedByClient = false; 3290 info->mData.clear(); 3291 3292 statsBufferSent(timeUs); 3293 } 3294 3295 return err; 3296 } 3297 3298 //static 3299 size_t MediaCodec::CreateFramesRenderedMessage( 3300 const std::list<FrameRenderTracker::Info> &done, sp<AMessage> &msg) { 3301 size_t index = 0; 3302 3303 for (std::list<FrameRenderTracker::Info>::const_iterator it = done.cbegin(); 3304 it != done.cend(); ++it) { 3305 if (it->getRenderTimeNs() < 0) { 3306 continue; // dropped frame from tracking 3307 } 3308 msg->setInt64(AStringPrintf("%zu-media-time-us", index).c_str(), it->getMediaTimeUs()); 3309 msg->setInt64(AStringPrintf("%zu-system-nano", index).c_str(), it->getRenderTimeNs()); 3310 ++index; 3311 } 3312 return index; 3313 } 3314 3315 status_t MediaCodec::onReleaseOutputBuffer(const sp<AMessage> &msg) { 3316 size_t index; 3317 CHECK(msg->findSize("index", &index)); 3318 3319 int32_t render; 3320 if (!msg->findInt32("render", &render)) { 3321 render = 0; 3322 } 3323 3324 if (!isExecuting()) { 3325 return -EINVAL; 3326 } 3327 3328 if (index >= mPortBuffers[kPortIndexOutput].size()) { 3329 return -ERANGE; 3330 } 3331 3332 BufferInfo *info = &mPortBuffers[kPortIndexOutput][index]; 3333 3334 if (info->mData == nullptr || !info->mOwnedByClient) { 3335 return -EACCES; 3336 } 3337 3338 // synchronization boundary for getBufferAndFormat 3339 sp<MediaCodecBuffer> buffer; 3340 { 3341 Mutex::Autolock al(mBufferLock); 3342 info->mOwnedByClient = false; 3343 buffer = info->mData; 3344 info->mData.clear(); 3345 } 3346 3347 if (render && buffer->size() != 0) { 3348 int64_t mediaTimeUs = -1; 3349 buffer->meta()->findInt64("timeUs", &mediaTimeUs); 3350 3351 int64_t renderTimeNs = 0; 3352 if (!msg->findInt64("timestampNs", &renderTimeNs)) { 3353 // use media timestamp if client did not request a specific render timestamp 3354 ALOGV("using buffer PTS of %lld", (long long)mediaTimeUs); 3355 renderTimeNs = mediaTimeUs * 1000; 3356 } 3357 3358 if (mSoftRenderer != NULL) { 3359 std::list<FrameRenderTracker::Info> doneFrames = mSoftRenderer->render( 3360 buffer->data(), buffer->size(), mediaTimeUs, renderTimeNs, 3361 mPortBuffers[kPortIndexOutput].size(), buffer->format()); 3362 3363 // if we are running, notify rendered frames 3364 if (!doneFrames.empty() && mState == STARTED && mOnFrameRenderedNotification != NULL) { 3365 sp<AMessage> notify = mOnFrameRenderedNotification->dup(); 3366 sp<AMessage> data = new AMessage; 3367 if (CreateFramesRenderedMessage(doneFrames, data)) { 3368 notify->setMessage("data", data); 3369 notify->post(); 3370 } 3371 } 3372 } 3373 mBufferChannel->renderOutputBuffer(buffer, renderTimeNs); 3374 } else { 3375 mBufferChannel->discardBuffer(buffer); 3376 } 3377 3378 return OK; 3379 } 3380 3381 ssize_t MediaCodec::dequeuePortBuffer(int32_t portIndex) { 3382 CHECK(portIndex == kPortIndexInput || portIndex == kPortIndexOutput); 3383 3384 List<size_t> *availBuffers = &mAvailPortBuffers[portIndex]; 3385 3386 if (availBuffers->empty()) { 3387 return -EAGAIN; 3388 } 3389 3390 size_t index = *availBuffers->begin(); 3391 availBuffers->erase(availBuffers->begin()); 3392 3393 BufferInfo *info = &mPortBuffers[portIndex][index]; 3394 CHECK(!info->mOwnedByClient); 3395 { 3396 Mutex::Autolock al(mBufferLock); 3397 info->mOwnedByClient = true; 3398 3399 // set image-data 3400 if (info->mData->format() != NULL) { 3401 sp<ABuffer> imageData; 3402 if (info->mData->format()->findBuffer("image-data", &imageData)) { 3403 info->mData->meta()->setBuffer("image-data", imageData); 3404 } 3405 int32_t left, top, right, bottom; 3406 if (info->mData->format()->findRect("crop", &left, &top, &right, &bottom)) { 3407 info->mData->meta()->setRect("crop-rect", left, top, right, bottom); 3408 } 3409 } 3410 } 3411 3412 return index; 3413 } 3414 3415 status_t MediaCodec::connectToSurface(const sp<Surface> &surface) { 3416 status_t err = OK; 3417 if (surface != NULL) { 3418 uint64_t oldId, newId; 3419 if (mSurface != NULL 3420 && surface->getUniqueId(&newId) == NO_ERROR 3421 && mSurface->getUniqueId(&oldId) == NO_ERROR 3422 && newId == oldId) { 3423 ALOGI("[%s] connecting to the same surface. Nothing to do.", mComponentName.c_str()); 3424 return ALREADY_EXISTS; 3425 } 3426 3427 err = nativeWindowConnect(surface.get(), "connectToSurface"); 3428 if (err == OK) { 3429 // Require a fresh set of buffers after each connect by using a unique generation 3430 // number. Rely on the fact that max supported process id by Linux is 2^22. 3431 // PID is never 0 so we don't have to worry that we use the default generation of 0. 3432 // TODO: come up with a unique scheme if other producers also set the generation number. 3433 static uint32_t mSurfaceGeneration = 0; 3434 uint32_t generation = (getpid() << 10) | (++mSurfaceGeneration & ((1 << 10) - 1)); 3435 surface->setGenerationNumber(generation); 3436 ALOGI("[%s] setting surface generation to %u", mComponentName.c_str(), generation); 3437 3438 // HACK: clear any free buffers. Remove when connect will automatically do this. 3439 // This is needed as the consumer may be holding onto stale frames that it can reattach 3440 // to this surface after disconnect/connect, and those free frames would inherit the new 3441 // generation number. Disconnecting after setting a unique generation prevents this. 3442 nativeWindowDisconnect(surface.get(), "connectToSurface(reconnect)"); 3443 err = nativeWindowConnect(surface.get(), "connectToSurface(reconnect)"); 3444 } 3445 3446 if (err != OK) { 3447 ALOGE("nativeWindowConnect returned an error: %s (%d)", strerror(-err), err); 3448 } else { 3449 if (!mAllowFrameDroppingBySurface) { 3450 disableLegacyBufferDropPostQ(surface); 3451 } 3452 } 3453 } 3454 // do not return ALREADY_EXISTS unless surfaces are the same 3455 return err == ALREADY_EXISTS ? BAD_VALUE : err; 3456 } 3457 3458 status_t MediaCodec::disconnectFromSurface() { 3459 status_t err = OK; 3460 if (mSurface != NULL) { 3461 // Resetting generation is not technically needed, but there is no need to keep it either 3462 mSurface->setGenerationNumber(0); 3463 err = nativeWindowDisconnect(mSurface.get(), "disconnectFromSurface"); 3464 if (err != OK) { 3465 ALOGW("nativeWindowDisconnect returned an error: %s (%d)", strerror(-err), err); 3466 } 3467 // assume disconnected even on error 3468 mSurface.clear(); 3469 } 3470 return err; 3471 } 3472 3473 status_t MediaCodec::handleSetSurface(const sp<Surface> &surface) { 3474 status_t err = OK; 3475 if (mSurface != NULL) { 3476 (void)disconnectFromSurface(); 3477 } 3478 if (surface != NULL) { 3479 err = connectToSurface(surface); 3480 if (err == OK) { 3481 mSurface = surface; 3482 } 3483 } 3484 return err; 3485 } 3486 3487 void MediaCodec::onInputBufferAvailable() { 3488 int32_t index; 3489 while ((index = dequeuePortBuffer(kPortIndexInput)) >= 0) { 3490 sp<AMessage> msg = mCallback->dup(); 3491 msg->setInt32("callbackID", CB_INPUT_AVAILABLE); 3492 msg->setInt32("index", index); 3493 msg->post(); 3494 } 3495 } 3496 3497 void MediaCodec::onOutputBufferAvailable() { 3498 int32_t index; 3499 while ((index = dequeuePortBuffer(kPortIndexOutput)) >= 0) { 3500 const sp<MediaCodecBuffer> &buffer = 3501 mPortBuffers[kPortIndexOutput][index].mData; 3502 sp<AMessage> msg = mCallback->dup(); 3503 msg->setInt32("callbackID", CB_OUTPUT_AVAILABLE); 3504 msg->setInt32("index", index); 3505 msg->setSize("offset", buffer->offset()); 3506 msg->setSize("size", buffer->size()); 3507 3508 int64_t timeUs; 3509 CHECK(buffer->meta()->findInt64("timeUs", &timeUs)); 3510 3511 msg->setInt64("timeUs", timeUs); 3512 3513 statsBufferReceived(timeUs); 3514 3515 int32_t flags; 3516 CHECK(buffer->meta()->findInt32("flags", &flags)); 3517 3518 msg->setInt32("flags", flags); 3519 3520 msg->post(); 3521 } 3522 } 3523 3524 void MediaCodec::onError(status_t err, int32_t actionCode, const char *detail) { 3525 if (mCallback != NULL) { 3526 sp<AMessage> msg = mCallback->dup(); 3527 msg->setInt32("callbackID", CB_ERROR); 3528 msg->setInt32("err", err); 3529 msg->setInt32("actionCode", actionCode); 3530 3531 if (detail != NULL) { 3532 msg->setString("detail", detail); 3533 } 3534 3535 msg->post(); 3536 } 3537 } 3538 3539 void MediaCodec::onOutputFormatChanged() { 3540 if (mCallback != NULL) { 3541 sp<AMessage> msg = mCallback->dup(); 3542 msg->setInt32("callbackID", CB_OUTPUT_FORMAT_CHANGED); 3543 msg->setMessage("format", mOutputFormat); 3544 msg->post(); 3545 } 3546 } 3547 3548 void MediaCodec::postActivityNotificationIfPossible() { 3549 if (mActivityNotify == NULL) { 3550 return; 3551 } 3552 3553 bool isErrorOrOutputChanged = 3554 (mFlags & (kFlagStickyError 3555 | kFlagOutputBuffersChanged 3556 | kFlagOutputFormatChanged)); 3557 3558 if (isErrorOrOutputChanged 3559 || !mAvailPortBuffers[kPortIndexInput].empty() 3560 || !mAvailPortBuffers[kPortIndexOutput].empty()) { 3561 mActivityNotify->setInt32("input-buffers", 3562 mAvailPortBuffers[kPortIndexInput].size()); 3563 3564 if (isErrorOrOutputChanged) { 3565 // we want consumer to dequeue as many times as it can 3566 mActivityNotify->setInt32("output-buffers", INT32_MAX); 3567 } else { 3568 mActivityNotify->setInt32("output-buffers", 3569 mAvailPortBuffers[kPortIndexOutput].size()); 3570 } 3571 mActivityNotify->post(); 3572 mActivityNotify.clear(); 3573 } 3574 } 3575 3576 status_t MediaCodec::setParameters(const sp<AMessage> ¶ms) { 3577 sp<AMessage> msg = new AMessage(kWhatSetParameters, this); 3578 msg->setMessage("params", params); 3579 3580 sp<AMessage> response; 3581 return PostAndAwaitResponse(msg, &response); 3582 } 3583 3584 status_t MediaCodec::onSetParameters(const sp<AMessage> ¶ms) { 3585 mCodec->signalSetParameters(params); 3586 3587 return OK; 3588 } 3589 3590 status_t MediaCodec::amendOutputFormatWithCodecSpecificData( 3591 const sp<MediaCodecBuffer> &buffer) { 3592 AString mime; 3593 CHECK(mOutputFormat->findString("mime", &mime)); 3594 3595 if (!strcasecmp(mime.c_str(), MEDIA_MIMETYPE_VIDEO_AVC)) { 3596 // Codec specific data should be SPS and PPS in a single buffer, 3597 // each prefixed by a startcode (0x00 0x00 0x00 0x01). 3598 // We separate the two and put them into the output format 3599 // under the keys "csd-0" and "csd-1". 3600 3601 unsigned csdIndex = 0; 3602 3603 const uint8_t *data = buffer->data(); 3604 size_t size = buffer->size(); 3605 3606 const uint8_t *nalStart; 3607 size_t nalSize; 3608 while (getNextNALUnit(&data, &size, &nalStart, &nalSize, true) == OK) { 3609 sp<ABuffer> csd = new ABuffer(nalSize + 4); 3610 memcpy(csd->data(), "\x00\x00\x00\x01", 4); 3611 memcpy(csd->data() + 4, nalStart, nalSize); 3612 3613 mOutputFormat->setBuffer( 3614 AStringPrintf("csd-%u", csdIndex).c_str(), csd); 3615 3616 ++csdIndex; 3617 } 3618 3619 if (csdIndex != 2) { 3620 return ERROR_MALFORMED; 3621 } 3622 } else { 3623 // For everything else we just stash the codec specific data into 3624 // the output format as a single piece of csd under "csd-0". 3625 sp<ABuffer> csd = new ABuffer(buffer->size()); 3626 memcpy(csd->data(), buffer->data(), buffer->size()); 3627 csd->setRange(0, buffer->size()); 3628 mOutputFormat->setBuffer("csd-0", csd); 3629 } 3630 3631 return OK; 3632 } 3633 3634 void MediaCodec::updateBatteryStat() { 3635 if (!mIsVideo) { 3636 return; 3637 } 3638 3639 if (mState == CONFIGURED && !mBatteryStatNotified) { 3640 BatteryNotifier::getInstance().noteStartVideo(mUid); 3641 mBatteryStatNotified = true; 3642 } else if (mState == UNINITIALIZED && mBatteryStatNotified) { 3643 BatteryNotifier::getInstance().noteStopVideo(mUid); 3644 mBatteryStatNotified = false; 3645 } 3646 } 3647 3648 std::string MediaCodec::stateString(State state) { 3649 const char *rval = NULL; 3650 char rawbuffer[16]; // room for "%d" 3651 3652 switch (state) { 3653 case UNINITIALIZED: rval = "UNINITIALIZED"; break; 3654 case INITIALIZING: rval = "INITIALIZING"; break; 3655 case INITIALIZED: rval = "INITIALIZED"; break; 3656 case CONFIGURING: rval = "CONFIGURING"; break; 3657 case CONFIGURED: rval = "CONFIGURED"; break; 3658 case STARTING: rval = "STARTING"; break; 3659 case STARTED: rval = "STARTED"; break; 3660 case FLUSHING: rval = "FLUSHING"; break; 3661 case FLUSHED: rval = "FLUSHED"; break; 3662 case STOPPING: rval = "STOPPING"; break; 3663 case RELEASING: rval = "RELEASING"; break; 3664 default: 3665 snprintf(rawbuffer, sizeof(rawbuffer), "%d", state); 3666 rval = rawbuffer; 3667 break; 3668 } 3669 return rval; 3670 } 3671 3672 } // namespace android 3673
