1 /* 2 * Copyright (C) 2017 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17 //#define LOG_NDEBUG 0 18 #define LOG_TAG "HeifDecoderImpl" 19 20 #include "HeifDecoderImpl.h" 21 22 #include <stdio.h> 23 24 #include <binder/IMemory.h> 25 #include <drm/drm_framework_common.h> 26 #include <media/IDataSource.h> 27 #include <media/mediametadataretriever.h> 28 #include <media/stagefright/foundation/ADebug.h> 29 #include <media/stagefright/MediaSource.h> 30 #include <private/media/VideoFrame.h> 31 #include <utils/Log.h> 32 #include <utils/RefBase.h> 33 34 HeifDecoder* createHeifDecoder() { 35 return new android::HeifDecoderImpl(); 36 } 37 38 namespace android { 39 40 /* 41 * HeifDataSource 42 * 43 * Proxies data requests over IDataSource interface from MediaMetadataRetriever 44 * to the HeifStream interface we received from the heif decoder client. 45 */ 46 class HeifDataSource : public BnDataSource { 47 public: 48 /* 49 * Constructs HeifDataSource; will take ownership of |stream|. 50 */ 51 HeifDataSource(HeifStream* stream) 52 : mStream(stream), mEOS(false), 53 mCachedOffset(0), mCachedSize(0), mCacheBufferSize(0) {} 54 55 ~HeifDataSource() override {} 56 57 /* 58 * Initializes internal resources. 59 */ 60 bool init(); 61 62 sp<IMemory> getIMemory() override { return mMemory; } 63 ssize_t readAt(off64_t offset, size_t size) override; 64 status_t getSize(off64_t* size) override ; 65 void close() {} 66 uint32_t getFlags() override { return 0; } 67 String8 toString() override { return String8("HeifDataSource"); } 68 sp<DecryptHandle> DrmInitialization(const char*) override { 69 return nullptr; 70 } 71 72 private: 73 enum { 74 /* 75 * Buffer size for passing the read data to mediaserver. Set to 64K 76 * (which is what MediaDataSource Java API's jni implementation uses). 77 */ 78 kBufferSize = 64 * 1024, 79 /* 80 * Initial and max cache buffer size. 81 */ 82 kInitialCacheBufferSize = 4 * 1024 * 1024, 83 kMaxCacheBufferSize = 64 * 1024 * 1024, 84 }; 85 sp<IMemory> mMemory; 86 std::unique_ptr<HeifStream> mStream; 87 bool mEOS; 88 std::unique_ptr<uint8_t> mCache; 89 off64_t mCachedOffset; 90 size_t mCachedSize; 91 size_t mCacheBufferSize; 92 }; 93 94 bool HeifDataSource::init() { 95 sp<MemoryDealer> memoryDealer = 96 new MemoryDealer(kBufferSize, "HeifDataSource"); 97 mMemory = memoryDealer->allocate(kBufferSize); 98 if (mMemory == nullptr) { 99 ALOGE("Failed to allocate shared memory!"); 100 return false; 101 } 102 mCache.reset(new uint8_t[kInitialCacheBufferSize]); 103 if (mCache.get() == nullptr) { 104 ALOGE("mFailed to allocate cache!"); 105 return false; 106 } 107 mCacheBufferSize = kInitialCacheBufferSize; 108 return true; 109 } 110 111 ssize_t HeifDataSource::readAt(off64_t offset, size_t size) { 112 ALOGV("readAt: offset=%lld, size=%zu", (long long)offset, size); 113 114 if (offset < mCachedOffset) { 115 // try seek, then rewind/skip, fail if none worked 116 if (mStream->seek(offset)) { 117 ALOGV("readAt: seek to offset=%lld", (long long)offset); 118 mCachedOffset = offset; 119 mCachedSize = 0; 120 mEOS = false; 121 } else if (mStream->rewind()) { 122 ALOGV("readAt: rewind to offset=0"); 123 mCachedOffset = 0; 124 mCachedSize = 0; 125 mEOS = false; 126 } else { 127 ALOGE("readAt: couldn't seek or rewind!"); 128 mEOS = true; 129 } 130 } 131 132 if (mEOS && (offset < mCachedOffset || 133 offset >= (off64_t)(mCachedOffset + mCachedSize))) { 134 ALOGV("readAt: EOS"); 135 return ERROR_END_OF_STREAM; 136 } 137 138 // at this point, offset must be >= mCachedOffset, other cases should 139 // have been caught above. 140 CHECK(offset >= mCachedOffset); 141 142 if (size == 0) { 143 return 0; 144 } 145 146 // Can only read max of kBufferSize 147 if (size > kBufferSize) { 148 size = kBufferSize; 149 } 150 151 // copy from cache if the request falls entirely in cache 152 if (offset + size <= mCachedOffset + mCachedSize) { 153 memcpy(mMemory->pointer(), mCache.get() + offset - mCachedOffset, size); 154 return size; 155 } 156 157 // need to fetch more, check if we need to expand the cache buffer. 158 if ((off64_t)(offset + size) > mCachedOffset + kMaxCacheBufferSize) { 159 // it's reaching max cache buffer size, need to roll window, and possibly 160 // expand the cache buffer. 161 size_t newCacheBufferSize = mCacheBufferSize; 162 std::unique_ptr<uint8_t> newCache; 163 uint8_t* dst = mCache.get(); 164 if (newCacheBufferSize < kMaxCacheBufferSize) { 165 newCacheBufferSize = kMaxCacheBufferSize; 166 newCache.reset(new uint8_t[newCacheBufferSize]); 167 dst = newCache.get(); 168 } 169 170 // when rolling the cache window, try to keep about half the old bytes 171 // in case that the client goes back. 172 off64_t newCachedOffset = offset - (off64_t)(newCacheBufferSize / 2); 173 if (newCachedOffset < mCachedOffset) { 174 newCachedOffset = mCachedOffset; 175 } 176 177 int64_t newCachedSize = (int64_t)(mCachedOffset + mCachedSize) - newCachedOffset; 178 if (newCachedSize > 0) { 179 // in this case, the new cache region partially overlop the old cache, 180 // move the portion of the cache we want to save to the beginning of 181 // the cache buffer. 182 memcpy(dst, mCache.get() + newCachedOffset - mCachedOffset, newCachedSize); 183 } else if (newCachedSize < 0){ 184 // in this case, the new cache region is entirely out of the old cache, 185 // in order to guarantee sequential read, we need to skip a number of 186 // bytes before reading. 187 size_t bytesToSkip = -newCachedSize; 188 size_t bytesSkipped = mStream->read(nullptr, bytesToSkip); 189 if (bytesSkipped != bytesToSkip) { 190 // bytesSkipped is invalid, there is not enough bytes to reach 191 // the requested offset. 192 ALOGE("readAt: skip failed, EOS"); 193 194 mEOS = true; 195 mCachedOffset = newCachedOffset; 196 mCachedSize = 0; 197 return ERROR_END_OF_STREAM; 198 } 199 // set cache size to 0, since we're not keeping any old cache 200 newCachedSize = 0; 201 } 202 203 if (newCache.get() != nullptr) { 204 mCache.reset(newCache.release()); 205 mCacheBufferSize = newCacheBufferSize; 206 } 207 mCachedOffset = newCachedOffset; 208 mCachedSize = newCachedSize; 209 210 ALOGV("readAt: rolling cache window to (%lld, %zu), cache buffer size %zu", 211 (long long)mCachedOffset, mCachedSize, mCacheBufferSize); 212 } else { 213 // expand cache buffer, but no need to roll the window 214 size_t newCacheBufferSize = mCacheBufferSize; 215 while (offset + size > mCachedOffset + newCacheBufferSize) { 216 newCacheBufferSize *= 2; 217 } 218 CHECK(newCacheBufferSize <= kMaxCacheBufferSize); 219 if (mCacheBufferSize < newCacheBufferSize) { 220 uint8_t* newCache = new uint8_t[newCacheBufferSize]; 221 memcpy(newCache, mCache.get(), mCachedSize); 222 mCache.reset(newCache); 223 mCacheBufferSize = newCacheBufferSize; 224 225 ALOGV("readAt: current cache window (%lld, %zu), new cache buffer size %zu", 226 (long long) mCachedOffset, mCachedSize, mCacheBufferSize); 227 } 228 } 229 size_t bytesToRead = offset + size - mCachedOffset - mCachedSize; 230 size_t bytesRead = mStream->read(mCache.get() + mCachedSize, bytesToRead); 231 if (bytesRead > bytesToRead || bytesRead == 0) { 232 // bytesRead is invalid 233 mEOS = true; 234 bytesRead = 0; 235 } else if (bytesRead < bytesToRead) { 236 // read some bytes but not all, set EOS 237 mEOS = true; 238 } 239 mCachedSize += bytesRead; 240 ALOGV("readAt: current cache window (%lld, %zu)", 241 (long long) mCachedOffset, mCachedSize); 242 243 // here bytesAvailable could be negative if offset jumped past EOS. 244 int64_t bytesAvailable = mCachedOffset + mCachedSize - offset; 245 if (bytesAvailable <= 0) { 246 return ERROR_END_OF_STREAM; 247 } 248 if (bytesAvailable < (int64_t)size) { 249 size = bytesAvailable; 250 } 251 memcpy(mMemory->pointer(), mCache.get() + offset - mCachedOffset, size); 252 return size; 253 } 254 255 status_t HeifDataSource::getSize(off64_t* size) { 256 if (!mStream->hasLength()) { 257 *size = -1; 258 ALOGE("getSize: not supported!"); 259 return ERROR_UNSUPPORTED; 260 } 261 *size = mStream->getLength(); 262 ALOGV("getSize: size=%lld", (long long)*size); 263 return OK; 264 } 265 266 ///////////////////////////////////////////////////////////////////////// 267 268 HeifDecoderImpl::HeifDecoderImpl() : 269 // output color format should always be set via setOutputColor(), in case 270 // it's not, default to HAL_PIXEL_FORMAT_RGB_565. 271 mOutputColor(HAL_PIXEL_FORMAT_RGB_565), 272 mCurScanline(0), 273 mFrameDecoded(false) { 274 } 275 276 HeifDecoderImpl::~HeifDecoderImpl() { 277 } 278 279 bool HeifDecoderImpl::init(HeifStream* stream, HeifFrameInfo* frameInfo) { 280 mFrameDecoded = false; 281 sp<HeifDataSource> dataSource = new HeifDataSource(stream); 282 if (!dataSource->init()) { 283 return false; 284 } 285 mDataSource = dataSource; 286 287 mRetriever = new MediaMetadataRetriever(); 288 status_t err = mRetriever->setDataSource(mDataSource, "video/mp4"); 289 if (err != OK) { 290 ALOGE("failed to set data source!"); 291 292 mRetriever.clear(); 293 mDataSource.clear(); 294 return false; 295 } 296 ALOGV("successfully set data source."); 297 298 const char* hasVideo = mRetriever->extractMetadata(METADATA_KEY_HAS_VIDEO); 299 if (!hasVideo || strcasecmp(hasVideo, "yes")) { 300 ALOGE("no video: %s", hasVideo ? hasVideo : "null"); 301 return false; 302 } 303 304 mFrameMemory = mRetriever->getFrameAtTime(0, 305 IMediaSource::ReadOptions::SEEK_PREVIOUS_SYNC, 306 mOutputColor, true /*metaOnly*/); 307 if (mFrameMemory == nullptr || mFrameMemory->pointer() == nullptr) { 308 ALOGE("getFrameAtTime: videoFrame is a nullptr"); 309 return false; 310 } 311 312 VideoFrame* videoFrame = static_cast<VideoFrame*>(mFrameMemory->pointer()); 313 314 ALOGV("Meta dimension %dx%d, display %dx%d, angle %d, iccSize %d", 315 videoFrame->mWidth, 316 videoFrame->mHeight, 317 videoFrame->mDisplayWidth, 318 videoFrame->mDisplayHeight, 319 videoFrame->mRotationAngle, 320 videoFrame->mIccSize); 321 322 if (frameInfo != nullptr) { 323 frameInfo->set( 324 videoFrame->mDisplayWidth, 325 videoFrame->mDisplayHeight, 326 videoFrame->mRotationAngle, 327 videoFrame->mBytesPerPixel, 328 videoFrame->mIccSize, 329 videoFrame->getFlattenedIccData()); 330 } 331 return true; 332 } 333 334 bool HeifDecoderImpl::getEncodedColor(HeifEncodedColor* /*outColor*/) const { 335 ALOGW("getEncodedColor: not implemented!"); 336 return false; 337 } 338 339 bool HeifDecoderImpl::setOutputColor(HeifColorFormat heifColor) { 340 switch(heifColor) { 341 case kHeifColorFormat_RGB565: 342 { 343 mOutputColor = HAL_PIXEL_FORMAT_RGB_565; 344 return true; 345 } 346 case kHeifColorFormat_RGBA_8888: 347 { 348 mOutputColor = HAL_PIXEL_FORMAT_RGBA_8888; 349 return true; 350 } 351 case kHeifColorFormat_BGRA_8888: 352 { 353 mOutputColor = HAL_PIXEL_FORMAT_BGRA_8888; 354 return true; 355 } 356 default: 357 break; 358 } 359 ALOGE("Unsupported output color format %d", heifColor); 360 return false; 361 } 362 363 bool HeifDecoderImpl::decode(HeifFrameInfo* frameInfo) { 364 // reset scanline pointer 365 mCurScanline = 0; 366 367 if (mFrameDecoded) { 368 return true; 369 } 370 371 mFrameMemory = mRetriever->getFrameAtTime(0, 372 IMediaSource::ReadOptions::SEEK_PREVIOUS_SYNC, mOutputColor); 373 if (mFrameMemory == nullptr || mFrameMemory->pointer() == nullptr) { 374 ALOGE("getFrameAtTime: videoFrame is a nullptr"); 375 return false; 376 } 377 378 VideoFrame* videoFrame = static_cast<VideoFrame*>(mFrameMemory->pointer()); 379 if (videoFrame->mSize == 0 || 380 mFrameMemory->size() < videoFrame->getFlattenedSize()) { 381 ALOGE("getFrameAtTime: videoFrame size is invalid"); 382 return false; 383 } 384 385 ALOGV("Decoded dimension %dx%d, display %dx%d, angle %d, rowbytes %d, size %d", 386 videoFrame->mWidth, 387 videoFrame->mHeight, 388 videoFrame->mDisplayWidth, 389 videoFrame->mDisplayHeight, 390 videoFrame->mRotationAngle, 391 videoFrame->mRowBytes, 392 videoFrame->mSize); 393 394 if (frameInfo != nullptr) { 395 frameInfo->set( 396 videoFrame->mDisplayWidth, 397 videoFrame->mDisplayHeight, 398 videoFrame->mRotationAngle, 399 videoFrame->mBytesPerPixel, 400 videoFrame->mIccSize, 401 videoFrame->getFlattenedIccData()); 402 } 403 mFrameDecoded = true; 404 405 // Aggressive clear to avoid holding on to resources 406 mRetriever.clear(); 407 mDataSource.clear(); 408 return true; 409 } 410 411 bool HeifDecoderImpl::getScanline(uint8_t* dst) { 412 if (mFrameMemory == nullptr || mFrameMemory->pointer() == nullptr) { 413 return false; 414 } 415 VideoFrame* videoFrame = static_cast<VideoFrame*>(mFrameMemory->pointer()); 416 if (mCurScanline >= videoFrame->mDisplayHeight) { 417 ALOGE("no more scanline available"); 418 return false; 419 } 420 uint8_t* src = videoFrame->getFlattenedData() + videoFrame->mRowBytes * mCurScanline++; 421 memcpy(dst, src, videoFrame->mBytesPerPixel * videoFrame->mDisplayWidth); 422 return true; 423 } 424 425 size_t HeifDecoderImpl::skipScanlines(size_t count) { 426 if (mFrameMemory == nullptr || mFrameMemory->pointer() == nullptr) { 427 return 0; 428 } 429 VideoFrame* videoFrame = static_cast<VideoFrame*>(mFrameMemory->pointer()); 430 431 uint32_t oldScanline = mCurScanline; 432 mCurScanline += count; 433 if (mCurScanline > videoFrame->mDisplayHeight) { 434 mCurScanline = videoFrame->mDisplayHeight; 435 } 436 return (mCurScanline > oldScanline) ? (mCurScanline - oldScanline) : 0; 437 } 438 439 } // namespace android 440