1 /* 2 * libjingle 3 * Copyright 2015 Google Inc. 4 * 5 * Redistribution and use in source and binary forms, with or without 6 * modification, are permitted provided that the following conditions are met: 7 * 8 * 1. Redistributions of source code must retain the above copyright notice, 9 * this list of conditions and the following disclaimer. 10 * 2. Redistributions in binary form must reproduce the above copyright notice, 11 * this list of conditions and the following disclaimer in the documentation 12 * and/or other materials provided with the distribution. 13 * 3. The name of the author may not be used to endorse or promote products 14 * derived from this software without specific prior written permission. 15 * 16 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR IMPLIED 17 * WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF 18 * MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO 19 * EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 20 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, 21 * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; 22 * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, 23 * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR 24 * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF 25 * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 26 * 27 */ 28 29 #include <algorithm> 30 #include <vector> 31 32 #include "talk/app/webrtc/java/jni/androidmediadecoder_jni.h" 33 #include "talk/app/webrtc/java/jni/androidmediacodeccommon.h" 34 #include "talk/app/webrtc/java/jni/classreferenceholder.h" 35 #include "talk/app/webrtc/java/jni/native_handle_impl.h" 36 #include "talk/app/webrtc/java/jni/surfacetexturehelper_jni.h" 37 #include "webrtc/base/bind.h" 38 #include "webrtc/base/checks.h" 39 #include "webrtc/base/logging.h" 40 #include "webrtc/base/scoped_ref_ptr.h" 41 #include "webrtc/base/thread.h" 42 #include "webrtc/base/timeutils.h" 43 #include "webrtc/common_video/include/i420_buffer_pool.h" 44 #include "webrtc/modules/video_coding/include/video_codec_interface.h" 45 #include "webrtc/system_wrappers/include/logcat_trace_context.h" 46 #include "webrtc/system_wrappers/include/tick_util.h" 47 #include "third_party/libyuv/include/libyuv/convert.h" 48 #include "third_party/libyuv/include/libyuv/convert_from.h" 49 #include "third_party/libyuv/include/libyuv/video_common.h" 50 51 using rtc::Bind; 52 using rtc::Thread; 53 using rtc::ThreadManager; 54 using rtc::scoped_ptr; 55 56 using webrtc::CodecSpecificInfo; 57 using webrtc::DecodedImageCallback; 58 using webrtc::EncodedImage; 59 using webrtc::VideoFrame; 60 using webrtc::RTPFragmentationHeader; 61 using webrtc::TickTime; 62 using webrtc::VideoCodec; 63 using webrtc::VideoCodecType; 64 using webrtc::kVideoCodecH264; 65 using webrtc::kVideoCodecVP8; 66 using webrtc::kVideoCodecVP9; 67 68 namespace webrtc_jni { 69 70 class MediaCodecVideoDecoder : public webrtc::VideoDecoder, 71 public rtc::MessageHandler { 72 public: 73 explicit MediaCodecVideoDecoder( 74 JNIEnv* jni, VideoCodecType codecType, jobject render_egl_context); 75 virtual ~MediaCodecVideoDecoder(); 76 77 int32_t InitDecode(const VideoCodec* codecSettings, int32_t numberOfCores) 78 override; 79 80 int32_t Decode( 81 const EncodedImage& inputImage, bool missingFrames, 82 const RTPFragmentationHeader* fragmentation, 83 const CodecSpecificInfo* codecSpecificInfo = NULL, 84 int64_t renderTimeMs = -1) override; 85 86 int32_t RegisterDecodeCompleteCallback(DecodedImageCallback* callback) 87 override; 88 89 int32_t Release() override; 90 91 int32_t Reset() override; 92 93 bool PrefersLateDecoding() const override { return true; } 94 95 // rtc::MessageHandler implementation. 96 void OnMessage(rtc::Message* msg) override; 97 98 const char* ImplementationName() const override; 99 100 private: 101 // CHECK-fail if not running on |codec_thread_|. 102 void CheckOnCodecThread(); 103 104 int32_t InitDecodeOnCodecThread(); 105 int32_t ReleaseOnCodecThread(); 106 int32_t DecodeOnCodecThread(const EncodedImage& inputImage); 107 // Deliver any outputs pending in the MediaCodec to our |callback_| and return 108 // true on success. 109 bool DeliverPendingOutputs(JNIEnv* jni, int dequeue_timeout_us); 110 int32_t ProcessHWErrorOnCodecThread(); 111 112 // Type of video codec. 113 VideoCodecType codecType_; 114 115 // Render EGL context - owned by factory, should not be allocated/destroyed 116 // by VideoDecoder. 117 jobject render_egl_context_; 118 119 bool key_frame_required_; 120 bool inited_; 121 bool sw_fallback_required_; 122 bool use_surface_; 123 VideoCodec codec_; 124 webrtc::I420BufferPool decoded_frame_pool_; 125 rtc::scoped_refptr<SurfaceTextureHelper> surface_texture_helper_; 126 DecodedImageCallback* callback_; 127 int frames_received_; // Number of frames received by decoder. 128 int frames_decoded_; // Number of frames decoded by decoder. 129 int64_t start_time_ms_; // Start time for statistics. 130 int current_frames_; // Number of frames in the current statistics interval. 131 int current_bytes_; // Encoded bytes in the current statistics interval. 132 int current_decoding_time_ms_; // Overall decoding time in the current second 133 uint32_t max_pending_frames_; // Maximum number of pending input frames 134 135 // State that is constant for the lifetime of this object once the ctor 136 // returns. 137 scoped_ptr<Thread> codec_thread_; // Thread on which to operate MediaCodec. 138 ScopedGlobalRef<jclass> j_media_codec_video_decoder_class_; 139 ScopedGlobalRef<jobject> j_media_codec_video_decoder_; 140 jmethodID j_init_decode_method_; 141 jmethodID j_release_method_; 142 jmethodID j_dequeue_input_buffer_method_; 143 jmethodID j_queue_input_buffer_method_; 144 jmethodID j_dequeue_byte_buffer_method_; 145 jmethodID j_dequeue_texture_buffer_method_; 146 jmethodID j_return_decoded_byte_buffer_method_; 147 // MediaCodecVideoDecoder fields. 148 jfieldID j_input_buffers_field_; 149 jfieldID j_output_buffers_field_; 150 jfieldID j_color_format_field_; 151 jfieldID j_width_field_; 152 jfieldID j_height_field_; 153 jfieldID j_stride_field_; 154 jfieldID j_slice_height_field_; 155 // MediaCodecVideoDecoder.DecodedTextureBuffer fields. 156 jfieldID j_texture_id_field_; 157 jfieldID j_transform_matrix_field_; 158 jfieldID j_texture_timestamp_ms_field_; 159 jfieldID j_texture_ntp_timestamp_ms_field_; 160 jfieldID j_texture_decode_time_ms_field_; 161 jfieldID j_texture_frame_delay_ms_field_; 162 // MediaCodecVideoDecoder.DecodedOutputBuffer fields. 163 jfieldID j_info_index_field_; 164 jfieldID j_info_offset_field_; 165 jfieldID j_info_size_field_; 166 jfieldID j_info_timestamp_ms_field_; 167 jfieldID j_info_ntp_timestamp_ms_field_; 168 jfieldID j_byte_buffer_decode_time_ms_field_; 169 170 // Global references; must be deleted in Release(). 171 std::vector<jobject> input_buffers_; 172 }; 173 174 MediaCodecVideoDecoder::MediaCodecVideoDecoder( 175 JNIEnv* jni, VideoCodecType codecType, jobject render_egl_context) : 176 codecType_(codecType), 177 render_egl_context_(render_egl_context), 178 key_frame_required_(true), 179 inited_(false), 180 sw_fallback_required_(false), 181 codec_thread_(new Thread()), 182 j_media_codec_video_decoder_class_( 183 jni, 184 FindClass(jni, "org/webrtc/MediaCodecVideoDecoder")), 185 j_media_codec_video_decoder_( 186 jni, 187 jni->NewObject(*j_media_codec_video_decoder_class_, 188 GetMethodID(jni, 189 *j_media_codec_video_decoder_class_, 190 "<init>", 191 "()V"))) { 192 ScopedLocalRefFrame local_ref_frame(jni); 193 codec_thread_->SetName("MediaCodecVideoDecoder", NULL); 194 RTC_CHECK(codec_thread_->Start()) << "Failed to start MediaCodecVideoDecoder"; 195 196 j_init_decode_method_ = GetMethodID( 197 jni, *j_media_codec_video_decoder_class_, "initDecode", 198 "(Lorg/webrtc/MediaCodecVideoDecoder$VideoCodecType;" 199 "IILorg/webrtc/SurfaceTextureHelper;)Z"); 200 j_release_method_ = 201 GetMethodID(jni, *j_media_codec_video_decoder_class_, "release", "()V"); 202 j_dequeue_input_buffer_method_ = GetMethodID( 203 jni, *j_media_codec_video_decoder_class_, "dequeueInputBuffer", "()I"); 204 j_queue_input_buffer_method_ = GetMethodID( 205 jni, *j_media_codec_video_decoder_class_, "queueInputBuffer", "(IIJJJ)Z"); 206 j_dequeue_byte_buffer_method_ = GetMethodID( 207 jni, *j_media_codec_video_decoder_class_, "dequeueOutputBuffer", 208 "(I)Lorg/webrtc/MediaCodecVideoDecoder$DecodedOutputBuffer;"); 209 j_dequeue_texture_buffer_method_ = GetMethodID( 210 jni, *j_media_codec_video_decoder_class_, "dequeueTextureBuffer", 211 "(I)Lorg/webrtc/MediaCodecVideoDecoder$DecodedTextureBuffer;"); 212 j_return_decoded_byte_buffer_method_ = 213 GetMethodID(jni, *j_media_codec_video_decoder_class_, 214 "returnDecodedOutputBuffer", "(I)V"); 215 216 j_input_buffers_field_ = GetFieldID( 217 jni, *j_media_codec_video_decoder_class_, 218 "inputBuffers", "[Ljava/nio/ByteBuffer;"); 219 j_output_buffers_field_ = GetFieldID( 220 jni, *j_media_codec_video_decoder_class_, 221 "outputBuffers", "[Ljava/nio/ByteBuffer;"); 222 j_color_format_field_ = GetFieldID( 223 jni, *j_media_codec_video_decoder_class_, "colorFormat", "I"); 224 j_width_field_ = GetFieldID( 225 jni, *j_media_codec_video_decoder_class_, "width", "I"); 226 j_height_field_ = GetFieldID( 227 jni, *j_media_codec_video_decoder_class_, "height", "I"); 228 j_stride_field_ = GetFieldID( 229 jni, *j_media_codec_video_decoder_class_, "stride", "I"); 230 j_slice_height_field_ = GetFieldID( 231 jni, *j_media_codec_video_decoder_class_, "sliceHeight", "I"); 232 233 jclass j_decoded_texture_buffer_class = FindClass(jni, 234 "org/webrtc/MediaCodecVideoDecoder$DecodedTextureBuffer"); 235 j_texture_id_field_ = GetFieldID( 236 jni, j_decoded_texture_buffer_class, "textureID", "I"); 237 j_transform_matrix_field_ = GetFieldID( 238 jni, j_decoded_texture_buffer_class, "transformMatrix", "[F"); 239 j_texture_timestamp_ms_field_ = GetFieldID( 240 jni, j_decoded_texture_buffer_class, "timeStampMs", "J"); 241 j_texture_ntp_timestamp_ms_field_ = GetFieldID( 242 jni, j_decoded_texture_buffer_class, "ntpTimeStampMs", "J"); 243 j_texture_decode_time_ms_field_ = GetFieldID( 244 jni, j_decoded_texture_buffer_class, "decodeTimeMs", "J"); 245 j_texture_frame_delay_ms_field_ = GetFieldID( 246 jni, j_decoded_texture_buffer_class, "frameDelayMs", "J"); 247 248 jclass j_decoded_output_buffer_class = FindClass(jni, 249 "org/webrtc/MediaCodecVideoDecoder$DecodedOutputBuffer"); 250 j_info_index_field_ = GetFieldID( 251 jni, j_decoded_output_buffer_class, "index", "I"); 252 j_info_offset_field_ = GetFieldID( 253 jni, j_decoded_output_buffer_class, "offset", "I"); 254 j_info_size_field_ = GetFieldID( 255 jni, j_decoded_output_buffer_class, "size", "I"); 256 j_info_timestamp_ms_field_ = GetFieldID( 257 jni, j_decoded_output_buffer_class, "timeStampMs", "J"); 258 j_info_ntp_timestamp_ms_field_ = GetFieldID( 259 jni, j_decoded_output_buffer_class, "ntpTimeStampMs", "J"); 260 j_byte_buffer_decode_time_ms_field_ = GetFieldID( 261 jni, j_decoded_output_buffer_class, "decodeTimeMs", "J"); 262 263 CHECK_EXCEPTION(jni) << "MediaCodecVideoDecoder ctor failed"; 264 use_surface_ = (render_egl_context_ != NULL); 265 ALOGD << "MediaCodecVideoDecoder ctor. Use surface: " << use_surface_; 266 memset(&codec_, 0, sizeof(codec_)); 267 AllowBlockingCalls(); 268 } 269 270 MediaCodecVideoDecoder::~MediaCodecVideoDecoder() { 271 // Call Release() to ensure no more callbacks to us after we are deleted. 272 Release(); 273 } 274 275 int32_t MediaCodecVideoDecoder::InitDecode(const VideoCodec* inst, 276 int32_t numberOfCores) { 277 ALOGD << "InitDecode."; 278 if (inst == NULL) { 279 ALOGE << "NULL VideoCodec instance"; 280 return WEBRTC_VIDEO_CODEC_ERR_PARAMETER; 281 } 282 // Factory should guard against other codecs being used with us. 283 RTC_CHECK(inst->codecType == codecType_) 284 << "Unsupported codec " << inst->codecType << " for " << codecType_; 285 286 if (sw_fallback_required_) { 287 ALOGE << "InitDecode() - fallback to SW decoder"; 288 return WEBRTC_VIDEO_CODEC_OK; 289 } 290 // Save VideoCodec instance for later. 291 if (&codec_ != inst) { 292 codec_ = *inst; 293 } 294 // If maxFramerate is not set then assume 30 fps. 295 codec_.maxFramerate = (codec_.maxFramerate >= 1) ? codec_.maxFramerate : 30; 296 297 // Call Java init. 298 return codec_thread_->Invoke<int32_t>( 299 Bind(&MediaCodecVideoDecoder::InitDecodeOnCodecThread, this)); 300 } 301 302 int32_t MediaCodecVideoDecoder::InitDecodeOnCodecThread() { 303 CheckOnCodecThread(); 304 JNIEnv* jni = AttachCurrentThreadIfNeeded(); 305 ScopedLocalRefFrame local_ref_frame(jni); 306 ALOGD << "InitDecodeOnCodecThread Type: " << (int)codecType_ << ". " 307 << codec_.width << " x " << codec_.height << ". Fps: " << 308 (int)codec_.maxFramerate; 309 310 // Release previous codec first if it was allocated before. 311 int ret_val = ReleaseOnCodecThread(); 312 if (ret_val < 0) { 313 ALOGE << "Release failure: " << ret_val << " - fallback to SW codec"; 314 sw_fallback_required_ = true; 315 return WEBRTC_VIDEO_CODEC_ERROR; 316 } 317 318 // Always start with a complete key frame. 319 key_frame_required_ = true; 320 frames_received_ = 0; 321 frames_decoded_ = 0; 322 323 jobject java_surface_texture_helper_ = nullptr; 324 if (use_surface_) { 325 java_surface_texture_helper_ = jni->CallStaticObjectMethod( 326 FindClass(jni, "org/webrtc/SurfaceTextureHelper"), 327 GetStaticMethodID(jni, 328 FindClass(jni, "org/webrtc/SurfaceTextureHelper"), 329 "create", 330 "(Lorg/webrtc/EglBase$Context;)" 331 "Lorg/webrtc/SurfaceTextureHelper;"), 332 render_egl_context_); 333 RTC_CHECK(java_surface_texture_helper_ != nullptr); 334 surface_texture_helper_ = new rtc::RefCountedObject<SurfaceTextureHelper>( 335 jni, java_surface_texture_helper_); 336 } 337 338 jobject j_video_codec_enum = JavaEnumFromIndex( 339 jni, "MediaCodecVideoDecoder$VideoCodecType", codecType_); 340 bool success = jni->CallBooleanMethod( 341 *j_media_codec_video_decoder_, 342 j_init_decode_method_, 343 j_video_codec_enum, 344 codec_.width, 345 codec_.height, 346 java_surface_texture_helper_); 347 if (CheckException(jni) || !success) { 348 ALOGE << "Codec initialization error - fallback to SW codec."; 349 sw_fallback_required_ = true; 350 return WEBRTC_VIDEO_CODEC_ERROR; 351 } 352 inited_ = true; 353 354 switch (codecType_) { 355 case kVideoCodecVP8: 356 max_pending_frames_ = kMaxPendingFramesVp8; 357 break; 358 case kVideoCodecVP9: 359 max_pending_frames_ = kMaxPendingFramesVp9; 360 break; 361 case kVideoCodecH264: 362 max_pending_frames_ = kMaxPendingFramesH264; 363 break; 364 default: 365 max_pending_frames_ = 0; 366 } 367 start_time_ms_ = GetCurrentTimeMs(); 368 current_frames_ = 0; 369 current_bytes_ = 0; 370 current_decoding_time_ms_ = 0; 371 372 jobjectArray input_buffers = (jobjectArray)GetObjectField( 373 jni, *j_media_codec_video_decoder_, j_input_buffers_field_); 374 size_t num_input_buffers = jni->GetArrayLength(input_buffers); 375 ALOGD << "Maximum amount of pending frames: " << max_pending_frames_; 376 input_buffers_.resize(num_input_buffers); 377 for (size_t i = 0; i < num_input_buffers; ++i) { 378 input_buffers_[i] = 379 jni->NewGlobalRef(jni->GetObjectArrayElement(input_buffers, i)); 380 if (CheckException(jni)) { 381 ALOGE << "NewGlobalRef error - fallback to SW codec."; 382 sw_fallback_required_ = true; 383 return WEBRTC_VIDEO_CODEC_ERROR; 384 } 385 } 386 387 codec_thread_->PostDelayed(kMediaCodecPollMs, this); 388 389 return WEBRTC_VIDEO_CODEC_OK; 390 } 391 392 int32_t MediaCodecVideoDecoder::Release() { 393 ALOGD << "DecoderRelease request"; 394 return codec_thread_->Invoke<int32_t>( 395 Bind(&MediaCodecVideoDecoder::ReleaseOnCodecThread, this)); 396 } 397 398 int32_t MediaCodecVideoDecoder::ReleaseOnCodecThread() { 399 if (!inited_) { 400 return WEBRTC_VIDEO_CODEC_OK; 401 } 402 CheckOnCodecThread(); 403 JNIEnv* jni = AttachCurrentThreadIfNeeded(); 404 ALOGD << "DecoderReleaseOnCodecThread: Frames received: " << 405 frames_received_ << ". Frames decoded: " << frames_decoded_; 406 ScopedLocalRefFrame local_ref_frame(jni); 407 for (size_t i = 0; i < input_buffers_.size(); i++) { 408 jni->DeleteGlobalRef(input_buffers_[i]); 409 } 410 input_buffers_.clear(); 411 jni->CallVoidMethod(*j_media_codec_video_decoder_, j_release_method_); 412 surface_texture_helper_ = nullptr; 413 inited_ = false; 414 rtc::MessageQueueManager::Clear(this); 415 if (CheckException(jni)) { 416 ALOGE << "Decoder release exception"; 417 return WEBRTC_VIDEO_CODEC_ERROR; 418 } 419 ALOGD << "DecoderReleaseOnCodecThread done"; 420 return WEBRTC_VIDEO_CODEC_OK; 421 } 422 423 void MediaCodecVideoDecoder::CheckOnCodecThread() { 424 RTC_CHECK(codec_thread_ == ThreadManager::Instance()->CurrentThread()) 425 << "Running on wrong thread!"; 426 } 427 428 int32_t MediaCodecVideoDecoder::ProcessHWErrorOnCodecThread() { 429 CheckOnCodecThread(); 430 int ret_val = ReleaseOnCodecThread(); 431 if (ret_val < 0) { 432 ALOGE << "ProcessHWError: Release failure"; 433 } 434 if (codecType_ == kVideoCodecH264) { 435 // For now there is no SW H.264 which can be used as fallback codec. 436 // So try to restart hw codec for now. 437 ret_val = InitDecodeOnCodecThread(); 438 ALOGE << "Reset H.264 codec done. Status: " << ret_val; 439 if (ret_val == WEBRTC_VIDEO_CODEC_OK) { 440 // H.264 codec was succesfully reset - return regular error code. 441 return WEBRTC_VIDEO_CODEC_ERROR; 442 } else { 443 // Fail to restart H.264 codec - return error code which should stop the 444 // call. 445 return WEBRTC_VIDEO_CODEC_FALLBACK_SOFTWARE; 446 } 447 } else { 448 sw_fallback_required_ = true; 449 ALOGE << "Return WEBRTC_VIDEO_CODEC_FALLBACK_SOFTWARE"; 450 return WEBRTC_VIDEO_CODEC_FALLBACK_SOFTWARE; 451 } 452 } 453 454 int32_t MediaCodecVideoDecoder::Decode( 455 const EncodedImage& inputImage, 456 bool missingFrames, 457 const RTPFragmentationHeader* fragmentation, 458 const CodecSpecificInfo* codecSpecificInfo, 459 int64_t renderTimeMs) { 460 if (sw_fallback_required_) { 461 ALOGE << "Decode() - fallback to SW codec"; 462 return WEBRTC_VIDEO_CODEC_FALLBACK_SOFTWARE; 463 } 464 if (callback_ == NULL) { 465 ALOGE << "Decode() - callback_ is NULL"; 466 return WEBRTC_VIDEO_CODEC_UNINITIALIZED; 467 } 468 if (inputImage._buffer == NULL && inputImage._length > 0) { 469 ALOGE << "Decode() - inputImage is incorrect"; 470 return WEBRTC_VIDEO_CODEC_ERR_PARAMETER; 471 } 472 if (!inited_) { 473 ALOGE << "Decode() - decoder is not initialized"; 474 return WEBRTC_VIDEO_CODEC_UNINITIALIZED; 475 } 476 477 // Check if encoded frame dimension has changed. 478 if ((inputImage._encodedWidth * inputImage._encodedHeight > 0) && 479 (inputImage._encodedWidth != codec_.width || 480 inputImage._encodedHeight != codec_.height)) { 481 codec_.width = inputImage._encodedWidth; 482 codec_.height = inputImage._encodedHeight; 483 int32_t ret = InitDecode(&codec_, 1); 484 if (ret < 0) { 485 ALOGE << "InitDecode failure: " << ret << " - fallback to SW codec"; 486 sw_fallback_required_ = true; 487 return WEBRTC_VIDEO_CODEC_FALLBACK_SOFTWARE; 488 } 489 } 490 491 // Always start with a complete key frame. 492 if (key_frame_required_) { 493 if (inputImage._frameType != webrtc::kVideoFrameKey) { 494 ALOGE << "Decode() - key frame is required"; 495 return WEBRTC_VIDEO_CODEC_ERROR; 496 } 497 if (!inputImage._completeFrame) { 498 ALOGE << "Decode() - complete frame is required"; 499 return WEBRTC_VIDEO_CODEC_ERROR; 500 } 501 key_frame_required_ = false; 502 } 503 if (inputImage._length == 0) { 504 return WEBRTC_VIDEO_CODEC_ERROR; 505 } 506 507 return codec_thread_->Invoke<int32_t>(Bind( 508 &MediaCodecVideoDecoder::DecodeOnCodecThread, this, inputImage)); 509 } 510 511 int32_t MediaCodecVideoDecoder::DecodeOnCodecThread( 512 const EncodedImage& inputImage) { 513 CheckOnCodecThread(); 514 JNIEnv* jni = AttachCurrentThreadIfNeeded(); 515 ScopedLocalRefFrame local_ref_frame(jni); 516 517 // Try to drain the decoder and wait until output is not too 518 // much behind the input. 519 const int64 drain_start = GetCurrentTimeMs(); 520 while ((frames_received_ > frames_decoded_ + max_pending_frames_) && 521 (GetCurrentTimeMs() - drain_start) < kMediaCodecTimeoutMs) { 522 ALOGV("Received: %d. Decoded: %d. Wait for output...", 523 frames_received_, frames_decoded_); 524 if (!DeliverPendingOutputs(jni, kMediaCodecPollMs)) { 525 ALOGE << "DeliverPendingOutputs error. Frames received: " << 526 frames_received_ << ". Frames decoded: " << frames_decoded_; 527 return ProcessHWErrorOnCodecThread(); 528 } 529 } 530 if (frames_received_ > frames_decoded_ + max_pending_frames_) { 531 ALOGE << "Output buffer dequeue timeout. Frames received: " << 532 frames_received_ << ". Frames decoded: " << frames_decoded_; 533 return ProcessHWErrorOnCodecThread(); 534 } 535 536 // Get input buffer. 537 int j_input_buffer_index = jni->CallIntMethod(*j_media_codec_video_decoder_, 538 j_dequeue_input_buffer_method_); 539 if (CheckException(jni) || j_input_buffer_index < 0) { 540 ALOGE << "dequeueInputBuffer error"; 541 return ProcessHWErrorOnCodecThread(); 542 } 543 544 // Copy encoded data to Java ByteBuffer. 545 jobject j_input_buffer = input_buffers_[j_input_buffer_index]; 546 uint8_t* buffer = 547 reinterpret_cast<uint8_t*>(jni->GetDirectBufferAddress(j_input_buffer)); 548 RTC_CHECK(buffer) << "Indirect buffer??"; 549 int64_t buffer_capacity = jni->GetDirectBufferCapacity(j_input_buffer); 550 if (CheckException(jni) || buffer_capacity < inputImage._length) { 551 ALOGE << "Input frame size "<< inputImage._length << 552 " is bigger than buffer size " << buffer_capacity; 553 return ProcessHWErrorOnCodecThread(); 554 } 555 jlong presentation_timestamp_us = 556 (frames_received_ * 1000000) / codec_.maxFramerate; 557 if (frames_decoded_ < kMaxDecodedLogFrames) { 558 ALOGD << "Decoder frame in # " << frames_received_ << ". Type: " 559 << inputImage._frameType << ". Buffer # " << 560 j_input_buffer_index << ". pTS: " 561 << (int)(presentation_timestamp_us / 1000) 562 << ". TS: " << inputImage._timeStamp 563 << ". Size: " << inputImage._length; 564 } 565 memcpy(buffer, inputImage._buffer, inputImage._length); 566 567 // Save input image timestamps for later output. 568 frames_received_++; 569 current_bytes_ += inputImage._length; 570 571 // Feed input to decoder. 572 bool success = jni->CallBooleanMethod( 573 *j_media_codec_video_decoder_, 574 j_queue_input_buffer_method_, 575 j_input_buffer_index, 576 inputImage._length, 577 presentation_timestamp_us, 578 static_cast<int64_t> (inputImage._timeStamp), 579 inputImage.ntp_time_ms_); 580 if (CheckException(jni) || !success) { 581 ALOGE << "queueInputBuffer error"; 582 return ProcessHWErrorOnCodecThread(); 583 } 584 585 // Try to drain the decoder 586 if (!DeliverPendingOutputs(jni, 0)) { 587 ALOGE << "DeliverPendingOutputs error"; 588 return ProcessHWErrorOnCodecThread(); 589 } 590 591 return WEBRTC_VIDEO_CODEC_OK; 592 } 593 594 bool MediaCodecVideoDecoder::DeliverPendingOutputs( 595 JNIEnv* jni, int dequeue_timeout_ms) { 596 if (frames_received_ <= frames_decoded_) { 597 // No need to query for output buffers - decoder is drained. 598 return true; 599 } 600 // Get decoder output. 601 jobject j_decoder_output_buffer = 602 jni->CallObjectMethod(*j_media_codec_video_decoder_, 603 use_surface_ ? j_dequeue_texture_buffer_method_ 604 : j_dequeue_byte_buffer_method_, 605 dequeue_timeout_ms); 606 607 if (CheckException(jni)) { 608 ALOGE << "dequeueOutputBuffer() error"; 609 return false; 610 } 611 if (IsNull(jni, j_decoder_output_buffer)) { 612 // No decoded frame ready. 613 return true; 614 } 615 616 // Get decoded video frame properties. 617 int color_format = GetIntField(jni, *j_media_codec_video_decoder_, 618 j_color_format_field_); 619 int width = GetIntField(jni, *j_media_codec_video_decoder_, j_width_field_); 620 int height = GetIntField(jni, *j_media_codec_video_decoder_, j_height_field_); 621 int stride = GetIntField(jni, *j_media_codec_video_decoder_, j_stride_field_); 622 int slice_height = GetIntField(jni, *j_media_codec_video_decoder_, 623 j_slice_height_field_); 624 625 rtc::scoped_refptr<webrtc::VideoFrameBuffer> frame_buffer; 626 int64_t output_timestamps_ms = 0; 627 int64_t output_ntp_timestamps_ms = 0; 628 int decode_time_ms = 0; 629 int64_t frame_delayed_ms = 0; 630 if (use_surface_) { 631 // Extract data from Java DecodedTextureBuffer. 632 const int texture_id = 633 GetIntField(jni, j_decoder_output_buffer, j_texture_id_field_); 634 if (texture_id != 0) { // |texture_id| == 0 represents a dropped frame. 635 const jfloatArray j_transform_matrix = 636 reinterpret_cast<jfloatArray>(GetObjectField( 637 jni, j_decoder_output_buffer, j_transform_matrix_field_)); 638 const int64_t timestamp_us = 639 GetLongField(jni, j_decoder_output_buffer, 640 j_texture_timestamp_ms_field_); 641 output_timestamps_ms = GetLongField(jni, j_decoder_output_buffer, 642 j_texture_timestamp_ms_field_); 643 output_ntp_timestamps_ms = 644 GetLongField(jni, j_decoder_output_buffer, 645 j_texture_ntp_timestamp_ms_field_); 646 decode_time_ms = GetLongField(jni, j_decoder_output_buffer, 647 j_texture_decode_time_ms_field_); 648 frame_delayed_ms = GetLongField(jni, j_decoder_output_buffer, 649 j_texture_frame_delay_ms_field_); 650 651 // Create webrtc::VideoFrameBuffer with native texture handle. 652 frame_buffer = surface_texture_helper_->CreateTextureFrame( 653 width, height, NativeHandleImpl(jni, texture_id, j_transform_matrix)); 654 } 655 } else { 656 // Extract data from Java ByteBuffer and create output yuv420 frame - 657 // for non surface decoding only. 658 const int output_buffer_index = 659 GetIntField(jni, j_decoder_output_buffer, j_info_index_field_); 660 const int output_buffer_offset = 661 GetIntField(jni, j_decoder_output_buffer, j_info_offset_field_); 662 const int output_buffer_size = 663 GetIntField(jni, j_decoder_output_buffer, j_info_size_field_); 664 output_timestamps_ms = GetLongField(jni, j_decoder_output_buffer, 665 j_info_timestamp_ms_field_); 666 output_ntp_timestamps_ms = 667 GetLongField(jni, j_decoder_output_buffer, 668 j_info_ntp_timestamp_ms_field_); 669 670 decode_time_ms = GetLongField(jni, j_decoder_output_buffer, 671 j_byte_buffer_decode_time_ms_field_); 672 673 if (output_buffer_size < width * height * 3 / 2) { 674 ALOGE << "Insufficient output buffer size: " << output_buffer_size; 675 return false; 676 } 677 jobjectArray output_buffers = reinterpret_cast<jobjectArray>(GetObjectField( 678 jni, *j_media_codec_video_decoder_, j_output_buffers_field_)); 679 jobject output_buffer = 680 jni->GetObjectArrayElement(output_buffers, output_buffer_index); 681 uint8_t* payload = reinterpret_cast<uint8_t*>(jni->GetDirectBufferAddress( 682 output_buffer)); 683 if (CheckException(jni)) { 684 return false; 685 } 686 payload += output_buffer_offset; 687 688 // Create yuv420 frame. 689 frame_buffer = decoded_frame_pool_.CreateBuffer(width, height); 690 if (color_format == COLOR_FormatYUV420Planar) { 691 RTC_CHECK_EQ(0, stride % 2); 692 RTC_CHECK_EQ(0, slice_height % 2); 693 const int uv_stride = stride / 2; 694 const int u_slice_height = slice_height / 2; 695 const uint8_t* y_ptr = payload; 696 const uint8_t* u_ptr = y_ptr + stride * slice_height; 697 const uint8_t* v_ptr = u_ptr + uv_stride * u_slice_height; 698 libyuv::I420Copy(y_ptr, stride, 699 u_ptr, uv_stride, 700 v_ptr, uv_stride, 701 frame_buffer->MutableData(webrtc::kYPlane), 702 frame_buffer->stride(webrtc::kYPlane), 703 frame_buffer->MutableData(webrtc::kUPlane), 704 frame_buffer->stride(webrtc::kUPlane), 705 frame_buffer->MutableData(webrtc::kVPlane), 706 frame_buffer->stride(webrtc::kVPlane), 707 width, height); 708 } else { 709 // All other supported formats are nv12. 710 const uint8_t* y_ptr = payload; 711 const uint8_t* uv_ptr = y_ptr + stride * slice_height; 712 libyuv::NV12ToI420( 713 y_ptr, stride, 714 uv_ptr, stride, 715 frame_buffer->MutableData(webrtc::kYPlane), 716 frame_buffer->stride(webrtc::kYPlane), 717 frame_buffer->MutableData(webrtc::kUPlane), 718 frame_buffer->stride(webrtc::kUPlane), 719 frame_buffer->MutableData(webrtc::kVPlane), 720 frame_buffer->stride(webrtc::kVPlane), 721 width, height); 722 } 723 // Return output byte buffer back to codec. 724 jni->CallVoidMethod( 725 *j_media_codec_video_decoder_, 726 j_return_decoded_byte_buffer_method_, 727 output_buffer_index); 728 if (CheckException(jni)) { 729 ALOGE << "returnDecodedOutputBuffer error"; 730 return false; 731 } 732 } 733 VideoFrame decoded_frame(frame_buffer, 0, 0, webrtc::kVideoRotation_0); 734 decoded_frame.set_timestamp(output_timestamps_ms); 735 decoded_frame.set_ntp_time_ms(output_ntp_timestamps_ms); 736 737 if (frames_decoded_ < kMaxDecodedLogFrames) { 738 ALOGD << "Decoder frame out # " << frames_decoded_ << ". " << width << 739 " x " << height << ". " << stride << " x " << slice_height << 740 ". Color: " << color_format << ". TS:" << decoded_frame.timestamp() << 741 ". DecTime: " << (int)decode_time_ms << 742 ". DelayTime: " << (int)frame_delayed_ms; 743 } 744 745 // Calculate and print decoding statistics - every 3 seconds. 746 frames_decoded_++; 747 current_frames_++; 748 current_decoding_time_ms_ += decode_time_ms; 749 int statistic_time_ms = GetCurrentTimeMs() - start_time_ms_; 750 if (statistic_time_ms >= kMediaCodecStatisticsIntervalMs && 751 current_frames_ > 0) { 752 ALOGD << "Decoded frames: " << frames_decoded_ << ". Received frames: " 753 << frames_received_ << ". Bitrate: " << 754 (current_bytes_ * 8 / statistic_time_ms) << " kbps, fps: " << 755 ((current_frames_ * 1000 + statistic_time_ms / 2) / statistic_time_ms) 756 << ". decTime: " << (current_decoding_time_ms_ / current_frames_) << 757 " for last " << statistic_time_ms << " ms."; 758 start_time_ms_ = GetCurrentTimeMs(); 759 current_frames_ = 0; 760 current_bytes_ = 0; 761 current_decoding_time_ms_ = 0; 762 } 763 764 // |.IsZeroSize())| returns true when a frame has been dropped. 765 if (!decoded_frame.IsZeroSize()) { 766 // Callback - output decoded frame. 767 const int32_t callback_status = 768 callback_->Decoded(decoded_frame, decode_time_ms); 769 if (callback_status > 0) { 770 ALOGE << "callback error"; 771 } 772 } 773 return true; 774 } 775 776 int32_t MediaCodecVideoDecoder::RegisterDecodeCompleteCallback( 777 DecodedImageCallback* callback) { 778 callback_ = callback; 779 return WEBRTC_VIDEO_CODEC_OK; 780 } 781 782 int32_t MediaCodecVideoDecoder::Reset() { 783 ALOGD << "DecoderReset"; 784 if (!inited_) { 785 return WEBRTC_VIDEO_CODEC_UNINITIALIZED; 786 } 787 return InitDecode(&codec_, 1); 788 } 789 790 void MediaCodecVideoDecoder::OnMessage(rtc::Message* msg) { 791 JNIEnv* jni = AttachCurrentThreadIfNeeded(); 792 ScopedLocalRefFrame local_ref_frame(jni); 793 if (!inited_) { 794 return; 795 } 796 // We only ever send one message to |this| directly (not through a Bind()'d 797 // functor), so expect no ID/data. 798 RTC_CHECK(!msg->message_id) << "Unexpected message!"; 799 RTC_CHECK(!msg->pdata) << "Unexpected message!"; 800 CheckOnCodecThread(); 801 802 if (!DeliverPendingOutputs(jni, 0)) { 803 ALOGE << "OnMessage: DeliverPendingOutputs error"; 804 ProcessHWErrorOnCodecThread(); 805 return; 806 } 807 codec_thread_->PostDelayed(kMediaCodecPollMs, this); 808 } 809 810 MediaCodecVideoDecoderFactory::MediaCodecVideoDecoderFactory() : 811 render_egl_context_(NULL) { 812 ALOGD << "MediaCodecVideoDecoderFactory ctor"; 813 JNIEnv* jni = AttachCurrentThreadIfNeeded(); 814 ScopedLocalRefFrame local_ref_frame(jni); 815 jclass j_decoder_class = FindClass(jni, "org/webrtc/MediaCodecVideoDecoder"); 816 supported_codec_types_.clear(); 817 818 bool is_vp8_hw_supported = jni->CallStaticBooleanMethod( 819 j_decoder_class, 820 GetStaticMethodID(jni, j_decoder_class, "isVp8HwSupported", "()Z")); 821 if (CheckException(jni)) { 822 is_vp8_hw_supported = false; 823 } 824 if (is_vp8_hw_supported) { 825 ALOGD << "VP8 HW Decoder supported."; 826 supported_codec_types_.push_back(kVideoCodecVP8); 827 } 828 829 bool is_vp9_hw_supported = jni->CallStaticBooleanMethod( 830 j_decoder_class, 831 GetStaticMethodID(jni, j_decoder_class, "isVp9HwSupported", "()Z")); 832 if (CheckException(jni)) { 833 is_vp9_hw_supported = false; 834 } 835 if (is_vp9_hw_supported) { 836 ALOGD << "VP9 HW Decoder supported."; 837 supported_codec_types_.push_back(kVideoCodecVP9); 838 } 839 840 bool is_h264_hw_supported = jni->CallStaticBooleanMethod( 841 j_decoder_class, 842 GetStaticMethodID(jni, j_decoder_class, "isH264HwSupported", "()Z")); 843 if (CheckException(jni)) { 844 is_h264_hw_supported = false; 845 } 846 if (is_h264_hw_supported) { 847 ALOGD << "H264 HW Decoder supported."; 848 supported_codec_types_.push_back(kVideoCodecH264); 849 } 850 } 851 852 MediaCodecVideoDecoderFactory::~MediaCodecVideoDecoderFactory() { 853 ALOGD << "MediaCodecVideoDecoderFactory dtor"; 854 if (render_egl_context_) { 855 JNIEnv* jni = AttachCurrentThreadIfNeeded(); 856 jni->DeleteGlobalRef(render_egl_context_); 857 render_egl_context_ = NULL; 858 } 859 } 860 861 void MediaCodecVideoDecoderFactory::SetEGLContext( 862 JNIEnv* jni, jobject render_egl_context) { 863 ALOGD << "MediaCodecVideoDecoderFactory::SetEGLContext"; 864 if (render_egl_context_) { 865 jni->DeleteGlobalRef(render_egl_context_); 866 render_egl_context_ = NULL; 867 } 868 if (!IsNull(jni, render_egl_context)) { 869 render_egl_context_ = jni->NewGlobalRef(render_egl_context); 870 if (CheckException(jni)) { 871 ALOGE << "error calling NewGlobalRef for EGL Context."; 872 render_egl_context_ = NULL; 873 } else { 874 jclass j_egl_context_class = 875 FindClass(jni, "org/webrtc/EglBase$Context"); 876 if (!jni->IsInstanceOf(render_egl_context_, j_egl_context_class)) { 877 ALOGE << "Wrong EGL Context."; 878 jni->DeleteGlobalRef(render_egl_context_); 879 render_egl_context_ = NULL; 880 } 881 } 882 } 883 if (render_egl_context_ == NULL) { 884 ALOGW << "NULL VideoDecoder EGL context - HW surface decoding is disabled."; 885 } 886 } 887 888 webrtc::VideoDecoder* MediaCodecVideoDecoderFactory::CreateVideoDecoder( 889 VideoCodecType type) { 890 if (supported_codec_types_.empty()) { 891 ALOGW << "No HW video decoder for type " << (int)type; 892 return NULL; 893 } 894 for (VideoCodecType codec_type : supported_codec_types_) { 895 if (codec_type == type) { 896 ALOGD << "Create HW video decoder for type " << (int)type; 897 return new MediaCodecVideoDecoder( 898 AttachCurrentThreadIfNeeded(), type, render_egl_context_); 899 } 900 } 901 ALOGW << "Can not find HW video decoder for type " << (int)type; 902 return NULL; 903 } 904 905 void MediaCodecVideoDecoderFactory::DestroyVideoDecoder( 906 webrtc::VideoDecoder* decoder) { 907 ALOGD << "Destroy video decoder."; 908 delete decoder; 909 } 910 911 const char* MediaCodecVideoDecoder::ImplementationName() const { 912 return "MediaCodec"; 913 } 914 915 } // namespace webrtc_jni 916 917
