1 /* 2 * libjingle 3 * Copyright 2015 Google Inc. 4 * 5 * Redistribution and use in source and binary forms, with or without 6 * modification, are permitted provided that the following conditions are met: 7 * 8 * 1. Redistributions of source code must retain the above copyright notice, 9 * this list of conditions and the following disclaimer. 10 * 2. Redistributions in binary form must reproduce the above copyright notice, 11 * this list of conditions and the following disclaimer in the documentation 12 * and/or other materials provided with the distribution. 13 * 3. The name of the author may not be used to endorse or promote products 14 * derived from this software without specific prior written permission. 15 * 16 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR IMPLIED 17 * WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF 18 * MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO 19 * EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 20 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, 21 * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; 22 * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, 23 * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR 24 * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF 25 * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 26 * 27 */ 28 29 #include "talk/app/webrtc/java/jni/androidmediaencoder_jni.h" 30 #include "talk/app/webrtc/java/jni/classreferenceholder.h" 31 #include "talk/app/webrtc/java/jni/androidmediacodeccommon.h" 32 #include "talk/app/webrtc/java/jni/native_handle_impl.h" 33 #include "webrtc/base/bind.h" 34 #include "webrtc/base/checks.h" 35 #include "webrtc/base/logging.h" 36 #include "webrtc/base/thread.h" 37 #include "webrtc/base/thread_checker.h" 38 #include "webrtc/modules/rtp_rtcp/source/h264_bitstream_parser.h" 39 #include "webrtc/modules/video_coding/include/video_codec_interface.h" 40 #include "webrtc/modules/video_coding/utility/quality_scaler.h" 41 #include "webrtc/modules/video_coding/utility/vp8_header_parser.h" 42 #include "webrtc/system_wrappers/include/field_trial.h" 43 #include "webrtc/system_wrappers/include/logcat_trace_context.h" 44 #include "third_party/libyuv/include/libyuv/convert.h" 45 #include "third_party/libyuv/include/libyuv/convert_from.h" 46 #include "third_party/libyuv/include/libyuv/video_common.h" 47 48 using rtc::Bind; 49 using rtc::Thread; 50 using rtc::ThreadManager; 51 using rtc::scoped_ptr; 52 53 using webrtc::CodecSpecificInfo; 54 using webrtc::EncodedImage; 55 using webrtc::VideoFrame; 56 using webrtc::RTPFragmentationHeader; 57 using webrtc::VideoCodec; 58 using webrtc::VideoCodecType; 59 using webrtc::kVideoCodecH264; 60 using webrtc::kVideoCodecVP8; 61 using webrtc::kVideoCodecVP9; 62 63 namespace webrtc_jni { 64 65 // H.264 start code length. 66 #define H264_SC_LENGTH 4 67 // Maximum allowed NALUs in one output frame. 68 #define MAX_NALUS_PERFRAME 32 69 // Maximum supported HW video encoder resolution. 70 #define MAX_VIDEO_WIDTH 1280 71 #define MAX_VIDEO_HEIGHT 1280 72 // Maximum supported HW video encoder fps. 73 #define MAX_VIDEO_FPS 30 74 75 // MediaCodecVideoEncoder is a webrtc::VideoEncoder implementation that uses 76 // Android's MediaCodec SDK API behind the scenes to implement (hopefully) 77 // HW-backed video encode. This C++ class is implemented as a very thin shim, 78 // delegating all of the interesting work to org.webrtc.MediaCodecVideoEncoder. 79 // MediaCodecVideoEncoder is created, operated, and destroyed on a single 80 // thread, currently the libjingle Worker thread. 81 class MediaCodecVideoEncoder : public webrtc::VideoEncoder, 82 public rtc::MessageHandler { 83 public: 84 virtual ~MediaCodecVideoEncoder(); 85 MediaCodecVideoEncoder(JNIEnv* jni, 86 VideoCodecType codecType, 87 jobject egl_context); 88 89 // webrtc::VideoEncoder implementation. Everything trampolines to 90 // |codec_thread_| for execution. 91 int32_t InitEncode(const webrtc::VideoCodec* codec_settings, 92 int32_t /* number_of_cores */, 93 size_t /* max_payload_size */) override; 94 int32_t Encode(const webrtc::VideoFrame& input_image, 95 const webrtc::CodecSpecificInfo* /* codec_specific_info */, 96 const std::vector<webrtc::FrameType>* frame_types) override; 97 int32_t RegisterEncodeCompleteCallback( 98 webrtc::EncodedImageCallback* callback) override; 99 int32_t Release() override; 100 int32_t SetChannelParameters(uint32_t /* packet_loss */, 101 int64_t /* rtt */) override; 102 int32_t SetRates(uint32_t new_bit_rate, uint32_t frame_rate) override; 103 104 // rtc::MessageHandler implementation. 105 void OnMessage(rtc::Message* msg) override; 106 107 void OnDroppedFrame() override; 108 109 int GetTargetFramerate() override; 110 111 bool SupportsNativeHandle() const override { return true; } 112 const char* ImplementationName() const override; 113 114 private: 115 // CHECK-fail if not running on |codec_thread_|. 116 void CheckOnCodecThread(); 117 118 private: 119 // ResetCodecOnCodecThread() calls ReleaseOnCodecThread() and 120 // InitEncodeOnCodecThread() in an attempt to restore the codec to an 121 // operable state. Necessary after all manner of OMX-layer errors. 122 bool ResetCodecOnCodecThread(); 123 124 // Implementation of webrtc::VideoEncoder methods above, all running on the 125 // codec thread exclusively. 126 // 127 // If width==0 then this is assumed to be a re-initialization and the 128 // previously-current values are reused instead of the passed parameters 129 // (makes it easier to reason about thread-safety). 130 int32_t InitEncodeOnCodecThread(int width, int height, int kbps, int fps, 131 bool use_surface); 132 // Reconfigure to match |frame| in width, height. Also reconfigures the 133 // encoder if |frame| is a texture/byte buffer and the encoder is initialized 134 // for byte buffer/texture. Returns false if reconfiguring fails. 135 bool MaybeReconfigureEncoderOnCodecThread(const webrtc::VideoFrame& frame); 136 int32_t EncodeOnCodecThread( 137 const webrtc::VideoFrame& input_image, 138 const std::vector<webrtc::FrameType>* frame_types); 139 bool EncodeByteBufferOnCodecThread(JNIEnv* jni, 140 bool key_frame, const webrtc::VideoFrame& frame, int input_buffer_index); 141 bool EncodeTextureOnCodecThread(JNIEnv* jni, 142 bool key_frame, const webrtc::VideoFrame& frame); 143 144 int32_t RegisterEncodeCompleteCallbackOnCodecThread( 145 webrtc::EncodedImageCallback* callback); 146 int32_t ReleaseOnCodecThread(); 147 int32_t SetRatesOnCodecThread(uint32_t new_bit_rate, uint32_t frame_rate); 148 149 // Helper accessors for MediaCodecVideoEncoder$OutputBufferInfo members. 150 int GetOutputBufferInfoIndex(JNIEnv* jni, jobject j_output_buffer_info); 151 jobject GetOutputBufferInfoBuffer(JNIEnv* jni, jobject j_output_buffer_info); 152 bool GetOutputBufferInfoIsKeyFrame(JNIEnv* jni, jobject j_output_buffer_info); 153 jlong GetOutputBufferInfoPresentationTimestampUs( 154 JNIEnv* jni, jobject j_output_buffer_info); 155 156 // Deliver any outputs pending in the MediaCodec to our |callback_| and return 157 // true on success. 158 bool DeliverPendingOutputs(JNIEnv* jni); 159 160 // Search for H.264 start codes. 161 int32_t NextNaluPosition(uint8_t *buffer, size_t buffer_size); 162 163 // Type of video codec. 164 VideoCodecType codecType_; 165 166 // Valid all the time since RegisterEncodeCompleteCallback() Invoke()s to 167 // |codec_thread_| synchronously. 168 webrtc::EncodedImageCallback* callback_; 169 170 // State that is constant for the lifetime of this object once the ctor 171 // returns. 172 scoped_ptr<Thread> codec_thread_; // Thread on which to operate MediaCodec. 173 rtc::ThreadChecker codec_thread_checker_; 174 ScopedGlobalRef<jclass> j_media_codec_video_encoder_class_; 175 ScopedGlobalRef<jobject> j_media_codec_video_encoder_; 176 jmethodID j_init_encode_method_; 177 jmethodID j_get_input_buffers_method_; 178 jmethodID j_dequeue_input_buffer_method_; 179 jmethodID j_encode_buffer_method_; 180 jmethodID j_encode_texture_method_; 181 jmethodID j_release_method_; 182 jmethodID j_set_rates_method_; 183 jmethodID j_dequeue_output_buffer_method_; 184 jmethodID j_release_output_buffer_method_; 185 jfieldID j_color_format_field_; 186 jfieldID j_info_index_field_; 187 jfieldID j_info_buffer_field_; 188 jfieldID j_info_is_key_frame_field_; 189 jfieldID j_info_presentation_timestamp_us_field_; 190 191 // State that is valid only between InitEncode() and the next Release(). 192 // Touched only on codec_thread_ so no explicit synchronization necessary. 193 int width_; // Frame width in pixels. 194 int height_; // Frame height in pixels. 195 bool inited_; 196 bool use_surface_; 197 uint16_t picture_id_; 198 enum libyuv::FourCC encoder_fourcc_; // Encoder color space format. 199 int last_set_bitrate_kbps_; // Last-requested bitrate in kbps. 200 int last_set_fps_; // Last-requested frame rate. 201 int64_t current_timestamp_us_; // Current frame timestamps in us. 202 int frames_received_; // Number of frames received by encoder. 203 int frames_encoded_; // Number of frames encoded by encoder. 204 int frames_dropped_; // Number of frames dropped by encoder. 205 int frames_in_queue_; // Number of frames in encoder queue. 206 int64_t start_time_ms_; // Start time for statistics. 207 int current_frames_; // Number of frames in the current statistics interval. 208 int current_bytes_; // Encoded bytes in the current statistics interval. 209 int current_encoding_time_ms_; // Overall encoding time in the current second 210 int64_t last_input_timestamp_ms_; // Timestamp of last received yuv frame. 211 int64_t last_output_timestamp_ms_; // Timestamp of last encoded frame. 212 std::vector<int32_t> timestamps_; // Video frames timestamp queue. 213 std::vector<int64_t> render_times_ms_; // Video frames render time queue. 214 std::vector<int64_t> frame_rtc_times_ms_; // Time when video frame is sent to 215 // encoder input. 216 int32_t output_timestamp_; // Last output frame timestamp from timestamps_ Q. 217 int64_t output_render_time_ms_; // Last output frame render time from 218 // render_times_ms_ queue. 219 // Frame size in bytes fed to MediaCodec. 220 int yuv_size_; 221 // True only when between a callback_->Encoded() call return a positive value 222 // and the next Encode() call being ignored. 223 bool drop_next_input_frame_; 224 // Global references; must be deleted in Release(). 225 std::vector<jobject> input_buffers_; 226 webrtc::QualityScaler quality_scaler_; 227 // Dynamic resolution change, off by default. 228 bool scale_; 229 230 // H264 bitstream parser, used to extract QP from encoded bitstreams. 231 webrtc::H264BitstreamParser h264_bitstream_parser_; 232 233 // VP9 variables to populate codec specific structure. 234 webrtc::GofInfoVP9 gof_; // Contains each frame's temporal information for 235 // non-flexible VP9 mode. 236 uint8_t tl0_pic_idx_; 237 size_t gof_idx_; 238 239 // EGL context - owned by factory, should not be allocated/destroyed 240 // by MediaCodecVideoEncoder. 241 jobject egl_context_; 242 }; 243 244 MediaCodecVideoEncoder::~MediaCodecVideoEncoder() { 245 // Call Release() to ensure no more callbacks to us after we are deleted. 246 Release(); 247 } 248 249 MediaCodecVideoEncoder::MediaCodecVideoEncoder( 250 JNIEnv* jni, VideoCodecType codecType, jobject egl_context) : 251 codecType_(codecType), 252 callback_(NULL), 253 codec_thread_(new Thread()), 254 j_media_codec_video_encoder_class_( 255 jni, 256 FindClass(jni, "org/webrtc/MediaCodecVideoEncoder")), 257 j_media_codec_video_encoder_( 258 jni, 259 jni->NewObject(*j_media_codec_video_encoder_class_, 260 GetMethodID(jni, 261 *j_media_codec_video_encoder_class_, 262 "<init>", 263 "()V"))), 264 inited_(false), 265 use_surface_(false), 266 picture_id_(0), 267 egl_context_(egl_context) { 268 ScopedLocalRefFrame local_ref_frame(jni); 269 // It would be nice to avoid spinning up a new thread per MediaCodec, and 270 // instead re-use e.g. the PeerConnectionFactory's |worker_thread_|, but bug 271 // 2732 means that deadlocks abound. This class synchronously trampolines 272 // to |codec_thread_|, so if anything else can be coming to _us_ from 273 // |codec_thread_|, or from any thread holding the |_sendCritSect| described 274 // in the bug, we have a problem. For now work around that with a dedicated 275 // thread. 276 codec_thread_->SetName("MediaCodecVideoEncoder", NULL); 277 RTC_CHECK(codec_thread_->Start()) << "Failed to start MediaCodecVideoEncoder"; 278 codec_thread_checker_.DetachFromThread(); 279 jclass j_output_buffer_info_class = 280 FindClass(jni, "org/webrtc/MediaCodecVideoEncoder$OutputBufferInfo"); 281 j_init_encode_method_ = GetMethodID( 282 jni, 283 *j_media_codec_video_encoder_class_, 284 "initEncode", 285 "(Lorg/webrtc/MediaCodecVideoEncoder$VideoCodecType;" 286 "IIIILorg/webrtc/EglBase14$Context;)Z"); 287 j_get_input_buffers_method_ = GetMethodID( 288 jni, 289 *j_media_codec_video_encoder_class_, 290 "getInputBuffers", 291 "()[Ljava/nio/ByteBuffer;"); 292 j_dequeue_input_buffer_method_ = GetMethodID( 293 jni, *j_media_codec_video_encoder_class_, "dequeueInputBuffer", "()I"); 294 j_encode_buffer_method_ = GetMethodID( 295 jni, *j_media_codec_video_encoder_class_, "encodeBuffer", "(ZIIJ)Z"); 296 j_encode_texture_method_ = GetMethodID( 297 jni, *j_media_codec_video_encoder_class_, "encodeTexture", 298 "(ZI[FJ)Z"); 299 j_release_method_ = 300 GetMethodID(jni, *j_media_codec_video_encoder_class_, "release", "()V"); 301 j_set_rates_method_ = GetMethodID( 302 jni, *j_media_codec_video_encoder_class_, "setRates", "(II)Z"); 303 j_dequeue_output_buffer_method_ = GetMethodID( 304 jni, 305 *j_media_codec_video_encoder_class_, 306 "dequeueOutputBuffer", 307 "()Lorg/webrtc/MediaCodecVideoEncoder$OutputBufferInfo;"); 308 j_release_output_buffer_method_ = GetMethodID( 309 jni, *j_media_codec_video_encoder_class_, "releaseOutputBuffer", "(I)Z"); 310 311 j_color_format_field_ = 312 GetFieldID(jni, *j_media_codec_video_encoder_class_, "colorFormat", "I"); 313 j_info_index_field_ = 314 GetFieldID(jni, j_output_buffer_info_class, "index", "I"); 315 j_info_buffer_field_ = GetFieldID( 316 jni, j_output_buffer_info_class, "buffer", "Ljava/nio/ByteBuffer;"); 317 j_info_is_key_frame_field_ = 318 GetFieldID(jni, j_output_buffer_info_class, "isKeyFrame", "Z"); 319 j_info_presentation_timestamp_us_field_ = GetFieldID( 320 jni, j_output_buffer_info_class, "presentationTimestampUs", "J"); 321 CHECK_EXCEPTION(jni) << "MediaCodecVideoEncoder ctor failed"; 322 srand(time(NULL)); 323 AllowBlockingCalls(); 324 } 325 326 int32_t MediaCodecVideoEncoder::InitEncode( 327 const webrtc::VideoCodec* codec_settings, 328 int32_t /* number_of_cores */, 329 size_t /* max_payload_size */) { 330 const int kMinWidth = 320; 331 const int kMinHeight = 180; 332 const int kLowQpThresholdDenominator = 3; 333 if (codec_settings == NULL) { 334 ALOGE << "NULL VideoCodec instance"; 335 return WEBRTC_VIDEO_CODEC_ERR_PARAMETER; 336 } 337 // Factory should guard against other codecs being used with us. 338 RTC_CHECK(codec_settings->codecType == codecType_) 339 << "Unsupported codec " << codec_settings->codecType << " for " 340 << codecType_; 341 342 ALOGD << "InitEncode request"; 343 scale_ = (codecType_ != kVideoCodecVP9) && (webrtc::field_trial::FindFullName( 344 "WebRTC-MediaCodecVideoEncoder-AutomaticResize") == "Enabled"); 345 ALOGD << "Encoder automatic resize " << (scale_ ? "enabled" : "disabled"); 346 if (scale_) { 347 if (codecType_ == kVideoCodecVP8) { 348 // QP is obtained from VP8-bitstream for HW, so the QP corresponds to the 349 // (internal) range: [0, 127]. And we cannot change QP_max in HW, so it is 350 // always = 127. Note that in SW, QP is that of the user-level range [0, 351 // 63]. 352 const int kMaxQp = 127; 353 // TODO(pbos): Investigate whether high-QP thresholds make sense for VP8. 354 // This effectively disables high QP as VP8 QP can't go above this 355 // threshold. 356 const int kDisabledBadQpThreshold = kMaxQp + 1; 357 quality_scaler_.Init(kMaxQp / kLowQpThresholdDenominator, 358 kDisabledBadQpThreshold, true); 359 } else if (codecType_ == kVideoCodecH264) { 360 // H264 QP is in the range [0, 51]. 361 const int kMaxQp = 51; 362 const int kBadQpThreshold = 40; 363 quality_scaler_.Init(kMaxQp / kLowQpThresholdDenominator, kBadQpThreshold, 364 false); 365 } else { 366 // When adding codec support to additional hardware codecs, also configure 367 // their QP thresholds for scaling. 368 RTC_NOTREACHED() << "Unsupported codec without configured QP thresholds."; 369 } 370 quality_scaler_.SetMinResolution(kMinWidth, kMinHeight); 371 quality_scaler_.ReportFramerate(codec_settings->maxFramerate); 372 } 373 return codec_thread_->Invoke<int32_t>( 374 Bind(&MediaCodecVideoEncoder::InitEncodeOnCodecThread, 375 this, 376 codec_settings->width, 377 codec_settings->height, 378 codec_settings->startBitrate, 379 codec_settings->maxFramerate, 380 false /* use_surface */)); 381 } 382 383 int32_t MediaCodecVideoEncoder::Encode( 384 const webrtc::VideoFrame& frame, 385 const webrtc::CodecSpecificInfo* /* codec_specific_info */, 386 const std::vector<webrtc::FrameType>* frame_types) { 387 return codec_thread_->Invoke<int32_t>(Bind( 388 &MediaCodecVideoEncoder::EncodeOnCodecThread, this, frame, frame_types)); 389 } 390 391 int32_t MediaCodecVideoEncoder::RegisterEncodeCompleteCallback( 392 webrtc::EncodedImageCallback* callback) { 393 return codec_thread_->Invoke<int32_t>( 394 Bind(&MediaCodecVideoEncoder::RegisterEncodeCompleteCallbackOnCodecThread, 395 this, 396 callback)); 397 } 398 399 int32_t MediaCodecVideoEncoder::Release() { 400 ALOGD << "EncoderRelease request"; 401 return codec_thread_->Invoke<int32_t>( 402 Bind(&MediaCodecVideoEncoder::ReleaseOnCodecThread, this)); 403 } 404 405 int32_t MediaCodecVideoEncoder::SetChannelParameters(uint32_t /* packet_loss */, 406 int64_t /* rtt */) { 407 return WEBRTC_VIDEO_CODEC_OK; 408 } 409 410 int32_t MediaCodecVideoEncoder::SetRates(uint32_t new_bit_rate, 411 uint32_t frame_rate) { 412 if (scale_) 413 quality_scaler_.ReportFramerate(frame_rate); 414 415 return codec_thread_->Invoke<int32_t>( 416 Bind(&MediaCodecVideoEncoder::SetRatesOnCodecThread, 417 this, 418 new_bit_rate, 419 frame_rate)); 420 } 421 422 void MediaCodecVideoEncoder::OnMessage(rtc::Message* msg) { 423 RTC_DCHECK(codec_thread_checker_.CalledOnValidThread()); 424 JNIEnv* jni = AttachCurrentThreadIfNeeded(); 425 ScopedLocalRefFrame local_ref_frame(jni); 426 427 // We only ever send one message to |this| directly (not through a Bind()'d 428 // functor), so expect no ID/data. 429 RTC_CHECK(!msg->message_id) << "Unexpected message!"; 430 RTC_CHECK(!msg->pdata) << "Unexpected message!"; 431 if (!inited_) { 432 return; 433 } 434 435 // It would be nice to recover from a failure here if one happened, but it's 436 // unclear how to signal such a failure to the app, so instead we stay silent 437 // about it and let the next app-called API method reveal the borkedness. 438 DeliverPendingOutputs(jni); 439 codec_thread_->PostDelayed(kMediaCodecPollMs, this); 440 } 441 442 bool MediaCodecVideoEncoder::ResetCodecOnCodecThread() { 443 RTC_DCHECK(codec_thread_checker_.CalledOnValidThread()); 444 ALOGE << "ResetOnCodecThread"; 445 if (ReleaseOnCodecThread() != WEBRTC_VIDEO_CODEC_OK || 446 InitEncodeOnCodecThread(width_, height_, 0, 0, false) != 447 WEBRTC_VIDEO_CODEC_OK) { 448 // TODO(fischman): wouldn't it be nice if there was a way to gracefully 449 // degrade to a SW encoder at this point? There isn't one AFAICT :( 450 // https://code.google.com/p/webrtc/issues/detail?id=2920 451 return false; 452 } 453 return true; 454 } 455 456 int32_t MediaCodecVideoEncoder::InitEncodeOnCodecThread( 457 int width, int height, int kbps, int fps, bool use_surface) { 458 RTC_DCHECK(codec_thread_checker_.CalledOnValidThread()); 459 RTC_CHECK(!use_surface || egl_context_ != nullptr) << "EGL context not set."; 460 JNIEnv* jni = AttachCurrentThreadIfNeeded(); 461 ScopedLocalRefFrame local_ref_frame(jni); 462 463 ALOGD << "InitEncodeOnCodecThread Type: " << (int)codecType_ << ", " << 464 width << " x " << height << ". Bitrate: " << kbps << 465 " kbps. Fps: " << fps; 466 if (kbps == 0) { 467 kbps = last_set_bitrate_kbps_; 468 } 469 if (fps == 0) { 470 fps = last_set_fps_; 471 } 472 473 width_ = width; 474 height_ = height; 475 last_set_bitrate_kbps_ = kbps; 476 last_set_fps_ = fps; 477 yuv_size_ = width_ * height_ * 3 / 2; 478 frames_received_ = 0; 479 frames_encoded_ = 0; 480 frames_dropped_ = 0; 481 frames_in_queue_ = 0; 482 current_timestamp_us_ = 0; 483 start_time_ms_ = GetCurrentTimeMs(); 484 current_frames_ = 0; 485 current_bytes_ = 0; 486 current_encoding_time_ms_ = 0; 487 last_input_timestamp_ms_ = -1; 488 last_output_timestamp_ms_ = -1; 489 output_timestamp_ = 0; 490 output_render_time_ms_ = 0; 491 timestamps_.clear(); 492 render_times_ms_.clear(); 493 frame_rtc_times_ms_.clear(); 494 drop_next_input_frame_ = false; 495 use_surface_ = use_surface; 496 picture_id_ = static_cast<uint16_t>(rand()) & 0x7FFF; 497 gof_.SetGofInfoVP9(webrtc::TemporalStructureMode::kTemporalStructureMode1); 498 tl0_pic_idx_ = static_cast<uint8_t>(rand()); 499 gof_idx_ = 0; 500 501 // We enforce no extra stride/padding in the format creation step. 502 jobject j_video_codec_enum = JavaEnumFromIndex( 503 jni, "MediaCodecVideoEncoder$VideoCodecType", codecType_); 504 const bool encode_status = jni->CallBooleanMethod( 505 *j_media_codec_video_encoder_, j_init_encode_method_, 506 j_video_codec_enum, width, height, kbps, fps, 507 (use_surface ? egl_context_ : nullptr)); 508 if (!encode_status) { 509 ALOGE << "Failed to configure encoder."; 510 return WEBRTC_VIDEO_CODEC_ERROR; 511 } 512 CHECK_EXCEPTION(jni); 513 514 if (!use_surface) { 515 jobjectArray input_buffers = reinterpret_cast<jobjectArray>( 516 jni->CallObjectMethod(*j_media_codec_video_encoder_, 517 j_get_input_buffers_method_)); 518 CHECK_EXCEPTION(jni); 519 if (IsNull(jni, input_buffers)) { 520 return WEBRTC_VIDEO_CODEC_ERROR; 521 } 522 523 switch (GetIntField(jni, *j_media_codec_video_encoder_, 524 j_color_format_field_)) { 525 case COLOR_FormatYUV420Planar: 526 encoder_fourcc_ = libyuv::FOURCC_YU12; 527 break; 528 case COLOR_FormatYUV420SemiPlanar: 529 case COLOR_QCOM_FormatYUV420SemiPlanar: 530 case COLOR_QCOM_FORMATYUV420PackedSemiPlanar32m: 531 encoder_fourcc_ = libyuv::FOURCC_NV12; 532 break; 533 default: 534 LOG(LS_ERROR) << "Wrong color format."; 535 return WEBRTC_VIDEO_CODEC_ERROR; 536 } 537 size_t num_input_buffers = jni->GetArrayLength(input_buffers); 538 RTC_CHECK(input_buffers_.empty()) 539 << "Unexpected double InitEncode without Release"; 540 input_buffers_.resize(num_input_buffers); 541 for (size_t i = 0; i < num_input_buffers; ++i) { 542 input_buffers_[i] = 543 jni->NewGlobalRef(jni->GetObjectArrayElement(input_buffers, i)); 544 int64_t yuv_buffer_capacity = 545 jni->GetDirectBufferCapacity(input_buffers_[i]); 546 CHECK_EXCEPTION(jni); 547 RTC_CHECK(yuv_buffer_capacity >= yuv_size_) << "Insufficient capacity"; 548 } 549 } 550 551 inited_ = true; 552 codec_thread_->PostDelayed(kMediaCodecPollMs, this); 553 return WEBRTC_VIDEO_CODEC_OK; 554 } 555 556 int32_t MediaCodecVideoEncoder::EncodeOnCodecThread( 557 const webrtc::VideoFrame& frame, 558 const std::vector<webrtc::FrameType>* frame_types) { 559 RTC_DCHECK(codec_thread_checker_.CalledOnValidThread()); 560 JNIEnv* jni = AttachCurrentThreadIfNeeded(); 561 ScopedLocalRefFrame local_ref_frame(jni); 562 563 if (!inited_) { 564 return WEBRTC_VIDEO_CODEC_UNINITIALIZED; 565 } 566 567 frames_received_++; 568 if (!DeliverPendingOutputs(jni)) { 569 if (!ResetCodecOnCodecThread()) 570 return WEBRTC_VIDEO_CODEC_ERROR; 571 } 572 573 if (drop_next_input_frame_) { 574 ALOGW << "Encoder drop frame - failed callback."; 575 drop_next_input_frame_ = false; 576 return WEBRTC_VIDEO_CODEC_OK; 577 } 578 579 RTC_CHECK(frame_types->size() == 1) << "Unexpected stream count"; 580 581 VideoFrame input_frame = frame; 582 if (scale_) { 583 // Check framerate before spatial resolution change. 584 quality_scaler_.OnEncodeFrame(frame); 585 const webrtc::QualityScaler::Resolution scaled_resolution = 586 quality_scaler_.GetScaledResolution(); 587 if (scaled_resolution.width != frame.width() || 588 scaled_resolution.height != frame.height()) { 589 if (frame.native_handle() != nullptr) { 590 rtc::scoped_refptr<webrtc::VideoFrameBuffer> scaled_buffer( 591 static_cast<AndroidTextureBuffer*>( 592 frame.video_frame_buffer().get())->ScaleAndRotate( 593 scaled_resolution.width, 594 scaled_resolution.height, 595 webrtc::kVideoRotation_0)); 596 input_frame.set_video_frame_buffer(scaled_buffer); 597 } else { 598 input_frame = quality_scaler_.GetScaledFrame(frame); 599 } 600 } 601 } 602 603 if (!MaybeReconfigureEncoderOnCodecThread(input_frame)) { 604 ALOGE << "Failed to reconfigure encoder."; 605 return WEBRTC_VIDEO_CODEC_ERROR; 606 } 607 608 // Check if we accumulated too many frames in encoder input buffers 609 // or the encoder latency exceeds 70 ms and drop frame if so. 610 if (frames_in_queue_ > 0 && last_input_timestamp_ms_ >= 0) { 611 int encoder_latency_ms = last_input_timestamp_ms_ - 612 last_output_timestamp_ms_; 613 if (frames_in_queue_ > 2 || encoder_latency_ms > 70) { 614 ALOGD << "Drop frame - encoder is behind by " << encoder_latency_ms << 615 " ms. Q size: " << frames_in_queue_; 616 frames_dropped_++; 617 // Report dropped frame to quality_scaler_. 618 OnDroppedFrame(); 619 return WEBRTC_VIDEO_CODEC_OK; 620 } 621 } 622 623 const bool key_frame = frame_types->front() != webrtc::kVideoFrameDelta; 624 bool encode_status = true; 625 if (!input_frame.native_handle()) { 626 int j_input_buffer_index = jni->CallIntMethod(*j_media_codec_video_encoder_, 627 j_dequeue_input_buffer_method_); 628 CHECK_EXCEPTION(jni); 629 if (j_input_buffer_index == -1) { 630 // Video codec falls behind - no input buffer available. 631 ALOGW << "Encoder drop frame - no input buffers available"; 632 frames_dropped_++; 633 // Report dropped frame to quality_scaler_. 634 OnDroppedFrame(); 635 return WEBRTC_VIDEO_CODEC_OK; // TODO(fischman): see webrtc bug 2887. 636 } 637 if (j_input_buffer_index == -2) { 638 ResetCodecOnCodecThread(); 639 return WEBRTC_VIDEO_CODEC_ERROR; 640 } 641 encode_status = EncodeByteBufferOnCodecThread(jni, key_frame, input_frame, 642 j_input_buffer_index); 643 } else { 644 encode_status = EncodeTextureOnCodecThread(jni, key_frame, input_frame); 645 } 646 647 if (!encode_status) { 648 ALOGE << "Failed encode frame with timestamp: " << input_frame.timestamp(); 649 ResetCodecOnCodecThread(); 650 return WEBRTC_VIDEO_CODEC_ERROR; 651 } 652 653 last_input_timestamp_ms_ = 654 current_timestamp_us_ / rtc::kNumMicrosecsPerMillisec; 655 frames_in_queue_++; 656 657 // Save input image timestamps for later output 658 timestamps_.push_back(input_frame.timestamp()); 659 render_times_ms_.push_back(input_frame.render_time_ms()); 660 frame_rtc_times_ms_.push_back(GetCurrentTimeMs()); 661 current_timestamp_us_ += rtc::kNumMicrosecsPerSec / last_set_fps_; 662 663 if (!DeliverPendingOutputs(jni)) { 664 ALOGE << "Failed deliver pending outputs."; 665 ResetCodecOnCodecThread(); 666 return WEBRTC_VIDEO_CODEC_ERROR; 667 } 668 return WEBRTC_VIDEO_CODEC_OK; 669 } 670 671 bool MediaCodecVideoEncoder::MaybeReconfigureEncoderOnCodecThread( 672 const webrtc::VideoFrame& frame) { 673 RTC_DCHECK(codec_thread_checker_.CalledOnValidThread()); 674 675 const bool is_texture_frame = frame.native_handle() != nullptr; 676 const bool reconfigure_due_to_format = is_texture_frame != use_surface_; 677 const bool reconfigure_due_to_size = 678 frame.width() != width_ || frame.height() != height_; 679 680 if (reconfigure_due_to_format) { 681 ALOGD << "Reconfigure encoder due to format change. " 682 << (use_surface_ ? 683 "Reconfiguring to encode from byte buffer." : 684 "Reconfiguring to encode from texture."); 685 } 686 if (reconfigure_due_to_size) { 687 ALOGD << "Reconfigure encoder due to frame resolution change from " 688 << width_ << " x " << height_ << " to " << frame.width() << " x " 689 << frame.height(); 690 width_ = frame.width(); 691 height_ = frame.height(); 692 } 693 694 if (!reconfigure_due_to_format && !reconfigure_due_to_size) 695 return true; 696 697 ReleaseOnCodecThread(); 698 699 return InitEncodeOnCodecThread(width_, height_, 0, 0 , is_texture_frame) == 700 WEBRTC_VIDEO_CODEC_OK; 701 } 702 703 bool MediaCodecVideoEncoder::EncodeByteBufferOnCodecThread(JNIEnv* jni, 704 bool key_frame, const webrtc::VideoFrame& frame, int input_buffer_index) { 705 RTC_DCHECK(codec_thread_checker_.CalledOnValidThread()); 706 RTC_CHECK(!use_surface_); 707 708 ALOGV("Encoder frame in # %d. TS: %lld. Q: %d", 709 frames_received_ - 1, current_timestamp_us_ / 1000, frames_in_queue_); 710 711 jobject j_input_buffer = input_buffers_[input_buffer_index]; 712 uint8_t* yuv_buffer = 713 reinterpret_cast<uint8_t*>(jni->GetDirectBufferAddress(j_input_buffer)); 714 CHECK_EXCEPTION(jni); 715 RTC_CHECK(yuv_buffer) << "Indirect buffer??"; 716 RTC_CHECK(!libyuv::ConvertFromI420( 717 frame.buffer(webrtc::kYPlane), frame.stride(webrtc::kYPlane), 718 frame.buffer(webrtc::kUPlane), frame.stride(webrtc::kUPlane), 719 frame.buffer(webrtc::kVPlane), frame.stride(webrtc::kVPlane), 720 yuv_buffer, width_, width_, height_, encoder_fourcc_)) 721 << "ConvertFromI420 failed"; 722 723 bool encode_status = jni->CallBooleanMethod(*j_media_codec_video_encoder_, 724 j_encode_buffer_method_, 725 key_frame, 726 input_buffer_index, 727 yuv_size_, 728 current_timestamp_us_); 729 CHECK_EXCEPTION(jni); 730 return encode_status; 731 } 732 733 bool MediaCodecVideoEncoder::EncodeTextureOnCodecThread(JNIEnv* jni, 734 bool key_frame, const webrtc::VideoFrame& frame) { 735 RTC_DCHECK(codec_thread_checker_.CalledOnValidThread()); 736 RTC_CHECK(use_surface_); 737 NativeHandleImpl* handle = 738 static_cast<NativeHandleImpl*>(frame.native_handle()); 739 jfloatArray sampling_matrix = jni->NewFloatArray(16); 740 jni->SetFloatArrayRegion(sampling_matrix, 0, 16, handle->sampling_matrix); 741 742 bool encode_status = jni->CallBooleanMethod(*j_media_codec_video_encoder_, 743 j_encode_texture_method_, 744 key_frame, 745 handle->oes_texture_id, 746 sampling_matrix, 747 current_timestamp_us_); 748 CHECK_EXCEPTION(jni); 749 return encode_status; 750 } 751 752 int32_t MediaCodecVideoEncoder::RegisterEncodeCompleteCallbackOnCodecThread( 753 webrtc::EncodedImageCallback* callback) { 754 RTC_DCHECK(codec_thread_checker_.CalledOnValidThread()); 755 JNIEnv* jni = AttachCurrentThreadIfNeeded(); 756 ScopedLocalRefFrame local_ref_frame(jni); 757 callback_ = callback; 758 return WEBRTC_VIDEO_CODEC_OK; 759 } 760 761 int32_t MediaCodecVideoEncoder::ReleaseOnCodecThread() { 762 RTC_DCHECK(codec_thread_checker_.CalledOnValidThread()); 763 if (!inited_) { 764 return WEBRTC_VIDEO_CODEC_OK; 765 } 766 JNIEnv* jni = AttachCurrentThreadIfNeeded(); 767 ALOGD << "EncoderReleaseOnCodecThread: Frames received: " << 768 frames_received_ << ". Encoded: " << frames_encoded_ << 769 ". Dropped: " << frames_dropped_; 770 ScopedLocalRefFrame local_ref_frame(jni); 771 for (size_t i = 0; i < input_buffers_.size(); ++i) 772 jni->DeleteGlobalRef(input_buffers_[i]); 773 input_buffers_.clear(); 774 jni->CallVoidMethod(*j_media_codec_video_encoder_, j_release_method_); 775 CHECK_EXCEPTION(jni); 776 rtc::MessageQueueManager::Clear(this); 777 inited_ = false; 778 use_surface_ = false; 779 ALOGD << "EncoderReleaseOnCodecThread done."; 780 return WEBRTC_VIDEO_CODEC_OK; 781 } 782 783 int32_t MediaCodecVideoEncoder::SetRatesOnCodecThread(uint32_t new_bit_rate, 784 uint32_t frame_rate) { 785 RTC_DCHECK(codec_thread_checker_.CalledOnValidThread()); 786 if (last_set_bitrate_kbps_ == new_bit_rate && 787 last_set_fps_ == frame_rate) { 788 return WEBRTC_VIDEO_CODEC_OK; 789 } 790 JNIEnv* jni = AttachCurrentThreadIfNeeded(); 791 ScopedLocalRefFrame local_ref_frame(jni); 792 if (new_bit_rate > 0) { 793 last_set_bitrate_kbps_ = new_bit_rate; 794 } 795 if (frame_rate > 0) { 796 last_set_fps_ = frame_rate; 797 } 798 bool ret = jni->CallBooleanMethod(*j_media_codec_video_encoder_, 799 j_set_rates_method_, 800 last_set_bitrate_kbps_, 801 last_set_fps_); 802 CHECK_EXCEPTION(jni); 803 if (!ret) { 804 ResetCodecOnCodecThread(); 805 return WEBRTC_VIDEO_CODEC_ERROR; 806 } 807 return WEBRTC_VIDEO_CODEC_OK; 808 } 809 810 int MediaCodecVideoEncoder::GetOutputBufferInfoIndex( 811 JNIEnv* jni, 812 jobject j_output_buffer_info) { 813 return GetIntField(jni, j_output_buffer_info, j_info_index_field_); 814 } 815 816 jobject MediaCodecVideoEncoder::GetOutputBufferInfoBuffer( 817 JNIEnv* jni, 818 jobject j_output_buffer_info) { 819 return GetObjectField(jni, j_output_buffer_info, j_info_buffer_field_); 820 } 821 822 bool MediaCodecVideoEncoder::GetOutputBufferInfoIsKeyFrame( 823 JNIEnv* jni, 824 jobject j_output_buffer_info) { 825 return GetBooleanField(jni, j_output_buffer_info, j_info_is_key_frame_field_); 826 } 827 828 jlong MediaCodecVideoEncoder::GetOutputBufferInfoPresentationTimestampUs( 829 JNIEnv* jni, 830 jobject j_output_buffer_info) { 831 return GetLongField( 832 jni, j_output_buffer_info, j_info_presentation_timestamp_us_field_); 833 } 834 835 bool MediaCodecVideoEncoder::DeliverPendingOutputs(JNIEnv* jni) { 836 RTC_DCHECK(codec_thread_checker_.CalledOnValidThread()); 837 while (true) { 838 jobject j_output_buffer_info = jni->CallObjectMethod( 839 *j_media_codec_video_encoder_, j_dequeue_output_buffer_method_); 840 CHECK_EXCEPTION(jni); 841 if (IsNull(jni, j_output_buffer_info)) { 842 break; 843 } 844 845 int output_buffer_index = 846 GetOutputBufferInfoIndex(jni, j_output_buffer_info); 847 if (output_buffer_index == -1) { 848 ResetCodecOnCodecThread(); 849 return false; 850 } 851 852 // Get key and config frame flags. 853 jobject j_output_buffer = 854 GetOutputBufferInfoBuffer(jni, j_output_buffer_info); 855 bool key_frame = GetOutputBufferInfoIsKeyFrame(jni, j_output_buffer_info); 856 857 // Get frame timestamps from a queue - for non config frames only. 858 int64_t frame_encoding_time_ms = 0; 859 last_output_timestamp_ms_ = 860 GetOutputBufferInfoPresentationTimestampUs(jni, j_output_buffer_info) / 861 1000; 862 if (frames_in_queue_ > 0) { 863 output_timestamp_ = timestamps_.front(); 864 timestamps_.erase(timestamps_.begin()); 865 output_render_time_ms_ = render_times_ms_.front(); 866 render_times_ms_.erase(render_times_ms_.begin()); 867 frame_encoding_time_ms = GetCurrentTimeMs() - frame_rtc_times_ms_.front(); 868 frame_rtc_times_ms_.erase(frame_rtc_times_ms_.begin()); 869 frames_in_queue_--; 870 } 871 872 // Extract payload. 873 size_t payload_size = jni->GetDirectBufferCapacity(j_output_buffer); 874 uint8_t* payload = reinterpret_cast<uint8_t*>( 875 jni->GetDirectBufferAddress(j_output_buffer)); 876 CHECK_EXCEPTION(jni); 877 878 ALOGV("Encoder frame out # %d. Key: %d. Size: %d. TS: %lld." 879 " Latency: %lld. EncTime: %lld", 880 frames_encoded_, key_frame, payload_size, 881 last_output_timestamp_ms_, 882 last_input_timestamp_ms_ - last_output_timestamp_ms_, 883 frame_encoding_time_ms); 884 885 // Calculate and print encoding statistics - every 3 seconds. 886 frames_encoded_++; 887 current_frames_++; 888 current_bytes_ += payload_size; 889 current_encoding_time_ms_ += frame_encoding_time_ms; 890 int statistic_time_ms = GetCurrentTimeMs() - start_time_ms_; 891 if (statistic_time_ms >= kMediaCodecStatisticsIntervalMs && 892 current_frames_ > 0) { 893 ALOGD << "Encoded frames: " << frames_encoded_ << ". Bitrate: " << 894 (current_bytes_ * 8 / statistic_time_ms) << 895 ", target: " << last_set_bitrate_kbps_ << " kbps, fps: " << 896 ((current_frames_ * 1000 + statistic_time_ms / 2) / statistic_time_ms) 897 << ", encTime: " << 898 (current_encoding_time_ms_ / current_frames_) << " for last " << 899 statistic_time_ms << " ms."; 900 start_time_ms_ = GetCurrentTimeMs(); 901 current_frames_ = 0; 902 current_bytes_ = 0; 903 current_encoding_time_ms_ = 0; 904 } 905 906 // Callback - return encoded frame. 907 int32_t callback_status = 0; 908 if (callback_) { 909 scoped_ptr<webrtc::EncodedImage> image( 910 new webrtc::EncodedImage(payload, payload_size, payload_size)); 911 image->_encodedWidth = width_; 912 image->_encodedHeight = height_; 913 image->_timeStamp = output_timestamp_; 914 image->capture_time_ms_ = output_render_time_ms_; 915 image->_frameType = 916 (key_frame ? webrtc::kVideoFrameKey : webrtc::kVideoFrameDelta); 917 image->_completeFrame = true; 918 image->adapt_reason_.quality_resolution_downscales = 919 scale_ ? quality_scaler_.downscale_shift() : -1; 920 921 webrtc::CodecSpecificInfo info; 922 memset(&info, 0, sizeof(info)); 923 info.codecType = codecType_; 924 if (codecType_ == kVideoCodecVP8) { 925 info.codecSpecific.VP8.pictureId = picture_id_; 926 info.codecSpecific.VP8.nonReference = false; 927 info.codecSpecific.VP8.simulcastIdx = 0; 928 info.codecSpecific.VP8.temporalIdx = webrtc::kNoTemporalIdx; 929 info.codecSpecific.VP8.layerSync = false; 930 info.codecSpecific.VP8.tl0PicIdx = webrtc::kNoTl0PicIdx; 931 info.codecSpecific.VP8.keyIdx = webrtc::kNoKeyIdx; 932 } else if (codecType_ == kVideoCodecVP9) { 933 if (key_frame) { 934 gof_idx_ = 0; 935 } 936 info.codecSpecific.VP9.picture_id = picture_id_; 937 info.codecSpecific.VP9.inter_pic_predicted = key_frame ? false : true; 938 info.codecSpecific.VP9.flexible_mode = false; 939 info.codecSpecific.VP9.ss_data_available = key_frame ? true : false; 940 info.codecSpecific.VP9.tl0_pic_idx = tl0_pic_idx_++; 941 info.codecSpecific.VP9.temporal_idx = webrtc::kNoTemporalIdx; 942 info.codecSpecific.VP9.spatial_idx = webrtc::kNoSpatialIdx; 943 info.codecSpecific.VP9.temporal_up_switch = true; 944 info.codecSpecific.VP9.inter_layer_predicted = false; 945 info.codecSpecific.VP9.gof_idx = 946 static_cast<uint8_t>(gof_idx_++ % gof_.num_frames_in_gof); 947 info.codecSpecific.VP9.num_spatial_layers = 1; 948 info.codecSpecific.VP9.spatial_layer_resolution_present = false; 949 if (info.codecSpecific.VP9.ss_data_available) { 950 info.codecSpecific.VP9.spatial_layer_resolution_present = true; 951 info.codecSpecific.VP9.width[0] = width_; 952 info.codecSpecific.VP9.height[0] = height_; 953 info.codecSpecific.VP9.gof.CopyGofInfoVP9(gof_); 954 } 955 } 956 picture_id_ = (picture_id_ + 1) & 0x7FFF; 957 958 // Generate a header describing a single fragment. 959 webrtc::RTPFragmentationHeader header; 960 memset(&header, 0, sizeof(header)); 961 if (codecType_ == kVideoCodecVP8 || codecType_ == kVideoCodecVP9) { 962 header.VerifyAndAllocateFragmentationHeader(1); 963 header.fragmentationOffset[0] = 0; 964 header.fragmentationLength[0] = image->_length; 965 header.fragmentationPlType[0] = 0; 966 header.fragmentationTimeDiff[0] = 0; 967 if (codecType_ == kVideoCodecVP8 && scale_) { 968 int qp; 969 if (webrtc::vp8::GetQp(payload, payload_size, &qp)) 970 quality_scaler_.ReportQP(qp); 971 } 972 } else if (codecType_ == kVideoCodecH264) { 973 if (scale_) { 974 h264_bitstream_parser_.ParseBitstream(payload, payload_size); 975 int qp; 976 if (h264_bitstream_parser_.GetLastSliceQp(&qp)) 977 quality_scaler_.ReportQP(qp); 978 } 979 // For H.264 search for start codes. 980 int32_t scPositions[MAX_NALUS_PERFRAME + 1] = {}; 981 int32_t scPositionsLength = 0; 982 int32_t scPosition = 0; 983 while (scPositionsLength < MAX_NALUS_PERFRAME) { 984 int32_t naluPosition = NextNaluPosition( 985 payload + scPosition, payload_size - scPosition); 986 if (naluPosition < 0) { 987 break; 988 } 989 scPosition += naluPosition; 990 scPositions[scPositionsLength++] = scPosition; 991 scPosition += H264_SC_LENGTH; 992 } 993 if (scPositionsLength == 0) { 994 ALOGE << "Start code is not found!"; 995 ALOGE << "Data:" << image->_buffer[0] << " " << image->_buffer[1] 996 << " " << image->_buffer[2] << " " << image->_buffer[3] 997 << " " << image->_buffer[4] << " " << image->_buffer[5]; 998 ResetCodecOnCodecThread(); 999 return false; 1000 } 1001 scPositions[scPositionsLength] = payload_size; 1002 header.VerifyAndAllocateFragmentationHeader(scPositionsLength); 1003 for (size_t i = 0; i < scPositionsLength; i++) { 1004 header.fragmentationOffset[i] = scPositions[i] + H264_SC_LENGTH; 1005 header.fragmentationLength[i] = 1006 scPositions[i + 1] - header.fragmentationOffset[i]; 1007 header.fragmentationPlType[i] = 0; 1008 header.fragmentationTimeDiff[i] = 0; 1009 } 1010 } 1011 1012 callback_status = callback_->Encoded(*image, &info, &header); 1013 } 1014 1015 // Return output buffer back to the encoder. 1016 bool success = jni->CallBooleanMethod(*j_media_codec_video_encoder_, 1017 j_release_output_buffer_method_, 1018 output_buffer_index); 1019 CHECK_EXCEPTION(jni); 1020 if (!success) { 1021 ResetCodecOnCodecThread(); 1022 return false; 1023 } 1024 1025 if (callback_status > 0) { 1026 drop_next_input_frame_ = true; 1027 // Theoretically could handle callback_status<0 here, but unclear what 1028 // that would mean for us. 1029 } 1030 } 1031 1032 return true; 1033 } 1034 1035 int32_t MediaCodecVideoEncoder::NextNaluPosition( 1036 uint8_t *buffer, size_t buffer_size) { 1037 if (buffer_size < H264_SC_LENGTH) { 1038 return -1; 1039 } 1040 uint8_t *head = buffer; 1041 // Set end buffer pointer to 4 bytes before actual buffer end so we can 1042 // access head[1], head[2] and head[3] in a loop without buffer overrun. 1043 uint8_t *end = buffer + buffer_size - H264_SC_LENGTH; 1044 1045 while (head < end) { 1046 if (head[0]) { 1047 head++; 1048 continue; 1049 } 1050 if (head[1]) { // got 00xx 1051 head += 2; 1052 continue; 1053 } 1054 if (head[2]) { // got 0000xx 1055 head += 3; 1056 continue; 1057 } 1058 if (head[3] != 0x01) { // got 000000xx 1059 head++; // xx != 1, continue searching. 1060 continue; 1061 } 1062 return (int32_t)(head - buffer); 1063 } 1064 return -1; 1065 } 1066 1067 void MediaCodecVideoEncoder::OnDroppedFrame() { 1068 if (scale_) 1069 quality_scaler_.ReportDroppedFrame(); 1070 } 1071 1072 int MediaCodecVideoEncoder::GetTargetFramerate() { 1073 return scale_ ? quality_scaler_.GetTargetFramerate() : -1; 1074 } 1075 1076 const char* MediaCodecVideoEncoder::ImplementationName() const { 1077 return "MediaCodec"; 1078 } 1079 1080 MediaCodecVideoEncoderFactory::MediaCodecVideoEncoderFactory() 1081 : egl_context_(nullptr) { 1082 JNIEnv* jni = AttachCurrentThreadIfNeeded(); 1083 ScopedLocalRefFrame local_ref_frame(jni); 1084 jclass j_encoder_class = FindClass(jni, "org/webrtc/MediaCodecVideoEncoder"); 1085 supported_codecs_.clear(); 1086 1087 bool is_vp8_hw_supported = jni->CallStaticBooleanMethod( 1088 j_encoder_class, 1089 GetStaticMethodID(jni, j_encoder_class, "isVp8HwSupported", "()Z")); 1090 CHECK_EXCEPTION(jni); 1091 if (is_vp8_hw_supported) { 1092 ALOGD << "VP8 HW Encoder supported."; 1093 supported_codecs_.push_back(VideoCodec(kVideoCodecVP8, "VP8", 1094 MAX_VIDEO_WIDTH, MAX_VIDEO_HEIGHT, MAX_VIDEO_FPS)); 1095 } 1096 1097 bool is_vp9_hw_supported = jni->CallStaticBooleanMethod( 1098 j_encoder_class, 1099 GetStaticMethodID(jni, j_encoder_class, "isVp9HwSupported", "()Z")); 1100 CHECK_EXCEPTION(jni); 1101 if (is_vp9_hw_supported) { 1102 ALOGD << "VP9 HW Encoder supported."; 1103 supported_codecs_.push_back(VideoCodec(kVideoCodecVP9, "VP9", 1104 MAX_VIDEO_WIDTH, MAX_VIDEO_HEIGHT, MAX_VIDEO_FPS)); 1105 } 1106 1107 bool is_h264_hw_supported = jni->CallStaticBooleanMethod( 1108 j_encoder_class, 1109 GetStaticMethodID(jni, j_encoder_class, "isH264HwSupported", "()Z")); 1110 CHECK_EXCEPTION(jni); 1111 if (is_h264_hw_supported) { 1112 ALOGD << "H.264 HW Encoder supported."; 1113 supported_codecs_.push_back(VideoCodec(kVideoCodecH264, "H264", 1114 MAX_VIDEO_WIDTH, MAX_VIDEO_HEIGHT, MAX_VIDEO_FPS)); 1115 } 1116 } 1117 1118 MediaCodecVideoEncoderFactory::~MediaCodecVideoEncoderFactory() {} 1119 1120 void MediaCodecVideoEncoderFactory::SetEGLContext( 1121 JNIEnv* jni, jobject render_egl_context) { 1122 ALOGD << "MediaCodecVideoEncoderFactory::SetEGLContext"; 1123 if (egl_context_) { 1124 jni->DeleteGlobalRef(egl_context_); 1125 egl_context_ = NULL; 1126 } 1127 if (!IsNull(jni, render_egl_context)) { 1128 egl_context_ = jni->NewGlobalRef(render_egl_context); 1129 if (CheckException(jni)) { 1130 ALOGE << "error calling NewGlobalRef for EGL Context."; 1131 egl_context_ = NULL; 1132 } else { 1133 jclass j_egl_context_class = 1134 FindClass(jni, "org/webrtc/EglBase14$Context"); 1135 if (!jni->IsInstanceOf(egl_context_, j_egl_context_class)) { 1136 ALOGE << "Wrong EGL Context."; 1137 jni->DeleteGlobalRef(egl_context_); 1138 egl_context_ = NULL; 1139 } 1140 } 1141 } 1142 if (egl_context_ == NULL) { 1143 ALOGW << "NULL VideoDecoder EGL context - HW surface encoding is disabled."; 1144 } 1145 } 1146 1147 webrtc::VideoEncoder* MediaCodecVideoEncoderFactory::CreateVideoEncoder( 1148 VideoCodecType type) { 1149 if (supported_codecs_.empty()) { 1150 ALOGW << "No HW video encoder for type " << (int)type; 1151 return NULL; 1152 } 1153 for (std::vector<VideoCodec>::const_iterator it = supported_codecs_.begin(); 1154 it != supported_codecs_.end(); ++it) { 1155 if (it->type == type) { 1156 ALOGD << "Create HW video encoder for type " << (int)type << 1157 " (" << it->name << ")."; 1158 return new MediaCodecVideoEncoder(AttachCurrentThreadIfNeeded(), type, 1159 egl_context_); 1160 } 1161 } 1162 ALOGW << "Can not find HW video encoder for type " << (int)type; 1163 return NULL; 1164 } 1165 1166 const std::vector<MediaCodecVideoEncoderFactory::VideoCodec>& 1167 MediaCodecVideoEncoderFactory::codecs() const { 1168 return supported_codecs_; 1169 } 1170 1171 void MediaCodecVideoEncoderFactory::DestroyVideoEncoder( 1172 webrtc::VideoEncoder* encoder) { 1173 ALOGD << "Destroy video encoder."; 1174 delete encoder; 1175 } 1176 1177 } // namespace webrtc_jni 1178 1179
