1 // Copyright (c) 2012 The Chromium Authors. All rights reserved. 2 // Use of this source code is governed by a BSD-style license that can be 3 // found in the LICENSE file. 4 5 #include "media/ffmpeg/ffmpeg_common.h" 6 7 #include "base/basictypes.h" 8 #include "base/logging.h" 9 #include "base/metrics/histogram.h" 10 #include "base/strings/string_number_conversions.h" 11 #include "media/base/decoder_buffer.h" 12 #include "media/base/video_frame.h" 13 #include "media/base/video_util.h" 14 15 namespace media { 16 17 // Why FF_INPUT_BUFFER_PADDING_SIZE? FFmpeg assumes all input buffers are 18 // padded. Check here to ensure FFmpeg only receives data padded to its 19 // specifications. 20 COMPILE_ASSERT(DecoderBuffer::kPaddingSize >= FF_INPUT_BUFFER_PADDING_SIZE, 21 decoder_buffer_padding_size_does_not_fit_ffmpeg_requirement); 22 23 // Alignment requirement by FFmpeg for input and output buffers. This need to 24 // be updated to match FFmpeg when it changes. 25 #if defined(ARCH_CPU_ARM_FAMILY) 26 static const int kFFmpegBufferAddressAlignment = 16; 27 #else 28 static const int kFFmpegBufferAddressAlignment = 32; 29 #endif 30 31 // Check here to ensure FFmpeg only receives data aligned to its specifications. 32 COMPILE_ASSERT( 33 DecoderBuffer::kAlignmentSize >= kFFmpegBufferAddressAlignment && 34 DecoderBuffer::kAlignmentSize % kFFmpegBufferAddressAlignment == 0, 35 decoder_buffer_alignment_size_does_not_fit_ffmpeg_requirement); 36 37 // Allows faster SIMD YUV convert. Also, FFmpeg overreads/-writes occasionally. 38 // See video_get_buffer() in libavcodec/utils.c. 39 static const int kFFmpegOutputBufferPaddingSize = 16; 40 41 COMPILE_ASSERT(VideoFrame::kFrameSizePadding >= kFFmpegOutputBufferPaddingSize, 42 video_frame_padding_size_does_not_fit_ffmpeg_requirement); 43 44 COMPILE_ASSERT( 45 VideoFrame::kFrameAddressAlignment >= kFFmpegBufferAddressAlignment && 46 VideoFrame::kFrameAddressAlignment % kFFmpegBufferAddressAlignment == 0, 47 video_frame_address_alignment_does_not_fit_ffmpeg_requirement); 48 49 static const AVRational kMicrosBase = { 1, base::Time::kMicrosecondsPerSecond }; 50 51 base::TimeDelta ConvertFromTimeBase(const AVRational& time_base, 52 int64 timestamp) { 53 int64 microseconds = av_rescale_q(timestamp, time_base, kMicrosBase); 54 return base::TimeDelta::FromMicroseconds(microseconds); 55 } 56 57 int64 ConvertToTimeBase(const AVRational& time_base, 58 const base::TimeDelta& timestamp) { 59 return av_rescale_q(timestamp.InMicroseconds(), kMicrosBase, time_base); 60 } 61 62 // Converts an FFmpeg audio codec ID into its corresponding supported codec id. 63 static AudioCodec CodecIDToAudioCodec(AVCodecID codec_id) { 64 switch (codec_id) { 65 case AV_CODEC_ID_AAC: 66 return kCodecAAC; 67 case AV_CODEC_ID_MP3: 68 return kCodecMP3; 69 case AV_CODEC_ID_VORBIS: 70 return kCodecVorbis; 71 case AV_CODEC_ID_PCM_U8: 72 case AV_CODEC_ID_PCM_S16LE: 73 case AV_CODEC_ID_PCM_S24LE: 74 case AV_CODEC_ID_PCM_F32LE: 75 return kCodecPCM; 76 case AV_CODEC_ID_PCM_S16BE: 77 return kCodecPCM_S16BE; 78 case AV_CODEC_ID_PCM_S24BE: 79 return kCodecPCM_S24BE; 80 case AV_CODEC_ID_FLAC: 81 return kCodecFLAC; 82 case AV_CODEC_ID_AMR_NB: 83 return kCodecAMR_NB; 84 case AV_CODEC_ID_AMR_WB: 85 return kCodecAMR_WB; 86 case AV_CODEC_ID_GSM_MS: 87 return kCodecGSM_MS; 88 case AV_CODEC_ID_PCM_ALAW: 89 return kCodecPCM_ALAW; 90 case AV_CODEC_ID_PCM_MULAW: 91 return kCodecPCM_MULAW; 92 case AV_CODEC_ID_OPUS: 93 return kCodecOpus; 94 default: 95 DVLOG(1) << "Unknown audio CodecID: " << codec_id; 96 } 97 return kUnknownAudioCodec; 98 } 99 100 static AVCodecID AudioCodecToCodecID(AudioCodec audio_codec, 101 SampleFormat sample_format) { 102 switch (audio_codec) { 103 case kCodecAAC: 104 return AV_CODEC_ID_AAC; 105 case kCodecMP3: 106 return AV_CODEC_ID_MP3; 107 case kCodecPCM: 108 switch (sample_format) { 109 case kSampleFormatU8: 110 return AV_CODEC_ID_PCM_U8; 111 case kSampleFormatS16: 112 return AV_CODEC_ID_PCM_S16LE; 113 case kSampleFormatS32: 114 return AV_CODEC_ID_PCM_S24LE; 115 case kSampleFormatF32: 116 return AV_CODEC_ID_PCM_F32LE; 117 default: 118 DVLOG(1) << "Unsupported sample format: " << sample_format; 119 } 120 break; 121 case kCodecPCM_S16BE: 122 return AV_CODEC_ID_PCM_S16BE; 123 case kCodecPCM_S24BE: 124 return AV_CODEC_ID_PCM_S24BE; 125 case kCodecVorbis: 126 return AV_CODEC_ID_VORBIS; 127 case kCodecFLAC: 128 return AV_CODEC_ID_FLAC; 129 case kCodecAMR_NB: 130 return AV_CODEC_ID_AMR_NB; 131 case kCodecAMR_WB: 132 return AV_CODEC_ID_AMR_WB; 133 case kCodecGSM_MS: 134 return AV_CODEC_ID_GSM_MS; 135 case kCodecPCM_ALAW: 136 return AV_CODEC_ID_PCM_ALAW; 137 case kCodecPCM_MULAW: 138 return AV_CODEC_ID_PCM_MULAW; 139 case kCodecOpus: 140 return AV_CODEC_ID_OPUS; 141 default: 142 DVLOG(1) << "Unknown AudioCodec: " << audio_codec; 143 } 144 return AV_CODEC_ID_NONE; 145 } 146 147 // Converts an FFmpeg video codec ID into its corresponding supported codec id. 148 static VideoCodec CodecIDToVideoCodec(AVCodecID codec_id) { 149 switch (codec_id) { 150 case AV_CODEC_ID_H264: 151 return kCodecH264; 152 case AV_CODEC_ID_THEORA: 153 return kCodecTheora; 154 case AV_CODEC_ID_MPEG4: 155 return kCodecMPEG4; 156 case AV_CODEC_ID_VP8: 157 return kCodecVP8; 158 case AV_CODEC_ID_VP9: 159 return kCodecVP9; 160 default: 161 DVLOG(1) << "Unknown video CodecID: " << codec_id; 162 } 163 return kUnknownVideoCodec; 164 } 165 166 static AVCodecID VideoCodecToCodecID(VideoCodec video_codec) { 167 switch (video_codec) { 168 case kCodecH264: 169 return AV_CODEC_ID_H264; 170 case kCodecTheora: 171 return AV_CODEC_ID_THEORA; 172 case kCodecMPEG4: 173 return AV_CODEC_ID_MPEG4; 174 case kCodecVP8: 175 return AV_CODEC_ID_VP8; 176 case kCodecVP9: 177 return AV_CODEC_ID_VP9; 178 default: 179 DVLOG(1) << "Unknown VideoCodec: " << video_codec; 180 } 181 return AV_CODEC_ID_NONE; 182 } 183 184 static VideoCodecProfile ProfileIDToVideoCodecProfile(int profile) { 185 // Clear out the CONSTRAINED & INTRA flags which are strict subsets of the 186 // corresponding profiles with which they're used. 187 profile &= ~FF_PROFILE_H264_CONSTRAINED; 188 profile &= ~FF_PROFILE_H264_INTRA; 189 switch (profile) { 190 case FF_PROFILE_H264_BASELINE: 191 return H264PROFILE_BASELINE; 192 case FF_PROFILE_H264_MAIN: 193 return H264PROFILE_MAIN; 194 case FF_PROFILE_H264_EXTENDED: 195 return H264PROFILE_EXTENDED; 196 case FF_PROFILE_H264_HIGH: 197 return H264PROFILE_HIGH; 198 case FF_PROFILE_H264_HIGH_10: 199 return H264PROFILE_HIGH10PROFILE; 200 case FF_PROFILE_H264_HIGH_422: 201 return H264PROFILE_HIGH422PROFILE; 202 case FF_PROFILE_H264_HIGH_444_PREDICTIVE: 203 return H264PROFILE_HIGH444PREDICTIVEPROFILE; 204 default: 205 DVLOG(1) << "Unknown profile id: " << profile; 206 } 207 return VIDEO_CODEC_PROFILE_UNKNOWN; 208 } 209 210 static int VideoCodecProfileToProfileID(VideoCodecProfile profile) { 211 switch (profile) { 212 case H264PROFILE_BASELINE: 213 return FF_PROFILE_H264_BASELINE; 214 case H264PROFILE_MAIN: 215 return FF_PROFILE_H264_MAIN; 216 case H264PROFILE_EXTENDED: 217 return FF_PROFILE_H264_EXTENDED; 218 case H264PROFILE_HIGH: 219 return FF_PROFILE_H264_HIGH; 220 case H264PROFILE_HIGH10PROFILE: 221 return FF_PROFILE_H264_HIGH_10; 222 case H264PROFILE_HIGH422PROFILE: 223 return FF_PROFILE_H264_HIGH_422; 224 case H264PROFILE_HIGH444PREDICTIVEPROFILE: 225 return FF_PROFILE_H264_HIGH_444_PREDICTIVE; 226 default: 227 DVLOG(1) << "Unknown VideoCodecProfile: " << profile; 228 } 229 return FF_PROFILE_UNKNOWN; 230 } 231 232 SampleFormat AVSampleFormatToSampleFormat(AVSampleFormat sample_format) { 233 switch (sample_format) { 234 case AV_SAMPLE_FMT_U8: 235 return kSampleFormatU8; 236 case AV_SAMPLE_FMT_S16: 237 return kSampleFormatS16; 238 case AV_SAMPLE_FMT_S32: 239 return kSampleFormatS32; 240 case AV_SAMPLE_FMT_FLT: 241 return kSampleFormatF32; 242 case AV_SAMPLE_FMT_S16P: 243 return kSampleFormatPlanarS16; 244 case AV_SAMPLE_FMT_FLTP: 245 return kSampleFormatPlanarF32; 246 default: 247 DVLOG(1) << "Unknown AVSampleFormat: " << sample_format; 248 } 249 return kUnknownSampleFormat; 250 } 251 252 static AVSampleFormat SampleFormatToAVSampleFormat(SampleFormat sample_format) { 253 switch (sample_format) { 254 case kSampleFormatU8: 255 return AV_SAMPLE_FMT_U8; 256 case kSampleFormatS16: 257 return AV_SAMPLE_FMT_S16; 258 case kSampleFormatS32: 259 return AV_SAMPLE_FMT_S32; 260 case kSampleFormatF32: 261 return AV_SAMPLE_FMT_FLT; 262 case kSampleFormatPlanarS16: 263 return AV_SAMPLE_FMT_S16P; 264 case kSampleFormatPlanarF32: 265 return AV_SAMPLE_FMT_FLTP; 266 default: 267 DVLOG(1) << "Unknown SampleFormat: " << sample_format; 268 } 269 return AV_SAMPLE_FMT_NONE; 270 } 271 272 static void AVCodecContextToAudioDecoderConfig( 273 const AVCodecContext* codec_context, 274 bool is_encrypted, 275 AudioDecoderConfig* config, 276 bool record_stats) { 277 DCHECK_EQ(codec_context->codec_type, AVMEDIA_TYPE_AUDIO); 278 279 AudioCodec codec = CodecIDToAudioCodec(codec_context->codec_id); 280 281 SampleFormat sample_format = 282 AVSampleFormatToSampleFormat(codec_context->sample_fmt); 283 284 ChannelLayout channel_layout = ChannelLayoutToChromeChannelLayout( 285 codec_context->channel_layout, codec_context->channels); 286 287 if (codec == kCodecOpus) { 288 // |codec_context->sample_fmt| is not set by FFmpeg because Opus decoding is 289 // not enabled in FFmpeg. It doesn't matter what value is set here, so long 290 // as it's valid, the true sample format is selected inside the decoder. 291 sample_format = kSampleFormatF32; 292 } 293 294 base::TimeDelta seek_preroll; 295 if (codec_context->seek_preroll > 0) { 296 seek_preroll = base::TimeDelta::FromMicroseconds( 297 codec_context->seek_preroll * 1000000.0 / codec_context->sample_rate); 298 } 299 300 base::TimeDelta codec_delay; 301 if (codec_context->delay > 0) { 302 codec_delay = base::TimeDelta::FromMicroseconds( 303 codec_context->delay * 1000000.0 / codec_context->sample_rate); 304 } 305 306 config->Initialize(codec, 307 sample_format, 308 channel_layout, 309 codec_context->sample_rate, 310 codec_context->extradata, 311 codec_context->extradata_size, 312 is_encrypted, 313 record_stats, 314 seek_preroll, 315 codec_delay); 316 if (codec != kCodecOpus) { 317 DCHECK_EQ(av_get_bytes_per_sample(codec_context->sample_fmt) * 8, 318 config->bits_per_channel()); 319 } 320 } 321 322 void AVStreamToAudioDecoderConfig( 323 const AVStream* stream, 324 AudioDecoderConfig* config, 325 bool record_stats) { 326 bool is_encrypted = false; 327 AVDictionaryEntry* key = av_dict_get(stream->metadata, "enc_key_id", NULL, 0); 328 if (key) 329 is_encrypted = true; 330 return AVCodecContextToAudioDecoderConfig( 331 stream->codec, is_encrypted, config, record_stats); 332 } 333 334 void AudioDecoderConfigToAVCodecContext(const AudioDecoderConfig& config, 335 AVCodecContext* codec_context) { 336 codec_context->codec_type = AVMEDIA_TYPE_AUDIO; 337 codec_context->codec_id = AudioCodecToCodecID(config.codec(), 338 config.sample_format()); 339 codec_context->sample_fmt = SampleFormatToAVSampleFormat( 340 config.sample_format()); 341 342 // TODO(scherkus): should we set |channel_layout|? I'm not sure if FFmpeg uses 343 // said information to decode. 344 codec_context->channels = 345 ChannelLayoutToChannelCount(config.channel_layout()); 346 codec_context->sample_rate = config.samples_per_second(); 347 348 if (config.extra_data()) { 349 codec_context->extradata_size = config.extra_data_size(); 350 codec_context->extradata = reinterpret_cast<uint8_t*>( 351 av_malloc(config.extra_data_size() + FF_INPUT_BUFFER_PADDING_SIZE)); 352 memcpy(codec_context->extradata, config.extra_data(), 353 config.extra_data_size()); 354 memset(codec_context->extradata + config.extra_data_size(), '\0', 355 FF_INPUT_BUFFER_PADDING_SIZE); 356 } else { 357 codec_context->extradata = NULL; 358 codec_context->extradata_size = 0; 359 } 360 } 361 362 void AVStreamToVideoDecoderConfig( 363 const AVStream* stream, 364 VideoDecoderConfig* config, 365 bool record_stats) { 366 gfx::Size coded_size(stream->codec->coded_width, stream->codec->coded_height); 367 368 // TODO(vrk): This assumes decoded frame data starts at (0, 0), which is true 369 // for now, but may not always be true forever. Fix this in the future. 370 gfx::Rect visible_rect(stream->codec->width, stream->codec->height); 371 372 AVRational aspect_ratio = { 1, 1 }; 373 if (stream->sample_aspect_ratio.num) 374 aspect_ratio = stream->sample_aspect_ratio; 375 else if (stream->codec->sample_aspect_ratio.num) 376 aspect_ratio = stream->codec->sample_aspect_ratio; 377 378 VideoCodec codec = CodecIDToVideoCodec(stream->codec->codec_id); 379 380 VideoCodecProfile profile = VIDEO_CODEC_PROFILE_UNKNOWN; 381 if (codec == kCodecVP8) 382 profile = VP8PROFILE_MAIN; 383 else if (codec == kCodecVP9) 384 profile = VP9PROFILE_MAIN; 385 else 386 profile = ProfileIDToVideoCodecProfile(stream->codec->profile); 387 388 gfx::Size natural_size = GetNaturalSize( 389 visible_rect.size(), aspect_ratio.num, aspect_ratio.den); 390 391 if (record_stats) { 392 UMA_HISTOGRAM_ENUMERATION("Media.VideoColorRange", 393 stream->codec->color_range, 394 AVCOL_RANGE_NB); 395 } 396 397 VideoFrame::Format format = PixelFormatToVideoFormat(stream->codec->pix_fmt); 398 if (codec == kCodecVP9) { 399 // TODO(tomfinegan): libavcodec doesn't know about VP9. 400 format = VideoFrame::YV12; 401 coded_size = natural_size; 402 } 403 404 bool is_encrypted = false; 405 AVDictionaryEntry* key = av_dict_get(stream->metadata, "enc_key_id", NULL, 0); 406 if (key) 407 is_encrypted = true; 408 409 AVDictionaryEntry* webm_alpha = 410 av_dict_get(stream->metadata, "alpha_mode", NULL, 0); 411 if (webm_alpha && !strcmp(webm_alpha->value, "1")) { 412 format = VideoFrame::YV12A; 413 } 414 415 config->Initialize(codec, 416 profile, 417 format, 418 coded_size, visible_rect, natural_size, 419 stream->codec->extradata, stream->codec->extradata_size, 420 is_encrypted, 421 record_stats); 422 } 423 424 void VideoDecoderConfigToAVCodecContext( 425 const VideoDecoderConfig& config, 426 AVCodecContext* codec_context) { 427 codec_context->codec_type = AVMEDIA_TYPE_VIDEO; 428 codec_context->codec_id = VideoCodecToCodecID(config.codec()); 429 codec_context->profile = VideoCodecProfileToProfileID(config.profile()); 430 codec_context->coded_width = config.coded_size().width(); 431 codec_context->coded_height = config.coded_size().height(); 432 codec_context->pix_fmt = VideoFormatToPixelFormat(config.format()); 433 434 if (config.extra_data()) { 435 codec_context->extradata_size = config.extra_data_size(); 436 codec_context->extradata = reinterpret_cast<uint8_t*>( 437 av_malloc(config.extra_data_size() + FF_INPUT_BUFFER_PADDING_SIZE)); 438 memcpy(codec_context->extradata, config.extra_data(), 439 config.extra_data_size()); 440 memset(codec_context->extradata + config.extra_data_size(), '\0', 441 FF_INPUT_BUFFER_PADDING_SIZE); 442 } else { 443 codec_context->extradata = NULL; 444 codec_context->extradata_size = 0; 445 } 446 } 447 448 ChannelLayout ChannelLayoutToChromeChannelLayout(int64_t layout, int channels) { 449 switch (layout) { 450 case AV_CH_LAYOUT_MONO: 451 return CHANNEL_LAYOUT_MONO; 452 case AV_CH_LAYOUT_STEREO: 453 return CHANNEL_LAYOUT_STEREO; 454 case AV_CH_LAYOUT_2_1: 455 return CHANNEL_LAYOUT_2_1; 456 case AV_CH_LAYOUT_SURROUND: 457 return CHANNEL_LAYOUT_SURROUND; 458 case AV_CH_LAYOUT_4POINT0: 459 return CHANNEL_LAYOUT_4_0; 460 case AV_CH_LAYOUT_2_2: 461 return CHANNEL_LAYOUT_2_2; 462 case AV_CH_LAYOUT_QUAD: 463 return CHANNEL_LAYOUT_QUAD; 464 case AV_CH_LAYOUT_5POINT0: 465 return CHANNEL_LAYOUT_5_0; 466 case AV_CH_LAYOUT_5POINT1: 467 return CHANNEL_LAYOUT_5_1; 468 case AV_CH_LAYOUT_5POINT0_BACK: 469 return CHANNEL_LAYOUT_5_0_BACK; 470 case AV_CH_LAYOUT_5POINT1_BACK: 471 return CHANNEL_LAYOUT_5_1_BACK; 472 case AV_CH_LAYOUT_7POINT0: 473 return CHANNEL_LAYOUT_7_0; 474 case AV_CH_LAYOUT_7POINT1: 475 return CHANNEL_LAYOUT_7_1; 476 case AV_CH_LAYOUT_7POINT1_WIDE: 477 return CHANNEL_LAYOUT_7_1_WIDE; 478 case AV_CH_LAYOUT_STEREO_DOWNMIX: 479 return CHANNEL_LAYOUT_STEREO_DOWNMIX; 480 case AV_CH_LAYOUT_2POINT1: 481 return CHANNEL_LAYOUT_2POINT1; 482 case AV_CH_LAYOUT_3POINT1: 483 return CHANNEL_LAYOUT_3_1; 484 case AV_CH_LAYOUT_4POINT1: 485 return CHANNEL_LAYOUT_4_1; 486 case AV_CH_LAYOUT_6POINT0: 487 return CHANNEL_LAYOUT_6_0; 488 case AV_CH_LAYOUT_6POINT0_FRONT: 489 return CHANNEL_LAYOUT_6_0_FRONT; 490 case AV_CH_LAYOUT_HEXAGONAL: 491 return CHANNEL_LAYOUT_HEXAGONAL; 492 case AV_CH_LAYOUT_6POINT1: 493 return CHANNEL_LAYOUT_6_1; 494 case AV_CH_LAYOUT_6POINT1_BACK: 495 return CHANNEL_LAYOUT_6_1_BACK; 496 case AV_CH_LAYOUT_6POINT1_FRONT: 497 return CHANNEL_LAYOUT_6_1_FRONT; 498 case AV_CH_LAYOUT_7POINT0_FRONT: 499 return CHANNEL_LAYOUT_7_0_FRONT; 500 #ifdef AV_CH_LAYOUT_7POINT1_WIDE_BACK 501 case AV_CH_LAYOUT_7POINT1_WIDE_BACK: 502 return CHANNEL_LAYOUT_7_1_WIDE_BACK; 503 #endif 504 case AV_CH_LAYOUT_OCTAGONAL: 505 return CHANNEL_LAYOUT_OCTAGONAL; 506 default: 507 // FFmpeg channel_layout is 0 for .wav and .mp3. Attempt to guess layout 508 // based on the channel count. 509 return GuessChannelLayout(channels); 510 } 511 } 512 513 VideoFrame::Format PixelFormatToVideoFormat(PixelFormat pixel_format) { 514 switch (pixel_format) { 515 case PIX_FMT_YUV422P: 516 return VideoFrame::YV16; 517 case PIX_FMT_YUV420P: 518 return VideoFrame::YV12; 519 case PIX_FMT_YUVJ420P: 520 return VideoFrame::YV12J; 521 case PIX_FMT_YUVA420P: 522 return VideoFrame::YV12A; 523 default: 524 DVLOG(1) << "Unsupported PixelFormat: " << pixel_format; 525 } 526 return VideoFrame::UNKNOWN; 527 } 528 529 PixelFormat VideoFormatToPixelFormat(VideoFrame::Format video_format) { 530 switch (video_format) { 531 case VideoFrame::YV16: 532 return PIX_FMT_YUV422P; 533 case VideoFrame::YV12: 534 return PIX_FMT_YUV420P; 535 case VideoFrame::YV12J: 536 return PIX_FMT_YUVJ420P; 537 case VideoFrame::YV12A: 538 return PIX_FMT_YUVA420P; 539 default: 540 DVLOG(1) << "Unsupported VideoFrame::Format: " << video_format; 541 } 542 return PIX_FMT_NONE; 543 } 544 545 } // namespace media 546