1 /* 2 * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. 3 * 4 * Use of this source code is governed by a BSD-style license 5 * that can be found in the LICENSE file in the root of the source 6 * tree. An additional intellectual property rights grant can be found 7 * in the file PATENTS. All contributing project authors may 8 * be found in the AUTHORS file in the root of the source tree. 9 */ 10 11 #include "webrtc/common_video/libyuv/include/webrtc_libyuv.h" 12 #include "webrtc/engine_configurations.h" 13 #include "webrtc/modules/media_file/interface/media_file.h" 14 #include "webrtc/modules/utility/source/file_recorder_impl.h" 15 #include "webrtc/system_wrappers/interface/logging.h" 16 17 #ifdef WEBRTC_MODULE_UTILITY_VIDEO 18 #include "critical_section_wrapper.h" 19 #include "frame_scaler.h" 20 #include "video_coder.h" 21 #include "video_frames_queue.h" 22 #endif 23 24 namespace webrtc { 25 FileRecorder* FileRecorder::CreateFileRecorder(uint32_t instanceID, 26 FileFormats fileFormat) 27 { 28 switch(fileFormat) 29 { 30 case kFileFormatWavFile: 31 case kFileFormatCompressedFile: 32 case kFileFormatPreencodedFile: 33 case kFileFormatPcm16kHzFile: 34 case kFileFormatPcm8kHzFile: 35 case kFileFormatPcm32kHzFile: 36 return new FileRecorderImpl(instanceID, fileFormat); 37 case kFileFormatAviFile: 38 #ifdef WEBRTC_MODULE_UTILITY_VIDEO 39 return new AviRecorder(instanceID, fileFormat); 40 #else 41 assert(false); 42 return NULL; 43 #endif 44 } 45 assert(false); 46 return NULL; 47 } 48 49 void FileRecorder::DestroyFileRecorder(FileRecorder* recorder) 50 { 51 delete recorder; 52 } 53 54 FileRecorderImpl::FileRecorderImpl(uint32_t instanceID, 55 FileFormats fileFormat) 56 : _instanceID(instanceID), 57 _fileFormat(fileFormat), 58 _moduleFile(MediaFile::CreateMediaFile(_instanceID)), 59 codec_info_(), 60 _amrFormat(AMRFileStorage), 61 _audioBuffer(), 62 _audioEncoder(instanceID), 63 _audioResampler() 64 { 65 } 66 67 FileRecorderImpl::~FileRecorderImpl() 68 { 69 MediaFile::DestroyMediaFile(_moduleFile); 70 } 71 72 FileFormats FileRecorderImpl::RecordingFileFormat() const 73 { 74 return _fileFormat; 75 } 76 77 int32_t FileRecorderImpl::RegisterModuleFileCallback( 78 FileCallback* callback) 79 { 80 if(_moduleFile == NULL) 81 { 82 return -1; 83 } 84 return _moduleFile->SetModuleFileCallback(callback); 85 } 86 87 int32_t FileRecorderImpl::StartRecordingAudioFile( 88 const char* fileName, 89 const CodecInst& codecInst, 90 uint32_t notificationTimeMs, 91 ACMAMRPackingFormat amrFormat) 92 { 93 if(_moduleFile == NULL) 94 { 95 return -1; 96 } 97 codec_info_ = codecInst; 98 _amrFormat = amrFormat; 99 100 int32_t retVal = 0; 101 if(_fileFormat != kFileFormatAviFile) 102 { 103 // AVI files should be started using StartRecordingVideoFile(..) all 104 // other formats should use this API. 105 retVal =_moduleFile->StartRecordingAudioFile(fileName, _fileFormat, 106 codecInst, 107 notificationTimeMs); 108 } 109 110 if( retVal == 0) 111 { 112 retVal = SetUpAudioEncoder(); 113 } 114 if( retVal != 0) 115 { 116 LOG(LS_WARNING) << "Failed to initialize file " << fileName 117 << " for recording."; 118 119 if(IsRecording()) 120 { 121 StopRecording(); 122 } 123 } 124 return retVal; 125 } 126 127 int32_t FileRecorderImpl::StartRecordingAudioFile( 128 OutStream& destStream, 129 const CodecInst& codecInst, 130 uint32_t notificationTimeMs, 131 ACMAMRPackingFormat amrFormat) 132 { 133 codec_info_ = codecInst; 134 _amrFormat = amrFormat; 135 136 int32_t retVal = _moduleFile->StartRecordingAudioStream( 137 destStream, 138 _fileFormat, 139 codecInst, 140 notificationTimeMs); 141 142 if( retVal == 0) 143 { 144 retVal = SetUpAudioEncoder(); 145 } 146 if( retVal != 0) 147 { 148 LOG(LS_WARNING) << "Failed to initialize outStream for recording."; 149 150 if(IsRecording()) 151 { 152 StopRecording(); 153 } 154 } 155 return retVal; 156 } 157 158 int32_t FileRecorderImpl::StopRecording() 159 { 160 memset(&codec_info_, 0, sizeof(CodecInst)); 161 return _moduleFile->StopRecording(); 162 } 163 164 bool FileRecorderImpl::IsRecording() const 165 { 166 return _moduleFile->IsRecording(); 167 } 168 169 int32_t FileRecorderImpl::RecordAudioToFile( 170 const AudioFrame& incomingAudioFrame, 171 const TickTime* playoutTS) 172 { 173 if (codec_info_.plfreq == 0) 174 { 175 LOG(LS_WARNING) << "RecordAudioToFile() recording audio is not " 176 << "turned on."; 177 return -1; 178 } 179 AudioFrame tempAudioFrame; 180 tempAudioFrame.samples_per_channel_ = 0; 181 if( incomingAudioFrame.num_channels_ == 2 && 182 !_moduleFile->IsStereo()) 183 { 184 // Recording mono but incoming audio is (interleaved) stereo. 185 tempAudioFrame.num_channels_ = 1; 186 tempAudioFrame.sample_rate_hz_ = incomingAudioFrame.sample_rate_hz_; 187 tempAudioFrame.samples_per_channel_ = 188 incomingAudioFrame.samples_per_channel_; 189 for (uint16_t i = 0; 190 i < (incomingAudioFrame.samples_per_channel_); i++) 191 { 192 // Sample value is the average of left and right buffer rounded to 193 // closest integer value. Note samples can be either 1 or 2 byte. 194 tempAudioFrame.data_[i] = 195 ((incomingAudioFrame.data_[2 * i] + 196 incomingAudioFrame.data_[(2 * i) + 1] + 1) >> 1); 197 } 198 } 199 else if( incomingAudioFrame.num_channels_ == 1 && 200 _moduleFile->IsStereo()) 201 { 202 // Recording stereo but incoming audio is mono. 203 tempAudioFrame.num_channels_ = 2; 204 tempAudioFrame.sample_rate_hz_ = incomingAudioFrame.sample_rate_hz_; 205 tempAudioFrame.samples_per_channel_ = 206 incomingAudioFrame.samples_per_channel_; 207 for (uint16_t i = 0; 208 i < (incomingAudioFrame.samples_per_channel_); i++) 209 { 210 // Duplicate sample to both channels 211 tempAudioFrame.data_[2*i] = 212 incomingAudioFrame.data_[i]; 213 tempAudioFrame.data_[2*i+1] = 214 incomingAudioFrame.data_[i]; 215 } 216 } 217 218 const AudioFrame* ptrAudioFrame = &incomingAudioFrame; 219 if(tempAudioFrame.samples_per_channel_ != 0) 220 { 221 // If ptrAudioFrame is not empty it contains the audio to be recorded. 222 ptrAudioFrame = &tempAudioFrame; 223 } 224 225 // Encode the audio data before writing to file. Don't encode if the codec 226 // is PCM. 227 // NOTE: stereo recording is only supported for WAV files. 228 // TODO (hellner): WAV expect PCM in little endian byte order. Not 229 // "encoding" with PCM coder should be a problem for big endian systems. 230 uint32_t encodedLenInBytes = 0; 231 if (_fileFormat == kFileFormatPreencodedFile || 232 STR_CASE_CMP(codec_info_.plname, "L16") != 0) 233 { 234 if (_audioEncoder.Encode(*ptrAudioFrame, _audioBuffer, 235 encodedLenInBytes) == -1) 236 { 237 LOG(LS_WARNING) << "RecordAudioToFile() codec " 238 << codec_info_.plname 239 << " not supported or failed to encode stream."; 240 return -1; 241 } 242 } else { 243 int outLen = 0; 244 if(ptrAudioFrame->num_channels_ == 2) 245 { 246 // ptrAudioFrame contains interleaved stereo audio. 247 _audioResampler.ResetIfNeeded(ptrAudioFrame->sample_rate_hz_, 248 codec_info_.plfreq, 249 kResamplerSynchronousStereo); 250 _audioResampler.Push(ptrAudioFrame->data_, 251 ptrAudioFrame->samples_per_channel_ * 252 ptrAudioFrame->num_channels_, 253 (int16_t*)_audioBuffer, 254 MAX_AUDIO_BUFFER_IN_BYTES, outLen); 255 } else { 256 _audioResampler.ResetIfNeeded(ptrAudioFrame->sample_rate_hz_, 257 codec_info_.plfreq, 258 kResamplerSynchronous); 259 _audioResampler.Push(ptrAudioFrame->data_, 260 ptrAudioFrame->samples_per_channel_, 261 (int16_t*)_audioBuffer, 262 MAX_AUDIO_BUFFER_IN_BYTES, outLen); 263 } 264 encodedLenInBytes = outLen * sizeof(int16_t); 265 } 266 267 // Codec may not be operating at a frame rate of 10 ms. Whenever enough 268 // 10 ms chunks of data has been pushed to the encoder an encoded frame 269 // will be available. Wait until then. 270 if (encodedLenInBytes) 271 { 272 uint16_t msOfData = 273 ptrAudioFrame->samples_per_channel_ / 274 uint16_t(ptrAudioFrame->sample_rate_hz_ / 1000); 275 if (WriteEncodedAudioData(_audioBuffer, 276 (uint16_t)encodedLenInBytes, 277 msOfData, playoutTS) == -1) 278 { 279 return -1; 280 } 281 } 282 return 0; 283 } 284 285 int32_t FileRecorderImpl::SetUpAudioEncoder() 286 { 287 if (_fileFormat == kFileFormatPreencodedFile || 288 STR_CASE_CMP(codec_info_.plname, "L16") != 0) 289 { 290 if(_audioEncoder.SetEncodeCodec(codec_info_,_amrFormat) == -1) 291 { 292 LOG(LS_ERROR) << "SetUpAudioEncoder() codec " 293 << codec_info_.plname << " not supported."; 294 return -1; 295 } 296 } 297 return 0; 298 } 299 300 int32_t FileRecorderImpl::codec_info(CodecInst& codecInst) const 301 { 302 if(codec_info_.plfreq == 0) 303 { 304 return -1; 305 } 306 codecInst = codec_info_; 307 return 0; 308 } 309 310 int32_t FileRecorderImpl::WriteEncodedAudioData( 311 const int8_t* audioBuffer, 312 uint16_t bufferLength, 313 uint16_t /*millisecondsOfData*/, 314 const TickTime* /*playoutTS*/) 315 { 316 return _moduleFile->IncomingAudioData(audioBuffer, bufferLength); 317 } 318 319 320 #ifdef WEBRTC_MODULE_UTILITY_VIDEO 321 AviRecorder::AviRecorder(uint32_t instanceID, FileFormats fileFormat) 322 : FileRecorderImpl(instanceID, fileFormat), 323 _videoOnly(false), 324 _thread( 0), 325 _timeEvent(*EventWrapper::Create()), 326 _critSec(CriticalSectionWrapper::CreateCriticalSection()), 327 _writtenVideoFramesCounter(0), 328 _writtenAudioMS(0), 329 _writtenVideoMS(0) 330 { 331 _videoEncoder = new VideoCoder(); 332 _frameScaler = new FrameScaler(); 333 _videoFramesQueue = new VideoFramesQueue(); 334 _thread = ThreadWrapper::CreateThread(Run, this, kNormalPriority, 335 "AviRecorder()"); 336 } 337 338 AviRecorder::~AviRecorder( ) 339 { 340 StopRecording( ); 341 342 delete _videoEncoder; 343 delete _frameScaler; 344 delete _videoFramesQueue; 345 delete _thread; 346 delete &_timeEvent; 347 delete _critSec; 348 } 349 350 int32_t AviRecorder::StartRecordingVideoFile( 351 const char* fileName, 352 const CodecInst& audioCodecInst, 353 const VideoCodec& videoCodecInst, 354 ACMAMRPackingFormat amrFormat, 355 bool videoOnly) 356 { 357 _firstAudioFrameReceived = false; 358 _videoCodecInst = videoCodecInst; 359 _videoOnly = videoOnly; 360 361 if(_moduleFile->StartRecordingVideoFile(fileName, _fileFormat, 362 audioCodecInst, videoCodecInst, 363 videoOnly) != 0) 364 { 365 return -1; 366 } 367 368 if(!videoOnly) 369 { 370 if(FileRecorderImpl::StartRecordingAudioFile(fileName,audioCodecInst, 0, 371 amrFormat) !=0) 372 { 373 StopRecording(); 374 return -1; 375 } 376 } 377 if( SetUpVideoEncoder() != 0) 378 { 379 StopRecording(); 380 return -1; 381 } 382 if(_videoOnly) 383 { 384 // Writing to AVI file is non-blocking. 385 // Start non-blocking timer if video only. If recording both video and 386 // audio let the pushing of audio frames be the timer. 387 _timeEvent.StartTimer(true, 1000 / _videoCodecInst.maxFramerate); 388 } 389 StartThread(); 390 return 0; 391 } 392 393 int32_t AviRecorder::StopRecording() 394 { 395 _timeEvent.StopTimer(); 396 397 StopThread(); 398 return FileRecorderImpl::StopRecording(); 399 } 400 401 int32_t AviRecorder::CalcI420FrameSize( ) const 402 { 403 return 3 * _videoCodecInst.width * _videoCodecInst.height / 2; 404 } 405 406 int32_t AviRecorder::SetUpVideoEncoder() 407 { 408 // Size of unencoded data (I420) should be the largest possible frame size 409 // in a file. 410 _videoMaxPayloadSize = CalcI420FrameSize(); 411 _videoEncodedData.VerifyAndAllocate(_videoMaxPayloadSize); 412 413 _videoCodecInst.plType = _videoEncoder->DefaultPayloadType( 414 _videoCodecInst.plName); 415 416 int32_t useNumberOfCores = 1; 417 // Set the max payload size to 16000. This means that the codec will try to 418 // create slices that will fit in 16000 kByte packets. However, the 419 // Encode() call will still generate one full frame. 420 if(_videoEncoder->SetEncodeCodec(_videoCodecInst, useNumberOfCores, 421 16000)) 422 { 423 return -1; 424 } 425 return 0; 426 } 427 428 int32_t AviRecorder::RecordVideoToFile(const I420VideoFrame& videoFrame) 429 { 430 CriticalSectionScoped lock(_critSec); 431 if(!IsRecording() || videoFrame.IsZeroSize()) 432 { 433 return -1; 434 } 435 // The frame is written to file in AviRecorder::Process(). 436 int32_t retVal = _videoFramesQueue->AddFrame(videoFrame); 437 if(retVal != 0) 438 { 439 StopRecording(); 440 } 441 return retVal; 442 } 443 444 bool AviRecorder::StartThread() 445 { 446 unsigned int id; 447 if( _thread == 0) 448 { 449 return false; 450 } 451 452 return _thread->Start(id); 453 } 454 455 bool AviRecorder::StopThread() 456 { 457 _critSec->Enter(); 458 459 if(_thread) 460 { 461 _thread->SetNotAlive(); 462 463 ThreadWrapper* thread = _thread; 464 _thread = NULL; 465 466 _timeEvent.Set(); 467 468 _critSec->Leave(); 469 470 if(thread->Stop()) 471 { 472 delete thread; 473 } else { 474 return false; 475 } 476 } else { 477 _critSec->Leave(); 478 } 479 return true; 480 } 481 482 bool AviRecorder::Run( ThreadObj threadObj) 483 { 484 return static_cast<AviRecorder*>( threadObj)->Process(); 485 } 486 487 int32_t AviRecorder::ProcessAudio() 488 { 489 if (_writtenVideoFramesCounter == 0) 490 { 491 // Get the most recent frame that is due for writing to file. Since 492 // frames are unencoded it's safe to throw away frames if necessary 493 // for synchronizing audio and video. 494 I420VideoFrame* frameToProcess = _videoFramesQueue->FrameToRecord(); 495 if(frameToProcess) 496 { 497 // Syncronize audio to the current frame to process by throwing away 498 // audio samples with older timestamp than the video frame. 499 size_t numberOfAudioElements = 500 _audioFramesToWrite.size(); 501 for (size_t i = 0; i < numberOfAudioElements; ++i) 502 { 503 AudioFrameFileInfo* frameInfo = _audioFramesToWrite.front(); 504 if(TickTime::TicksToMilliseconds( 505 frameInfo->_playoutTS.Ticks()) < 506 frameToProcess->render_time_ms()) 507 { 508 delete frameInfo; 509 _audioFramesToWrite.pop_front(); 510 } else 511 { 512 break; 513 } 514 } 515 } 516 } 517 // Write all audio up to current timestamp. 518 int32_t error = 0; 519 size_t numberOfAudioElements = _audioFramesToWrite.size(); 520 for (size_t i = 0; i < numberOfAudioElements; ++i) 521 { 522 AudioFrameFileInfo* frameInfo = _audioFramesToWrite.front(); 523 if((TickTime::Now() - frameInfo->_playoutTS).Milliseconds() > 0) 524 { 525 _moduleFile->IncomingAudioData(frameInfo->_audioData, 526 frameInfo->_audioSize); 527 _writtenAudioMS += frameInfo->_audioMS; 528 delete frameInfo; 529 _audioFramesToWrite.pop_front(); 530 } else { 531 break; 532 } 533 } 534 return error; 535 } 536 537 bool AviRecorder::Process() 538 { 539 switch(_timeEvent.Wait(500)) 540 { 541 case kEventSignaled: 542 if(_thread == NULL) 543 { 544 return false; 545 } 546 break; 547 case kEventError: 548 return false; 549 case kEventTimeout: 550 // No events triggered. No work to do. 551 return true; 552 } 553 CriticalSectionScoped lock( _critSec); 554 555 // Get the most recent frame to write to file (if any). Synchronize it with 556 // the audio stream (if any). Synchronization the video based on its render 557 // timestamp (i.e. VideoFrame::RenderTimeMS()) 558 I420VideoFrame* frameToProcess = _videoFramesQueue->FrameToRecord(); 559 if( frameToProcess == NULL) 560 { 561 return true; 562 } 563 int32_t error = 0; 564 if(!_videoOnly) 565 { 566 if(!_firstAudioFrameReceived) 567 { 568 // Video and audio can only be synchronized if both have been 569 // received. 570 return true; 571 } 572 error = ProcessAudio(); 573 574 while (_writtenAudioMS > _writtenVideoMS) 575 { 576 error = EncodeAndWriteVideoToFile( *frameToProcess); 577 if( error != 0) 578 { 579 LOG(LS_ERROR) << "AviRecorder::Process() error writing to " 580 << "file."; 581 break; 582 } else { 583 uint32_t frameLengthMS = 1000 / 584 _videoCodecInst.maxFramerate; 585 _writtenVideoFramesCounter++; 586 _writtenVideoMS += frameLengthMS; 587 // A full seconds worth of frames have been written. 588 if(_writtenVideoFramesCounter%_videoCodecInst.maxFramerate == 0) 589 { 590 // Frame rate is in frames per seconds. Frame length is 591 // calculated as an integer division which means it may 592 // be rounded down. Compensate for this every second. 593 uint32_t rest = 1000 % frameLengthMS; 594 _writtenVideoMS += rest; 595 } 596 } 597 } 598 } else { 599 // Frame rate is in frames per seconds. Frame length is calculated as an 600 // integer division which means it may be rounded down. This introduces 601 // drift. Once a full frame worth of drift has happened, skip writing 602 // one frame. Note that frame rate is in frames per second so the 603 // drift is completely compensated for. 604 uint32_t frameLengthMS = 1000/_videoCodecInst.maxFramerate; 605 uint32_t restMS = 1000 % frameLengthMS; 606 uint32_t frameSkip = (_videoCodecInst.maxFramerate * 607 frameLengthMS) / restMS; 608 609 _writtenVideoFramesCounter++; 610 if(_writtenVideoFramesCounter % frameSkip == 0) 611 { 612 _writtenVideoMS += frameLengthMS; 613 return true; 614 } 615 616 error = EncodeAndWriteVideoToFile( *frameToProcess); 617 if(error != 0) 618 { 619 LOG(LS_ERROR) << "AviRecorder::Process() error writing to file."; 620 } else { 621 _writtenVideoMS += frameLengthMS; 622 } 623 } 624 return error == 0; 625 } 626 627 int32_t AviRecorder::EncodeAndWriteVideoToFile(I420VideoFrame& videoFrame) 628 { 629 if (!IsRecording() || videoFrame.IsZeroSize()) 630 { 631 return -1; 632 } 633 634 if(_frameScaler->ResizeFrameIfNeeded(&videoFrame, _videoCodecInst.width, 635 _videoCodecInst.height) != 0) 636 { 637 return -1; 638 } 639 640 _videoEncodedData.payloadSize = 0; 641 642 if( STR_CASE_CMP(_videoCodecInst.plName, "I420") == 0) 643 { 644 int length = CalcBufferSize(kI420, videoFrame.width(), 645 videoFrame.height()); 646 _videoEncodedData.VerifyAndAllocate(length); 647 648 // I420 is raw data. No encoding needed (each sample is represented by 649 // 1 byte so there is no difference depending on endianness). 650 int ret_length = ExtractBuffer(videoFrame, length, 651 _videoEncodedData.payloadData); 652 if (ret_length < 0) 653 return -1; 654 655 _videoEncodedData.payloadSize = ret_length; 656 _videoEncodedData.frameType = kVideoFrameKey; 657 }else { 658 if( _videoEncoder->Encode(videoFrame, _videoEncodedData) != 0) 659 { 660 return -1; 661 } 662 } 663 664 if(_videoEncodedData.payloadSize > 0) 665 { 666 if(_moduleFile->IncomingAVIVideoData( 667 (int8_t*)(_videoEncodedData.payloadData), 668 _videoEncodedData.payloadSize)) 669 { 670 LOG(LS_ERROR) << "Error writing AVI file."; 671 return -1; 672 } 673 } else { 674 LOG(LS_ERROR) << "FileRecorder::RecordVideoToFile() frame dropped by " 675 << "encoder, bitrate likely too low."; 676 } 677 return 0; 678 } 679 680 // Store audio frame in the _audioFramesToWrite buffer. The writing to file 681 // happens in AviRecorder::Process(). 682 int32_t AviRecorder::WriteEncodedAudioData( 683 const int8_t* audioBuffer, 684 uint16_t bufferLength, 685 uint16_t millisecondsOfData, 686 const TickTime* playoutTS) 687 { 688 CriticalSectionScoped lock(_critSec); 689 690 if (!IsRecording()) 691 { 692 return -1; 693 } 694 if (bufferLength > MAX_AUDIO_BUFFER_IN_BYTES) 695 { 696 return -1; 697 } 698 if (_videoOnly) 699 { 700 return -1; 701 } 702 if (_audioFramesToWrite.size() > kMaxAudioBufferQueueLength) 703 { 704 StopRecording(); 705 return -1; 706 } 707 _firstAudioFrameReceived = true; 708 709 if(playoutTS) 710 { 711 _audioFramesToWrite.push_back(new AudioFrameFileInfo(audioBuffer, 712 bufferLength, 713 millisecondsOfData, 714 *playoutTS)); 715 } else { 716 _audioFramesToWrite.push_back(new AudioFrameFileInfo(audioBuffer, 717 bufferLength, 718 millisecondsOfData, 719 TickTime::Now())); 720 } 721 _timeEvent.Set(); 722 return 0; 723 } 724 725 #endif // WEBRTC_MODULE_UTILITY_VIDEO 726 } // namespace webrtc 727
