Home | History | Annotate | Download | only in source 
      1 /*
      2  *  Copyright (c) 2013 The WebRTC project authors. All Rights Reserved.
      3  *
      4  *  Use of this source code is governed by a BSD-style license
      5  *  that can be found in the LICENSE file in the root of the source
      6  *  tree. An additional intellectual property rights grant can be found
      7  *  in the file PATENTS.  All contributing project authors may
      8  *  be found in the AUTHORS file in the root of the source tree.
      9  */
     10 
     11 #include "webrtc/common_types.h"
     12 
     13 #include <algorithm>  // std::max
     14 
     15 #include "webrtc/common_video/libyuv/include/webrtc_libyuv.h"
     16 #include "webrtc/modules/video_coding/codecs/interface/video_codec_interface.h"
     17 #include "webrtc/modules/video_coding/main/source/encoded_frame.h"
     18 #include "webrtc/modules/video_coding/main/source/video_coding_impl.h"
     19 #include "webrtc/system_wrappers/interface/clock.h"
     20 #include "webrtc/system_wrappers/interface/logging.h"
     21 
     22 namespace webrtc {
     23 namespace vcm {
     24 
     25 class DebugRecorder {
     26  public:
     27   DebugRecorder()
     28       : cs_(CriticalSectionWrapper::CreateCriticalSection()), file_(NULL) {}
     29 
     30   ~DebugRecorder() { Stop(); }
     31 
     32   int Start(const char* file_name_utf8) {
     33     CriticalSectionScoped cs(cs_.get());
     34     if (file_)
     35       fclose(file_);
     36     file_ = fopen(file_name_utf8, "wb");
     37     if (!file_)
     38       return VCM_GENERAL_ERROR;
     39     return VCM_OK;
     40   }
     41 
     42   void Stop() {
     43     CriticalSectionScoped cs(cs_.get());
     44     if (file_) {
     45       fclose(file_);
     46       file_ = NULL;
     47     }
     48   }
     49 
     50   void Add(const I420VideoFrame& frame) {
     51     CriticalSectionScoped cs(cs_.get());
     52     if (file_)
     53       PrintI420VideoFrame(frame, file_);
     54   }
     55 
     56  private:
     57   scoped_ptr<CriticalSectionWrapper> cs_;
     58   FILE* file_ GUARDED_BY(cs_);
     59 };
     60 
     61 VideoSender::VideoSender(Clock* clock,
     62                          EncodedImageCallback* post_encode_callback)
     63     : clock_(clock),
     64       recorder_(new DebugRecorder()),
     65       process_crit_sect_(CriticalSectionWrapper::CreateCriticalSection()),
     66       _sendCritSect(CriticalSectionWrapper::CreateCriticalSection()),
     67       _encoder(),
     68       _encodedFrameCallback(post_encode_callback),
     69       _nextFrameTypes(1, kVideoFrameDelta),
     70       _mediaOpt(clock_),
     71       _sendStatsCallback(NULL),
     72       _codecDataBase(),
     73       frame_dropper_enabled_(true),
     74       _sendStatsTimer(1000, clock_),
     75       qm_settings_callback_(NULL),
     76       protection_callback_(NULL) {}
     77 
     78 VideoSender::~VideoSender() {
     79   delete _sendCritSect;
     80 }
     81 
     82 int32_t VideoSender::Process() {
     83   int32_t returnValue = VCM_OK;
     84 
     85   if (_sendStatsTimer.TimeUntilProcess() == 0) {
     86     _sendStatsTimer.Processed();
     87     CriticalSectionScoped cs(process_crit_sect_.get());
     88     if (_sendStatsCallback != NULL) {
     89       uint32_t bitRate;
     90       uint32_t frameRate;
     91       {
     92         CriticalSectionScoped cs(_sendCritSect);
     93         bitRate = _mediaOpt.SentBitRate();
     94         frameRate = _mediaOpt.SentFrameRate();
     95       }
     96       _sendStatsCallback->SendStatistics(bitRate, frameRate);
     97     }
     98   }
     99 
    100   return returnValue;
    101 }
    102 
    103 // Reset send side to initial state - all components
    104 int32_t VideoSender::InitializeSender() {
    105   CriticalSectionScoped cs(_sendCritSect);
    106   _codecDataBase.ResetSender();
    107   _encoder = NULL;
    108   _encodedFrameCallback.SetTransportCallback(NULL);
    109   _mediaOpt.Reset();  // Resetting frame dropper
    110   return VCM_OK;
    111 }
    112 
    113 int32_t VideoSender::TimeUntilNextProcess() {
    114   return _sendStatsTimer.TimeUntilProcess();
    115 }
    116 
    117 // Register the send codec to be used.
    118 int32_t VideoSender::RegisterSendCodec(const VideoCodec* sendCodec,
    119                                        uint32_t numberOfCores,
    120                                        uint32_t maxPayloadSize) {
    121   CriticalSectionScoped cs(_sendCritSect);
    122   if (sendCodec == NULL) {
    123     return VCM_PARAMETER_ERROR;
    124   }
    125 
    126   bool ret = _codecDataBase.SetSendCodec(
    127       sendCodec, numberOfCores, maxPayloadSize, &_encodedFrameCallback);
    128 
    129   // Update encoder regardless of result to make sure that we're not holding on
    130   // to a deleted instance.
    131   _encoder = _codecDataBase.GetEncoder();
    132 
    133   if (!ret) {
    134     LOG(LS_ERROR) << "Failed to initialize the encoder with payload name "
    135                   << sendCodec->plName << ". Error code: " << ret;
    136     return VCM_CODEC_ERROR;
    137   }
    138 
    139   int numLayers = (sendCodec->codecType != kVideoCodecVP8)
    140                       ? 1
    141                       : sendCodec->codecSpecific.VP8.numberOfTemporalLayers;
    142   // If we have screensharing and we have layers, we disable frame dropper.
    143   bool disable_frame_dropper =
    144       numLayers > 1 && sendCodec->mode == kScreensharing;
    145   if (disable_frame_dropper) {
    146     _mediaOpt.EnableFrameDropper(false);
    147   } else if (frame_dropper_enabled_) {
    148     _mediaOpt.EnableFrameDropper(true);
    149   }
    150   _nextFrameTypes.clear();
    151   _nextFrameTypes.resize(VCM_MAX(sendCodec->numberOfSimulcastStreams, 1),
    152                          kVideoFrameDelta);
    153 
    154   _mediaOpt.SetEncodingData(sendCodec->codecType,
    155                             sendCodec->maxBitrate * 1000,
    156                             sendCodec->maxFramerate * 1000,
    157                             sendCodec->startBitrate * 1000,
    158                             sendCodec->width,
    159                             sendCodec->height,
    160                             numLayers,
    161                             maxPayloadSize);
    162   return VCM_OK;
    163 }
    164 
    165 // Get current send codec
    166 int32_t VideoSender::SendCodec(VideoCodec* currentSendCodec) const {
    167   CriticalSectionScoped cs(_sendCritSect);
    168 
    169   if (currentSendCodec == NULL) {
    170     return VCM_PARAMETER_ERROR;
    171   }
    172   return _codecDataBase.SendCodec(currentSendCodec) ? 0 : -1;
    173 }
    174 
    175 // Get the current send codec type
    176 VideoCodecType VideoSender::SendCodec() const {
    177   CriticalSectionScoped cs(_sendCritSect);
    178 
    179   return _codecDataBase.SendCodec();
    180 }
    181 
    182 // Register an external decoder object.
    183 // This can not be used together with external decoder callbacks.
    184 int32_t VideoSender::RegisterExternalEncoder(VideoEncoder* externalEncoder,
    185                                              uint8_t payloadType,
    186                                              bool internalSource /*= false*/) {
    187   CriticalSectionScoped cs(_sendCritSect);
    188 
    189   if (externalEncoder == NULL) {
    190     bool wasSendCodec = false;
    191     const bool ret =
    192         _codecDataBase.DeregisterExternalEncoder(payloadType, &wasSendCodec);
    193     if (wasSendCodec) {
    194       // Make sure the VCM doesn't use the de-registered codec
    195       _encoder = NULL;
    196     }
    197     return ret ? 0 : -1;
    198   }
    199   _codecDataBase.RegisterExternalEncoder(
    200       externalEncoder, payloadType, internalSource);
    201   return 0;
    202 }
    203 
    204 // Get codec config parameters
    205 int32_t VideoSender::CodecConfigParameters(uint8_t* buffer,
    206                                            int32_t size) const {
    207   CriticalSectionScoped cs(_sendCritSect);
    208   if (_encoder != NULL) {
    209     return _encoder->CodecConfigParameters(buffer, size);
    210   }
    211   return VCM_UNINITIALIZED;
    212 }
    213 
    214 // TODO(andresp): Make const once media_opt is thread-safe and this has a
    215 // pointer to it.
    216 int32_t VideoSender::SentFrameCount(VCMFrameCount* frameCount) {
    217   CriticalSectionScoped cs(_sendCritSect);
    218   *frameCount = _mediaOpt.SentFrameCount();
    219   return VCM_OK;
    220 }
    221 
    222 // Get encode bitrate
    223 int VideoSender::Bitrate(unsigned int* bitrate) const {
    224   CriticalSectionScoped cs(_sendCritSect);
    225   // return the bit rate which the encoder is set to
    226   if (!_encoder) {
    227     return VCM_UNINITIALIZED;
    228   }
    229   *bitrate = _encoder->BitRate();
    230   return 0;
    231 }
    232 
    233 // Get encode frame rate
    234 int VideoSender::FrameRate(unsigned int* framerate) const {
    235   CriticalSectionScoped cs(_sendCritSect);
    236   // input frame rate, not compensated
    237   if (!_encoder) {
    238     return VCM_UNINITIALIZED;
    239   }
    240   *framerate = _encoder->FrameRate();
    241   return 0;
    242 }
    243 
    244 // Set channel parameters
    245 int32_t VideoSender::SetChannelParameters(uint32_t target_bitrate,
    246                                           uint8_t lossRate,
    247                                           uint32_t rtt) {
    248   int32_t ret = 0;
    249   {
    250     CriticalSectionScoped sendCs(_sendCritSect);
    251     uint32_t targetRate = _mediaOpt.SetTargetRates(target_bitrate,
    252                                                    lossRate,
    253                                                    rtt,
    254                                                    protection_callback_,
    255                                                    qm_settings_callback_);
    256     if (_encoder != NULL) {
    257       ret = _encoder->SetChannelParameters(lossRate, rtt);
    258       if (ret < 0) {
    259         return ret;
    260       }
    261       ret = (int32_t)_encoder->SetRates(targetRate, _mediaOpt.InputFrameRate());
    262       if (ret < 0) {
    263         return ret;
    264       }
    265     } else {
    266       return VCM_UNINITIALIZED;
    267     }  // encoder
    268   }    // send side
    269   return VCM_OK;
    270 }
    271 
    272 int32_t VideoSender::RegisterTransportCallback(
    273     VCMPacketizationCallback* transport) {
    274   CriticalSectionScoped cs(_sendCritSect);
    275   _encodedFrameCallback.SetMediaOpt(&_mediaOpt);
    276   _encodedFrameCallback.SetTransportCallback(transport);
    277   return VCM_OK;
    278 }
    279 
    280 // Register video output information callback which will be called to deliver
    281 // information about the video stream produced by the encoder, for instance the
    282 // average frame rate and bit rate.
    283 int32_t VideoSender::RegisterSendStatisticsCallback(
    284     VCMSendStatisticsCallback* sendStats) {
    285   CriticalSectionScoped cs(process_crit_sect_.get());
    286   _sendStatsCallback = sendStats;
    287   return VCM_OK;
    288 }
    289 
    290 // Register a video quality settings callback which will be called when frame
    291 // rate/dimensions need to be updated for video quality optimization
    292 int32_t VideoSender::RegisterVideoQMCallback(
    293     VCMQMSettingsCallback* qm_settings_callback) {
    294   CriticalSectionScoped cs(_sendCritSect);
    295   qm_settings_callback_ = qm_settings_callback;
    296   _mediaOpt.EnableQM(qm_settings_callback_ != NULL);
    297   return VCM_OK;
    298 }
    299 
    300 // Register a video protection callback which will be called to deliver the
    301 // requested FEC rate and NACK status (on/off).
    302 int32_t VideoSender::RegisterProtectionCallback(
    303     VCMProtectionCallback* protection_callback) {
    304   CriticalSectionScoped cs(_sendCritSect);
    305   protection_callback_ = protection_callback;
    306   return VCM_OK;
    307 }
    308 
    309 // Enable or disable a video protection method.
    310 // Note: This API should be deprecated, as it does not offer a distinction
    311 // between the protection method and decoding with or without errors. If such a
    312 // behavior is desired, use the following API: SetReceiverRobustnessMode.
    313 int32_t VideoSender::SetVideoProtection(VCMVideoProtection videoProtection,
    314                                         bool enable) {
    315   switch (videoProtection) {
    316     case kProtectionNack:
    317     case kProtectionNackSender: {
    318       CriticalSectionScoped cs(_sendCritSect);
    319       _mediaOpt.EnableProtectionMethod(enable, media_optimization::kNack);
    320       break;
    321     }
    322 
    323     case kProtectionNackFEC: {
    324       CriticalSectionScoped cs(_sendCritSect);
    325       _mediaOpt.EnableProtectionMethod(enable, media_optimization::kNackFec);
    326       break;
    327     }
    328 
    329     case kProtectionFEC: {
    330       CriticalSectionScoped cs(_sendCritSect);
    331       _mediaOpt.EnableProtectionMethod(enable, media_optimization::kFec);
    332       break;
    333     }
    334 
    335     case kProtectionPeriodicKeyFrames: {
    336       CriticalSectionScoped cs(_sendCritSect);
    337       return _codecDataBase.SetPeriodicKeyFrames(enable) ? 0 : -1;
    338       break;
    339     }
    340     case kProtectionNackReceiver:
    341     case kProtectionDualDecoder:
    342     case kProtectionKeyOnLoss:
    343     case kProtectionKeyOnKeyLoss:
    344       // Ignore decoder modes.
    345       return VCM_OK;
    346   }
    347   return VCM_OK;
    348 }
    349 // Add one raw video frame to the encoder, blocking.
    350 int32_t VideoSender::AddVideoFrame(const I420VideoFrame& videoFrame,
    351                                    const VideoContentMetrics* contentMetrics,
    352                                    const CodecSpecificInfo* codecSpecificInfo) {
    353   CriticalSectionScoped cs(_sendCritSect);
    354   if (_encoder == NULL) {
    355     return VCM_UNINITIALIZED;
    356   }
    357   // TODO(holmer): Add support for dropping frames per stream. Currently we
    358   // only have one frame dropper for all streams.
    359   if (_nextFrameTypes[0] == kFrameEmpty) {
    360     return VCM_OK;
    361   }
    362   if (_mediaOpt.DropFrame()) {
    363     return VCM_OK;
    364   }
    365   _mediaOpt.UpdateContentData(contentMetrics);
    366   int32_t ret =
    367       _encoder->Encode(videoFrame, codecSpecificInfo, _nextFrameTypes);
    368   recorder_->Add(videoFrame);
    369   if (ret < 0) {
    370     LOG(LS_ERROR) << "Failed to encode frame. Error code: " << ret;
    371     return ret;
    372   }
    373   for (size_t i = 0; i < _nextFrameTypes.size(); ++i) {
    374     _nextFrameTypes[i] = kVideoFrameDelta;  // Default frame type.
    375   }
    376   return VCM_OK;
    377 }
    378 
    379 int32_t VideoSender::IntraFrameRequest(int stream_index) {
    380   CriticalSectionScoped cs(_sendCritSect);
    381   if (stream_index < 0 ||
    382       static_cast<unsigned int>(stream_index) >= _nextFrameTypes.size()) {
    383     return -1;
    384   }
    385   _nextFrameTypes[stream_index] = kVideoFrameKey;
    386   if (_encoder != NULL && _encoder->InternalSource()) {
    387     // Try to request the frame if we have an external encoder with
    388     // internal source since AddVideoFrame never will be called.
    389     if (_encoder->RequestFrame(_nextFrameTypes) == WEBRTC_VIDEO_CODEC_OK) {
    390       _nextFrameTypes[stream_index] = kVideoFrameDelta;
    391     }
    392   }
    393   return VCM_OK;
    394 }
    395 
    396 int32_t VideoSender::EnableFrameDropper(bool enable) {
    397   CriticalSectionScoped cs(_sendCritSect);
    398   frame_dropper_enabled_ = enable;
    399   _mediaOpt.EnableFrameDropper(enable);
    400   return VCM_OK;
    401 }
    402 
    403 int VideoSender::SetSenderNackMode(SenderNackMode mode) {
    404   CriticalSectionScoped cs(_sendCritSect);
    405 
    406   switch (mode) {
    407     case VideoCodingModule::kNackNone:
    408       _mediaOpt.EnableProtectionMethod(false, media_optimization::kNack);
    409       break;
    410     case VideoCodingModule::kNackAll:
    411       _mediaOpt.EnableProtectionMethod(true, media_optimization::kNack);
    412       break;
    413     case VideoCodingModule::kNackSelective:
    414       return VCM_NOT_IMPLEMENTED;
    415       break;
    416   }
    417   return VCM_OK;
    418 }
    419 
    420 int VideoSender::SetSenderReferenceSelection(bool enable) {
    421   return VCM_NOT_IMPLEMENTED;
    422 }
    423 
    424 int VideoSender::SetSenderFEC(bool enable) {
    425   CriticalSectionScoped cs(_sendCritSect);
    426   _mediaOpt.EnableProtectionMethod(enable, media_optimization::kFec);
    427   return VCM_OK;
    428 }
    429 
    430 int VideoSender::SetSenderKeyFramePeriod(int periodMs) {
    431   return VCM_NOT_IMPLEMENTED;
    432 }
    433 
    434 int VideoSender::StartDebugRecording(const char* file_name_utf8) {
    435   return recorder_->Start(file_name_utf8);
    436 }
    437 
    438 void VideoSender::StopDebugRecording() {
    439   recorder_->Stop();
    440 }
    441 
    442 void VideoSender::SuspendBelowMinBitrate() {
    443   CriticalSectionScoped cs(_sendCritSect);
    444   VideoCodec current_send_codec;
    445   if (SendCodec(&current_send_codec) != 0) {
    446     assert(false);  // Must set a send codec before SuspendBelowMinBitrate.
    447     return;
    448   }
    449   int threshold_bps;
    450   if (current_send_codec.numberOfSimulcastStreams == 0) {
    451     threshold_bps = current_send_codec.minBitrate * 1000;
    452   } else {
    453     threshold_bps = current_send_codec.simulcastStream[0].minBitrate * 1000;
    454   }
    455   // Set the hysteresis window to be at 10% of the threshold, but at least
    456   // 10 kbps.
    457   int window_bps = std::max(threshold_bps / 10, 10000);
    458   _mediaOpt.SuspendBelowMinBitrate(threshold_bps, window_bps);
    459 }
    460 
    461 bool VideoSender::VideoSuspended() const {
    462   CriticalSectionScoped cs(_sendCritSect);
    463   return _mediaOpt.IsVideoSuspended();
    464 }
    465 }  // namespace vcm
    466 }  // namespace webrtc
    467