Home | History | Annotate | Download | only in video 
      1 /*
      2  *  Copyright (c) 2013 The WebRTC project authors. All Rights Reserved.
      3  *
      4  *  Use of this source code is governed by a BSD-style license
      5  *  that can be found in the LICENSE file in the root of the source
      6  *  tree. An additional intellectual property rights grant can be found
      7  *  in the file PATENTS.  All contributing project authors may
      8  *  be found in the AUTHORS file in the root of the source tree.
      9  */
     10 
     11 #include "webrtc/video/video_send_stream.h"
     12 
     13 #include <algorithm>
     14 #include <sstream>
     15 #include <string>
     16 #include <vector>
     17 
     18 #include "webrtc/common_video/libyuv/include/webrtc_libyuv.h"
     19 #include "webrtc/system_wrappers/interface/logging.h"
     20 #include "webrtc/video_engine/include/vie_base.h"
     21 #include "webrtc/video_engine/include/vie_capture.h"
     22 #include "webrtc/video_engine/include/vie_codec.h"
     23 #include "webrtc/video_engine/include/vie_external_codec.h"
     24 #include "webrtc/video_engine/include/vie_image_process.h"
     25 #include "webrtc/video_engine/include/vie_network.h"
     26 #include "webrtc/video_engine/include/vie_rtp_rtcp.h"
     27 #include "webrtc/video_engine/vie_defines.h"
     28 #include "webrtc/video_send_stream.h"
     29 
     30 namespace webrtc {
     31 std::string
     32 VideoSendStream::Config::EncoderSettings::ToString() const {
     33   std::stringstream ss;
     34   ss << "{payload_name: " << payload_name;
     35   ss << ", payload_type: " << payload_type;
     36   if (encoder != NULL)
     37     ss << ", encoder: " << (encoder != NULL ? "(encoder)" : "NULL");
     38   ss << '}';
     39   return ss.str();
     40 }
     41 
     42 std::string VideoSendStream::Config::Rtp::Rtx::ToString()
     43     const {
     44   std::stringstream ss;
     45   ss << "{ssrcs: {";
     46   for (size_t i = 0; i < ssrcs.size(); ++i) {
     47     ss << ssrcs[i];
     48     if (i != ssrcs.size() - 1)
     49       ss << "}, {";
     50   }
     51   ss << '}';
     52 
     53   ss << ", payload_type: " << payload_type;
     54   ss << '}';
     55   return ss.str();
     56 }
     57 
     58 std::string VideoSendStream::Config::Rtp::ToString() const {
     59   std::stringstream ss;
     60   ss << "{ssrcs: {";
     61   for (size_t i = 0; i < ssrcs.size(); ++i) {
     62     ss << ssrcs[i];
     63     if (i != ssrcs.size() - 1)
     64       ss << "}, {";
     65   }
     66   ss << '}';
     67 
     68   ss << ", max_packet_size: " << max_packet_size;
     69   if (min_transmit_bitrate_bps != 0)
     70     ss << ", min_transmit_bitrate_bps: " << min_transmit_bitrate_bps;
     71 
     72   ss << ", extensions: {";
     73   for (size_t i = 0; i < extensions.size(); ++i) {
     74     ss << extensions[i].ToString();
     75     if (i != extensions.size() - 1)
     76       ss << "}, {";
     77   }
     78   ss << '}';
     79 
     80   if (nack.rtp_history_ms != 0)
     81     ss << ", nack.rtp_history_ms: " << nack.rtp_history_ms;
     82   if (fec.ulpfec_payload_type != -1 || fec.red_payload_type != -1)
     83     ss << ", fec: " << fec.ToString();
     84   if (rtx.payload_type != 0 || !rtx.ssrcs.empty())
     85     ss << ", rtx: " << rtx.ToString();
     86   if (c_name != "")
     87     ss << ", c_name: " << c_name;
     88   ss << '}';
     89   return ss.str();
     90 }
     91 
     92 std::string VideoSendStream::Config::ToString() const {
     93   std::stringstream ss;
     94   ss << "{encoder_settings: " << encoder_settings.ToString();
     95   ss << ", rtp: " << rtp.ToString();
     96   if (pre_encode_callback != NULL)
     97     ss << ", (pre_encode_callback)";
     98   if (post_encode_callback != NULL)
     99     ss << ", (post_encode_callback)";
    100   if (local_renderer != NULL) {
    101     ss << ", (local_renderer, render_delay_ms: " << render_delay_ms << ")";
    102   }
    103   if (target_delay_ms > 0)
    104     ss << ", target_delay_ms: " << target_delay_ms;
    105   if (suspend_below_min_bitrate)
    106     ss << ", suspend_below_min_bitrate: on";
    107   ss << '}';
    108   return ss.str();
    109 }
    110 
    111 namespace internal {
    112 VideoSendStream::VideoSendStream(
    113     newapi::Transport* transport,
    114     CpuOveruseObserver* overuse_observer,
    115     webrtc::VideoEngine* video_engine,
    116     const VideoSendStream::Config& config,
    117     const VideoEncoderConfig& encoder_config,
    118     const std::map<uint32_t, RtpState>& suspended_ssrcs,
    119     int base_channel,
    120     int start_bitrate_bps)
    121     : transport_adapter_(transport),
    122       encoded_frame_proxy_(config.post_encode_callback),
    123       config_(config),
    124       start_bitrate_bps_(start_bitrate_bps),
    125       suspended_ssrcs_(suspended_ssrcs),
    126       external_codec_(NULL),
    127       channel_(-1),
    128       stats_proxy_(config) {
    129   video_engine_base_ = ViEBase::GetInterface(video_engine);
    130   video_engine_base_->CreateChannel(channel_, base_channel);
    131   assert(channel_ != -1);
    132   assert(start_bitrate_bps_ > 0);
    133 
    134   rtp_rtcp_ = ViERTP_RTCP::GetInterface(video_engine);
    135   assert(rtp_rtcp_ != NULL);
    136 
    137   assert(config_.rtp.ssrcs.size() > 0);
    138 
    139   assert(config_.rtp.min_transmit_bitrate_bps >= 0);
    140   rtp_rtcp_->SetMinTransmitBitrate(channel_,
    141                                    config_.rtp.min_transmit_bitrate_bps / 1000);
    142 
    143   for (size_t i = 0; i < config_.rtp.extensions.size(); ++i) {
    144     const std::string& extension = config_.rtp.extensions[i].name;
    145     int id = config_.rtp.extensions[i].id;
    146     if (extension == RtpExtension::kTOffset) {
    147       if (rtp_rtcp_->SetSendTimestampOffsetStatus(channel_, true, id) != 0)
    148         abort();
    149     } else if (extension == RtpExtension::kAbsSendTime) {
    150       if (rtp_rtcp_->SetSendAbsoluteSendTimeStatus(channel_, true, id) != 0)
    151         abort();
    152     } else {
    153       abort();  // Unsupported extension.
    154     }
    155   }
    156 
    157   rtp_rtcp_->SetRembStatus(channel_, true, false);
    158 
    159   // Enable NACK, FEC or both.
    160   if (config_.rtp.fec.red_payload_type != -1) {
    161     assert(config_.rtp.fec.ulpfec_payload_type != -1);
    162     if (config_.rtp.nack.rtp_history_ms > 0) {
    163       rtp_rtcp_->SetHybridNACKFECStatus(
    164           channel_,
    165           true,
    166           static_cast<unsigned char>(config_.rtp.fec.red_payload_type),
    167           static_cast<unsigned char>(config_.rtp.fec.ulpfec_payload_type));
    168     } else {
    169       rtp_rtcp_->SetFECStatus(
    170           channel_,
    171           true,
    172           static_cast<unsigned char>(config_.rtp.fec.red_payload_type),
    173           static_cast<unsigned char>(config_.rtp.fec.ulpfec_payload_type));
    174     }
    175   } else {
    176     rtp_rtcp_->SetNACKStatus(channel_, config_.rtp.nack.rtp_history_ms > 0);
    177   }
    178 
    179   ConfigureSsrcs();
    180 
    181   char rtcp_cname[ViERTP_RTCP::KMaxRTCPCNameLength];
    182   assert(config_.rtp.c_name.length() < ViERTP_RTCP::KMaxRTCPCNameLength);
    183   strncpy(rtcp_cname, config_.rtp.c_name.c_str(), sizeof(rtcp_cname) - 1);
    184   rtcp_cname[sizeof(rtcp_cname) - 1] = '\0';
    185 
    186   rtp_rtcp_->SetRTCPCName(channel_, rtcp_cname);
    187 
    188   capture_ = ViECapture::GetInterface(video_engine);
    189   capture_->AllocateExternalCaptureDevice(capture_id_, external_capture_);
    190   capture_->ConnectCaptureDevice(capture_id_, channel_);
    191 
    192   network_ = ViENetwork::GetInterface(video_engine);
    193   assert(network_ != NULL);
    194 
    195   network_->RegisterSendTransport(channel_, transport_adapter_);
    196   // 28 to match packet overhead in ModuleRtpRtcpImpl.
    197   network_->SetMTU(channel_,
    198                    static_cast<unsigned int>(config_.rtp.max_packet_size + 28));
    199 
    200   assert(config.encoder_settings.encoder != NULL);
    201   assert(config.encoder_settings.payload_type >= 0);
    202   assert(config.encoder_settings.payload_type <= 127);
    203   external_codec_ = ViEExternalCodec::GetInterface(video_engine);
    204   if (external_codec_->RegisterExternalSendCodec(
    205           channel_,
    206           config.encoder_settings.payload_type,
    207           config.encoder_settings.encoder,
    208           false) != 0) {
    209     abort();
    210   }
    211 
    212   codec_ = ViECodec::GetInterface(video_engine);
    213   if (!ReconfigureVideoEncoder(encoder_config))
    214     abort();
    215 
    216   if (overuse_observer)
    217     video_engine_base_->RegisterCpuOveruseObserver(channel_, overuse_observer);
    218 
    219   video_engine_base_->RegisterSendSideDelayObserver(channel_, &stats_proxy_);
    220 
    221   image_process_ = ViEImageProcess::GetInterface(video_engine);
    222   image_process_->RegisterPreEncodeCallback(channel_,
    223                                             config_.pre_encode_callback);
    224   if (config_.post_encode_callback) {
    225     image_process_->RegisterPostEncodeImageCallback(channel_,
    226                                                     &encoded_frame_proxy_);
    227   }
    228 
    229   if (config_.suspend_below_min_bitrate)
    230     codec_->SuspendBelowMinBitrate(channel_);
    231 
    232   rtp_rtcp_->RegisterSendChannelRtcpStatisticsCallback(channel_,
    233                                                        &stats_proxy_);
    234   rtp_rtcp_->RegisterSendChannelRtpStatisticsCallback(channel_,
    235                                                       &stats_proxy_);
    236   rtp_rtcp_->RegisterSendBitrateObserver(channel_, &stats_proxy_);
    237   rtp_rtcp_->RegisterSendFrameCountObserver(channel_, &stats_proxy_);
    238 
    239   codec_->RegisterEncoderObserver(channel_, stats_proxy_);
    240   capture_->RegisterObserver(capture_id_, stats_proxy_);
    241 }
    242 
    243 VideoSendStream::~VideoSendStream() {
    244   capture_->DeregisterObserver(capture_id_);
    245   codec_->DeregisterEncoderObserver(channel_);
    246 
    247   rtp_rtcp_->DeregisterSendFrameCountObserver(channel_, &stats_proxy_);
    248   rtp_rtcp_->DeregisterSendBitrateObserver(channel_, &stats_proxy_);
    249   rtp_rtcp_->DeregisterSendChannelRtpStatisticsCallback(channel_,
    250                                                         &stats_proxy_);
    251   rtp_rtcp_->DeregisterSendChannelRtcpStatisticsCallback(channel_,
    252                                                          &stats_proxy_);
    253 
    254   image_process_->DeRegisterPreEncodeCallback(channel_);
    255 
    256   network_->DeregisterSendTransport(channel_);
    257 
    258   capture_->DisconnectCaptureDevice(channel_);
    259   capture_->ReleaseCaptureDevice(capture_id_);
    260 
    261   external_codec_->DeRegisterExternalSendCodec(
    262       channel_, config_.encoder_settings.payload_type);
    263 
    264   video_engine_base_->DeleteChannel(channel_);
    265 
    266   image_process_->Release();
    267   video_engine_base_->Release();
    268   capture_->Release();
    269   codec_->Release();
    270   if (external_codec_)
    271     external_codec_->Release();
    272   network_->Release();
    273   rtp_rtcp_->Release();
    274 }
    275 
    276 void VideoSendStream::SwapFrame(I420VideoFrame* frame) {
    277   // TODO(pbos): Local rendering should not be done on the capture thread.
    278   if (config_.local_renderer != NULL)
    279     config_.local_renderer->RenderFrame(*frame, 0);
    280 
    281   external_capture_->SwapFrame(frame);
    282 }
    283 
    284 VideoSendStreamInput* VideoSendStream::Input() { return this; }
    285 
    286 void VideoSendStream::Start() {
    287   transport_adapter_.Enable();
    288   video_engine_base_->StartSend(channel_);
    289   video_engine_base_->StartReceive(channel_);
    290 }
    291 
    292 void VideoSendStream::Stop() {
    293   video_engine_base_->StopSend(channel_);
    294   video_engine_base_->StopReceive(channel_);
    295   transport_adapter_.Disable();
    296 }
    297 
    298 bool VideoSendStream::ReconfigureVideoEncoder(
    299     const VideoEncoderConfig& config) {
    300   const std::vector<VideoStream>& streams = config.streams;
    301   assert(!streams.empty());
    302   assert(config_.rtp.ssrcs.size() >= streams.size());
    303 
    304   VideoCodec video_codec;
    305   memset(&video_codec, 0, sizeof(video_codec));
    306   if (config_.encoder_settings.payload_name == "VP8") {
    307     video_codec.codecType = kVideoCodecVP8;
    308   } else if (config_.encoder_settings.payload_name == "H264") {
    309     video_codec.codecType = kVideoCodecH264;
    310   } else {
    311     video_codec.codecType = kVideoCodecGeneric;
    312   }
    313   switch (config.content_type) {
    314     case VideoEncoderConfig::kRealtimeVideo:
    315       video_codec.mode = kRealtimeVideo;
    316       break;
    317     case VideoEncoderConfig::kScreenshare:
    318       video_codec.mode = kScreensharing;
    319       break;
    320   }
    321 
    322   if (video_codec.codecType == kVideoCodecVP8) {
    323     video_codec.codecSpecific.VP8 = VideoEncoder::GetDefaultVp8Settings();
    324   } else if (video_codec.codecType == kVideoCodecH264) {
    325     video_codec.codecSpecific.H264 = VideoEncoder::GetDefaultH264Settings();
    326   }
    327 
    328   if (video_codec.codecType == kVideoCodecVP8) {
    329     if (config.encoder_specific_settings != NULL) {
    330       video_codec.codecSpecific.VP8 = *reinterpret_cast<const VideoCodecVP8*>(
    331                                           config.encoder_specific_settings);
    332     }
    333     video_codec.codecSpecific.VP8.numberOfTemporalLayers =
    334         static_cast<unsigned char>(streams.back().temporal_layers.size());
    335   } else {
    336     // TODO(pbos): Support encoder_settings codec-agnostically.
    337     assert(config.encoder_specific_settings == NULL);
    338   }
    339 
    340   strncpy(video_codec.plName,
    341           config_.encoder_settings.payload_name.c_str(),
    342           kPayloadNameSize - 1);
    343   video_codec.plName[kPayloadNameSize - 1] = '\0';
    344   video_codec.plType = config_.encoder_settings.payload_type;
    345   video_codec.numberOfSimulcastStreams =
    346       static_cast<unsigned char>(streams.size());
    347   video_codec.minBitrate = streams[0].min_bitrate_bps / 1000;
    348   assert(streams.size() <= kMaxSimulcastStreams);
    349   for (size_t i = 0; i < streams.size(); ++i) {
    350     SimulcastStream* sim_stream = &video_codec.simulcastStream[i];
    351     assert(streams[i].width > 0);
    352     assert(streams[i].height > 0);
    353     assert(streams[i].max_framerate > 0);
    354     // Different framerates not supported per stream at the moment.
    355     assert(streams[i].max_framerate == streams[0].max_framerate);
    356     assert(streams[i].min_bitrate_bps >= 0);
    357     assert(streams[i].target_bitrate_bps >= streams[i].min_bitrate_bps);
    358     assert(streams[i].max_bitrate_bps >= streams[i].target_bitrate_bps);
    359     assert(streams[i].max_qp >= 0);
    360 
    361     sim_stream->width = static_cast<unsigned short>(streams[i].width);
    362     sim_stream->height = static_cast<unsigned short>(streams[i].height);
    363     sim_stream->minBitrate = streams[i].min_bitrate_bps / 1000;
    364     sim_stream->targetBitrate = streams[i].target_bitrate_bps / 1000;
    365     sim_stream->maxBitrate = streams[i].max_bitrate_bps / 1000;
    366     sim_stream->qpMax = streams[i].max_qp;
    367     sim_stream->numberOfTemporalLayers =
    368         static_cast<unsigned char>(streams[i].temporal_layers.size());
    369 
    370     video_codec.width = std::max(video_codec.width,
    371                                  static_cast<unsigned short>(streams[i].width));
    372     video_codec.height = std::max(
    373         video_codec.height, static_cast<unsigned short>(streams[i].height));
    374     video_codec.minBitrate =
    375         std::min(video_codec.minBitrate,
    376                  static_cast<unsigned int>(streams[i].min_bitrate_bps / 1000));
    377     video_codec.maxBitrate += streams[i].max_bitrate_bps / 1000;
    378     video_codec.qpMax = std::max(video_codec.qpMax,
    379                                  static_cast<unsigned int>(streams[i].max_qp));
    380   }
    381   video_codec.startBitrate =
    382       static_cast<unsigned int>(start_bitrate_bps_) / 1000;
    383 
    384   if (video_codec.minBitrate < kViEMinCodecBitrate)
    385     video_codec.minBitrate = kViEMinCodecBitrate;
    386   if (video_codec.maxBitrate < kViEMinCodecBitrate)
    387     video_codec.maxBitrate = kViEMinCodecBitrate;
    388   if (video_codec.startBitrate < video_codec.minBitrate)
    389     video_codec.startBitrate = video_codec.minBitrate;
    390   if (video_codec.startBitrate > video_codec.maxBitrate)
    391     video_codec.startBitrate = video_codec.maxBitrate;
    392 
    393   if (video_codec.startBitrate < video_codec.minBitrate)
    394     video_codec.startBitrate = video_codec.minBitrate;
    395   if (video_codec.startBitrate > video_codec.maxBitrate)
    396     video_codec.startBitrate = video_codec.maxBitrate;
    397 
    398   assert(streams[0].max_framerate > 0);
    399   video_codec.maxFramerate = streams[0].max_framerate;
    400 
    401   return codec_->SetSendCodec(channel_, video_codec) == 0;
    402 }
    403 
    404 bool VideoSendStream::DeliverRtcp(const uint8_t* packet, size_t length) {
    405   return network_->ReceivedRTCPPacket(
    406              channel_, packet, static_cast<int>(length)) == 0;
    407 }
    408 
    409 VideoSendStream::Stats VideoSendStream::GetStats() const {
    410   return stats_proxy_.GetStats();
    411 }
    412 
    413 void VideoSendStream::ConfigureSsrcs() {
    414   for (size_t i = 0; i < config_.rtp.ssrcs.size(); ++i) {
    415     uint32_t ssrc = config_.rtp.ssrcs[i];
    416     rtp_rtcp_->SetLocalSSRC(
    417         channel_, ssrc, kViEStreamTypeNormal, static_cast<unsigned char>(i));
    418     RtpStateMap::iterator it = suspended_ssrcs_.find(ssrc);
    419     if (it != suspended_ssrcs_.end())
    420       rtp_rtcp_->SetRtpStateForSsrc(channel_, ssrc, it->second);
    421   }
    422 
    423   if (config_.rtp.rtx.ssrcs.empty()) {
    424     assert(!config_.rtp.rtx.pad_with_redundant_payloads);
    425     return;
    426   }
    427 
    428   // Set up RTX.
    429   assert(config_.rtp.rtx.ssrcs.size() == config_.rtp.ssrcs.size());
    430   for (size_t i = 0; i < config_.rtp.rtx.ssrcs.size(); ++i) {
    431     uint32_t ssrc = config_.rtp.rtx.ssrcs[i];
    432     rtp_rtcp_->SetLocalSSRC(channel_,
    433                             config_.rtp.rtx.ssrcs[i],
    434                             kViEStreamTypeRtx,
    435                             static_cast<unsigned char>(i));
    436     RtpStateMap::iterator it = suspended_ssrcs_.find(ssrc);
    437     if (it != suspended_ssrcs_.end())
    438       rtp_rtcp_->SetRtpStateForSsrc(channel_, ssrc, it->second);
    439   }
    440 
    441   if (config_.rtp.rtx.pad_with_redundant_payloads) {
    442     rtp_rtcp_->SetPadWithRedundantPayloads(channel_, true);
    443   }
    444 
    445   assert(config_.rtp.rtx.payload_type >= 0);
    446   rtp_rtcp_->SetRtxSendPayloadType(channel_, config_.rtp.rtx.payload_type);
    447 }
    448 
    449 std::map<uint32_t, RtpState> VideoSendStream::GetRtpStates() const {
    450   std::map<uint32_t, RtpState> rtp_states;
    451   for (size_t i = 0; i < config_.rtp.ssrcs.size(); ++i) {
    452     uint32_t ssrc = config_.rtp.ssrcs[i];
    453     rtp_states[ssrc] = rtp_rtcp_->GetRtpStateForSsrc(channel_, ssrc);
    454   }
    455 
    456   for (size_t i = 0; i < config_.rtp.rtx.ssrcs.size(); ++i) {
    457     uint32_t ssrc = config_.rtp.rtx.ssrcs[i];
    458     rtp_states[ssrc] = rtp_rtcp_->GetRtpStateForSsrc(channel_, ssrc);
    459   }
    460 
    461   return rtp_states;
    462 }
    463 
    464 void VideoSendStream::SignalNetworkState(Call::NetworkState state) {
    465   // When network goes up, enable RTCP status before setting transmission state.
    466   // When it goes down, disable RTCP afterwards. This ensures that any packets
    467   // sent due to the network state changed will not be dropped.
    468   if (state == Call::kNetworkUp)
    469     rtp_rtcp_->SetRTCPStatus(channel_, kRtcpCompound_RFC4585);
    470   network_->SetNetworkTransmissionState(channel_, state == Call::kNetworkUp);
    471   if (state == Call::kNetworkDown)
    472     rtp_rtcp_->SetRTCPStatus(channel_, kRtcpNone);
    473 }
    474 
    475 }  // namespace internal
    476 }  // namespace webrtc
    477