Home | History | Annotate | Download | only in video 
      1 /*
      2  *  Copyright (c) 2013 The WebRTC project authors. All Rights Reserved.
      3  *
      4  *  Use of this source code is governed by a BSD-style license
      5  *  that can be found in the LICENSE file in the root of the source
      6  *  tree. An additional intellectual property rights grant can be found
      7  *  in the file PATENTS.  All contributing project authors may
      8  *  be found in the AUTHORS file in the root of the source tree.
      9  */
     10 
     11 #include "webrtc/video/video_receive_stream.h"
     12 
     13 #include <stdlib.h>
     14 
     15 #include <set>
     16 #include <string>
     17 
     18 #include "webrtc/base/checks.h"
     19 #include "webrtc/base/logging.h"
     20 #include "webrtc/call/congestion_controller.h"
     21 #include "webrtc/common_video/libyuv/include/webrtc_libyuv.h"
     22 #include "webrtc/system_wrappers/include/clock.h"
     23 #include "webrtc/video/call_stats.h"
     24 #include "webrtc/video/receive_statistics_proxy.h"
     25 #include "webrtc/video_receive_stream.h"
     26 
     27 namespace webrtc {
     28 
     29 static bool UseSendSideBwe(const std::vector<RtpExtension>& extensions) {
     30   for (const auto& extension : extensions) {
     31     if (extension.name == RtpExtension::kTransportSequenceNumber)
     32       return true;
     33   }
     34   return false;
     35 }
     36 
     37 std::string VideoReceiveStream::Decoder::ToString() const {
     38   std::stringstream ss;
     39   ss << "{decoder: " << (decoder != nullptr ? "(VideoDecoder)" : "nullptr");
     40   ss << ", payload_type: " << payload_type;
     41   ss << ", payload_name: " << payload_name;
     42   ss << '}';
     43 
     44   return ss.str();
     45 }
     46 
     47 std::string VideoReceiveStream::Config::ToString() const {
     48   std::stringstream ss;
     49   ss << "{decoders: [";
     50   for (size_t i = 0; i < decoders.size(); ++i) {
     51     ss << decoders[i].ToString();
     52     if (i != decoders.size() - 1)
     53       ss << ", ";
     54   }
     55   ss << ']';
     56   ss << ", rtp: " << rtp.ToString();
     57   ss << ", renderer: " << (renderer != nullptr ? "(renderer)" : "nullptr");
     58   ss << ", render_delay_ms: " << render_delay_ms;
     59   if (!sync_group.empty())
     60     ss << ", sync_group: " << sync_group;
     61   ss << ", pre_decode_callback: "
     62      << (pre_decode_callback != nullptr ? "(EncodedFrameObserver)" : "nullptr");
     63   ss << ", pre_render_callback: "
     64      << (pre_render_callback != nullptr ? "(I420FrameCallback)" : "nullptr");
     65   ss << ", target_delay_ms: " << target_delay_ms;
     66   ss << '}';
     67 
     68   return ss.str();
     69 }
     70 
     71 std::string VideoReceiveStream::Config::Rtp::ToString() const {
     72   std::stringstream ss;
     73   ss << "{remote_ssrc: " << remote_ssrc;
     74   ss << ", local_ssrc: " << local_ssrc;
     75   ss << ", rtcp_mode: "
     76      << (rtcp_mode == RtcpMode::kCompound ? "RtcpMode::kCompound"
     77                                           : "RtcpMode::kReducedSize");
     78   ss << ", rtcp_xr: ";
     79   ss << "{receiver_reference_time_report: "
     80      << (rtcp_xr.receiver_reference_time_report ? "on" : "off");
     81   ss << '}';
     82   ss << ", remb: " << (remb ? "on" : "off");
     83   ss << ", transport_cc: " << (transport_cc ? "on" : "off");
     84   ss << ", nack: {rtp_history_ms: " << nack.rtp_history_ms << '}';
     85   ss << ", fec: " << fec.ToString();
     86   ss << ", rtx: {";
     87   for (auto& kv : rtx) {
     88     ss << kv.first << " -> ";
     89     ss << "{ssrc: " << kv.second.ssrc;
     90     ss << ", payload_type: " << kv.second.payload_type;
     91     ss << '}';
     92   }
     93   ss << '}';
     94   ss << ", extensions: [";
     95   for (size_t i = 0; i < extensions.size(); ++i) {
     96     ss << extensions[i].ToString();
     97     if (i != extensions.size() - 1)
     98       ss << ", ";
     99   }
    100   ss << ']';
    101   ss << '}';
    102   return ss.str();
    103 }
    104 
    105 namespace internal {
    106 namespace {
    107 
    108 VideoCodec CreateDecoderVideoCodec(const VideoReceiveStream::Decoder& decoder) {
    109   VideoCodec codec;
    110   memset(&codec, 0, sizeof(codec));
    111 
    112   codec.plType = decoder.payload_type;
    113   strncpy(codec.plName, decoder.payload_name.c_str(), sizeof(codec.plName));
    114   if (decoder.payload_name == "VP8") {
    115     codec.codecType = kVideoCodecVP8;
    116   } else if (decoder.payload_name == "VP9") {
    117     codec.codecType = kVideoCodecVP9;
    118   } else if (decoder.payload_name == "H264") {
    119     codec.codecType = kVideoCodecH264;
    120   } else {
    121     codec.codecType = kVideoCodecGeneric;
    122   }
    123 
    124   if (codec.codecType == kVideoCodecVP8) {
    125     codec.codecSpecific.VP8 = VideoEncoder::GetDefaultVp8Settings();
    126   } else if (codec.codecType == kVideoCodecVP9) {
    127     codec.codecSpecific.VP9 = VideoEncoder::GetDefaultVp9Settings();
    128   } else if (codec.codecType == kVideoCodecH264) {
    129     codec.codecSpecific.H264 = VideoEncoder::GetDefaultH264Settings();
    130   }
    131 
    132   codec.width = 320;
    133   codec.height = 180;
    134   codec.startBitrate = codec.minBitrate = codec.maxBitrate =
    135       Call::Config::kDefaultStartBitrateBps / 1000;
    136 
    137   return codec;
    138 }
    139 }  // namespace
    140 
    141 VideoReceiveStream::VideoReceiveStream(
    142     int num_cpu_cores,
    143     CongestionController* congestion_controller,
    144     const VideoReceiveStream::Config& config,
    145     webrtc::VoiceEngine* voice_engine,
    146     ProcessThread* process_thread,
    147     CallStats* call_stats)
    148     : transport_adapter_(config.rtcp_send_transport),
    149       encoded_frame_proxy_(config.pre_decode_callback),
    150       config_(config),
    151       clock_(Clock::GetRealTimeClock()),
    152       congestion_controller_(congestion_controller),
    153       call_stats_(call_stats) {
    154   LOG(LS_INFO) << "VideoReceiveStream: " << config_.ToString();
    155 
    156   bool send_side_bwe =
    157       config.rtp.transport_cc && UseSendSideBwe(config_.rtp.extensions);
    158 
    159   RemoteBitrateEstimator* bitrate_estimator =
    160       congestion_controller_->GetRemoteBitrateEstimator(send_side_bwe);
    161 
    162   vie_channel_.reset(new ViEChannel(
    163       num_cpu_cores, &transport_adapter_, process_thread, nullptr,
    164       nullptr, nullptr, bitrate_estimator, call_stats_->rtcp_rtt_stats(),
    165       congestion_controller_->pacer(), congestion_controller_->packet_router(),
    166       1, false));
    167 
    168   RTC_CHECK(vie_channel_->Init() == 0);
    169 
    170   // Register the channel to receive stats updates.
    171   call_stats_->RegisterStatsObserver(vie_channel_->GetStatsObserver());
    172 
    173   // TODO(pbos): This is not fine grained enough...
    174   vie_channel_->SetProtectionMode(config_.rtp.nack.rtp_history_ms > 0, false,
    175                                   -1, -1);
    176   RTC_DCHECK(config_.rtp.rtcp_mode != RtcpMode::kOff)
    177       << "A stream should not be configured with RTCP disabled. This value is "
    178          "reserved for internal usage.";
    179   vie_channel_->SetRTCPMode(config_.rtp.rtcp_mode);
    180 
    181   RTC_DCHECK(config_.rtp.remote_ssrc != 0);
    182   // TODO(pbos): What's an appropriate local_ssrc for receive-only streams?
    183   RTC_DCHECK(config_.rtp.local_ssrc != 0);
    184   RTC_DCHECK(config_.rtp.remote_ssrc != config_.rtp.local_ssrc);
    185 
    186   vie_channel_->SetSSRC(config_.rtp.local_ssrc, kViEStreamTypeNormal, 0);
    187   // TODO(pbos): Support multiple RTX, per video payload.
    188   Config::Rtp::RtxMap::const_iterator it = config_.rtp.rtx.begin();
    189   for (; it != config_.rtp.rtx.end(); ++it) {
    190     RTC_DCHECK(it->second.ssrc != 0);
    191     RTC_DCHECK(it->second.payload_type != 0);
    192 
    193     vie_channel_->SetRemoteSSRCType(kViEStreamTypeRtx, it->second.ssrc);
    194     vie_channel_->SetRtxReceivePayloadType(it->second.payload_type, it->first);
    195   }
    196   // TODO(holmer): When Chrome no longer depends on this being false by default,
    197   // always use the mapping and remove this whole codepath.
    198   vie_channel_->SetUseRtxPayloadMappingOnRestore(
    199       config_.rtp.use_rtx_payload_mapping_on_restore);
    200 
    201   congestion_controller_->SetChannelRembStatus(false, config_.rtp.remb,
    202                                                vie_channel_->rtp_rtcp());
    203 
    204   for (size_t i = 0; i < config_.rtp.extensions.size(); ++i) {
    205     const std::string& extension = config_.rtp.extensions[i].name;
    206     int id = config_.rtp.extensions[i].id;
    207     // One-byte-extension local identifiers are in the range 1-14 inclusive.
    208     RTC_DCHECK_GE(id, 1);
    209     RTC_DCHECK_LE(id, 14);
    210     if (extension == RtpExtension::kTOffset) {
    211       RTC_CHECK_EQ(0, vie_channel_->SetReceiveTimestampOffsetStatus(true, id));
    212     } else if (extension == RtpExtension::kAbsSendTime) {
    213       RTC_CHECK_EQ(0, vie_channel_->SetReceiveAbsoluteSendTimeStatus(true, id));
    214     } else if (extension == RtpExtension::kVideoRotation) {
    215       RTC_CHECK_EQ(0, vie_channel_->SetReceiveVideoRotationStatus(true, id));
    216     } else if (extension == RtpExtension::kTransportSequenceNumber) {
    217       RTC_CHECK_EQ(0,
    218                    vie_channel_->SetReceiveTransportSequenceNumber(true, id));
    219     } else {
    220       RTC_NOTREACHED() << "Unsupported RTP extension.";
    221     }
    222   }
    223 
    224   if (config_.rtp.fec.ulpfec_payload_type != -1) {
    225     // ULPFEC without RED doesn't make sense.
    226     RTC_DCHECK(config_.rtp.fec.red_payload_type != -1);
    227     VideoCodec codec;
    228     memset(&codec, 0, sizeof(codec));
    229     codec.codecType = kVideoCodecULPFEC;
    230     strncpy(codec.plName, "ulpfec", sizeof(codec.plName));
    231     codec.plType = config_.rtp.fec.ulpfec_payload_type;
    232     RTC_CHECK_EQ(0, vie_channel_->SetReceiveCodec(codec));
    233   }
    234   if (config_.rtp.fec.red_payload_type != -1) {
    235     VideoCodec codec;
    236     memset(&codec, 0, sizeof(codec));
    237     codec.codecType = kVideoCodecRED;
    238     strncpy(codec.plName, "red", sizeof(codec.plName));
    239     codec.plType = config_.rtp.fec.red_payload_type;
    240     RTC_CHECK_EQ(0, vie_channel_->SetReceiveCodec(codec));
    241     if (config_.rtp.fec.red_rtx_payload_type != -1) {
    242       vie_channel_->SetRtxReceivePayloadType(
    243           config_.rtp.fec.red_rtx_payload_type,
    244           config_.rtp.fec.red_payload_type);
    245     }
    246   }
    247 
    248   if (config.rtp.rtcp_xr.receiver_reference_time_report)
    249     vie_channel_->SetRtcpXrRrtrStatus(true);
    250 
    251   stats_proxy_.reset(
    252       new ReceiveStatisticsProxy(config_.rtp.remote_ssrc, clock_));
    253 
    254   vie_channel_->RegisterReceiveStatisticsProxy(stats_proxy_.get());
    255   vie_channel_->RegisterReceiveChannelRtcpStatisticsCallback(
    256       stats_proxy_.get());
    257   vie_channel_->RegisterReceiveChannelRtpStatisticsCallback(stats_proxy_.get());
    258   vie_channel_->RegisterRtcpPacketTypeCounterObserver(stats_proxy_.get());
    259 
    260   RTC_DCHECK(!config_.decoders.empty());
    261   std::set<int> decoder_payload_types;
    262   for (size_t i = 0; i < config_.decoders.size(); ++i) {
    263     const Decoder& decoder = config_.decoders[i];
    264     RTC_CHECK(decoder.decoder);
    265     RTC_CHECK(decoder_payload_types.find(decoder.payload_type) ==
    266               decoder_payload_types.end())
    267         << "Duplicate payload type (" << decoder.payload_type
    268         << ") for different decoders.";
    269     decoder_payload_types.insert(decoder.payload_type);
    270     vie_channel_->RegisterExternalDecoder(decoder.payload_type,
    271                                           decoder.decoder);
    272 
    273     VideoCodec codec = CreateDecoderVideoCodec(decoder);
    274 
    275     RTC_CHECK_EQ(0, vie_channel_->SetReceiveCodec(codec));
    276   }
    277 
    278   incoming_video_stream_.reset(new IncomingVideoStream(
    279       0, config.renderer ? config.renderer->SmoothsRenderedFrames() : false));
    280   incoming_video_stream_->SetExpectedRenderDelay(config.render_delay_ms);
    281   vie_channel_->SetExpectedRenderDelay(config.render_delay_ms);
    282   incoming_video_stream_->SetExternalCallback(this);
    283   vie_channel_->SetIncomingVideoStream(incoming_video_stream_.get());
    284 
    285   vie_channel_->RegisterPreDecodeImageCallback(this);
    286   vie_channel_->RegisterPreRenderCallback(this);
    287 }
    288 
    289 VideoReceiveStream::~VideoReceiveStream() {
    290   LOG(LS_INFO) << "~VideoReceiveStream: " << config_.ToString();
    291   incoming_video_stream_->Stop();
    292   vie_channel_->RegisterPreRenderCallback(nullptr);
    293   vie_channel_->RegisterPreDecodeImageCallback(nullptr);
    294 
    295   call_stats_->DeregisterStatsObserver(vie_channel_->GetStatsObserver());
    296   congestion_controller_->SetChannelRembStatus(false, false,
    297                                                vie_channel_->rtp_rtcp());
    298 
    299   uint32_t remote_ssrc = vie_channel_->GetRemoteSSRC();
    300   bool send_side_bwe = UseSendSideBwe(config_.rtp.extensions);
    301   congestion_controller_->GetRemoteBitrateEstimator(send_side_bwe)->
    302       RemoveStream(remote_ssrc);
    303 }
    304 
    305 void VideoReceiveStream::Start() {
    306   transport_adapter_.Enable();
    307   incoming_video_stream_->Start();
    308   vie_channel_->StartReceive();
    309 }
    310 
    311 void VideoReceiveStream::Stop() {
    312   incoming_video_stream_->Stop();
    313   vie_channel_->StopReceive();
    314   transport_adapter_.Disable();
    315 }
    316 
    317 void VideoReceiveStream::SetSyncChannel(VoiceEngine* voice_engine,
    318                                         int audio_channel_id) {
    319   if (voice_engine != nullptr && audio_channel_id != -1) {
    320     VoEVideoSync* voe_sync_interface = VoEVideoSync::GetInterface(voice_engine);
    321     vie_channel_->SetVoiceChannel(audio_channel_id, voe_sync_interface);
    322     voe_sync_interface->Release();
    323   } else {
    324     vie_channel_->SetVoiceChannel(-1, nullptr);
    325   }
    326 }
    327 
    328 VideoReceiveStream::Stats VideoReceiveStream::GetStats() const {
    329   return stats_proxy_->GetStats();
    330 }
    331 
    332 bool VideoReceiveStream::DeliverRtcp(const uint8_t* packet, size_t length) {
    333   return vie_channel_->ReceivedRTCPPacket(packet, length) == 0;
    334 }
    335 
    336 bool VideoReceiveStream::DeliverRtp(const uint8_t* packet,
    337                                     size_t length,
    338                                     const PacketTime& packet_time) {
    339   return vie_channel_->ReceivedRTPPacket(packet, length, packet_time) == 0;
    340 }
    341 
    342 void VideoReceiveStream::FrameCallback(VideoFrame* video_frame) {
    343   stats_proxy_->OnDecodedFrame();
    344 
    345   // Post processing is not supported if the frame is backed by a texture.
    346   if (video_frame->native_handle() == NULL) {
    347     if (config_.pre_render_callback)
    348       config_.pre_render_callback->FrameCallback(video_frame);
    349   }
    350 }
    351 
    352 int VideoReceiveStream::RenderFrame(const uint32_t /*stream_id*/,
    353                                     const VideoFrame& video_frame) {
    354   // TODO(pbos): Wire up config_.render->IsTextureSupported() and convert if not
    355   // supported. Or provide methods for converting a texture frame in
    356   // VideoFrame.
    357 
    358   if (config_.renderer != nullptr)
    359     config_.renderer->RenderFrame(
    360         video_frame,
    361         video_frame.render_time_ms() - clock_->TimeInMilliseconds());
    362 
    363   stats_proxy_->OnRenderedFrame(video_frame.width(), video_frame.height());
    364 
    365   return 0;
    366 }
    367 
    368 // TODO(asapersson): Consider moving callback from video_encoder.h or
    369 // creating a different callback.
    370 int32_t VideoReceiveStream::Encoded(
    371     const EncodedImage& encoded_image,
    372     const CodecSpecificInfo* codec_specific_info,
    373     const RTPFragmentationHeader* fragmentation) {
    374   stats_proxy_->OnPreDecode(encoded_image, codec_specific_info);
    375   if (config_.pre_decode_callback) {
    376     // TODO(asapersson): Remove EncodedFrameCallbackAdapter.
    377     encoded_frame_proxy_.Encoded(
    378         encoded_image, codec_specific_info, fragmentation);
    379   }
    380   return 0;
    381 }
    382 
    383 void VideoReceiveStream::SignalNetworkState(NetworkState state) {
    384   vie_channel_->SetRTCPMode(state == kNetworkUp ? config_.rtp.rtcp_mode
    385                                                 : RtcpMode::kOff);
    386 }
    387 
    388 }  // namespace internal
    389 }  // namespace webrtc
    390