1 /* 2 * libjingle 3 * Copyright 2004 Google Inc. 4 * 5 * Redistribution and use in source and binary forms, with or without 6 * modification, are permitted provided that the following conditions are met: 7 * 8 * 1. Redistributions of source code must retain the above copyright notice, 9 * this list of conditions and the following disclaimer. 10 * 2. Redistributions in binary form must reproduce the above copyright notice, 11 * this list of conditions and the following disclaimer in the documentation 12 * and/or other materials provided with the distribution. 13 * 3. The name of the author may not be used to endorse or promote products 14 * derived from this software without specific prior written permission. 15 * 16 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR IMPLIED 17 * WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF 18 * MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO 19 * EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 20 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, 21 * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; 22 * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, 23 * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR 24 * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF 25 * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 26 */ 27 28 #ifndef TALK_MEDIA_WEBRTCVIDEOENGINE_H_ 29 #define TALK_MEDIA_WEBRTCVIDEOENGINE_H_ 30 31 #include <map> 32 #include <vector> 33 34 #include "talk/base/scoped_ptr.h" 35 #include "talk/media/base/codec.h" 36 #include "talk/media/base/videocommon.h" 37 #include "talk/media/webrtc/webrtccommon.h" 38 #include "talk/media/webrtc/webrtcexport.h" 39 #include "talk/media/webrtc/webrtcvideoencoderfactory.h" 40 #include "talk/session/media/channel.h" 41 #include "webrtc/video_engine/include/vie_base.h" 42 43 #if !defined(LIBPEERCONNECTION_LIB) && \ 44 !defined(LIBPEERCONNECTION_IMPLEMENTATION) 45 #error "Bogus include." 46 #endif 47 48 namespace webrtc { 49 class VideoCaptureModule; 50 class VideoDecoder; 51 class VideoEncoder; 52 class VideoRender; 53 class ViEExternalCapture; 54 class ViERTP_RTCP; 55 } 56 57 namespace talk_base { 58 class CpuMonitor; 59 } // namespace talk_base 60 61 namespace cricket { 62 63 class VideoCapturer; 64 class VideoFrame; 65 class VideoProcessor; 66 class VideoRenderer; 67 class ViETraceWrapper; 68 class ViEWrapper; 69 class VoiceMediaChannel; 70 class WebRtcDecoderObserver; 71 class WebRtcEncoderObserver; 72 class WebRtcLocalStreamInfo; 73 class WebRtcRenderAdapter; 74 class WebRtcVideoChannelRecvInfo; 75 class WebRtcVideoChannelSendInfo; 76 class WebRtcVideoDecoderFactory; 77 class WebRtcVideoEncoderFactory; 78 class WebRtcVideoMediaChannel; 79 class WebRtcVoiceEngine; 80 81 struct CapturedFrame; 82 struct Device; 83 84 class WebRtcVideoEngine : public sigslot::has_slots<>, 85 public webrtc::TraceCallback, 86 public WebRtcVideoEncoderFactory::Observer { 87 public: 88 // Creates the WebRtcVideoEngine with internal VideoCaptureModule. 89 WebRtcVideoEngine(); 90 // For testing purposes. Allows the WebRtcVoiceEngine, 91 // ViEWrapper and CpuMonitor to be mocks. 92 // TODO(juberti): Remove the 3-arg ctor once fake tracing is implemented. 93 WebRtcVideoEngine(WebRtcVoiceEngine* voice_engine, 94 ViEWrapper* vie_wrapper, 95 talk_base::CpuMonitor* cpu_monitor); 96 WebRtcVideoEngine(WebRtcVoiceEngine* voice_engine, 97 ViEWrapper* vie_wrapper, 98 ViETraceWrapper* tracing, 99 talk_base::CpuMonitor* cpu_monitor); 100 ~WebRtcVideoEngine(); 101 102 // Basic video engine implementation. 103 bool Init(talk_base::Thread* worker_thread); 104 void Terminate(); 105 106 int GetCapabilities(); 107 bool SetOptions(const VideoOptions &options); 108 bool SetDefaultEncoderConfig(const VideoEncoderConfig& config); 109 VideoEncoderConfig GetDefaultEncoderConfig() const; 110 111 WebRtcVideoMediaChannel* CreateChannel(VoiceMediaChannel* voice_channel); 112 113 const std::vector<VideoCodec>& codecs() const; 114 const std::vector<RtpHeaderExtension>& rtp_header_extensions() const; 115 void SetLogging(int min_sev, const char* filter); 116 117 bool SetLocalRenderer(VideoRenderer* renderer); 118 sigslot::repeater2<VideoCapturer*, CaptureState> SignalCaptureStateChange; 119 120 // Set the VoiceEngine for A/V sync. This can only be called before Init. 121 bool SetVoiceEngine(WebRtcVoiceEngine* voice_engine); 122 // Set a WebRtcVideoDecoderFactory for external decoding. Video engine does 123 // not take the ownership of |decoder_factory|. The caller needs to make sure 124 // that |decoder_factory| outlives the video engine. 125 void SetExternalDecoderFactory(WebRtcVideoDecoderFactory* decoder_factory); 126 // Set a WebRtcVideoEncoderFactory for external encoding. Video engine does 127 // not take the ownership of |encoder_factory|. The caller needs to make sure 128 // that |encoder_factory| outlives the video engine. 129 void SetExternalEncoderFactory(WebRtcVideoEncoderFactory* encoder_factory); 130 // Enable the render module with timing control. 131 bool EnableTimedRender(); 132 133 // Returns an external decoder for the given codec type. The return value 134 // can be NULL if decoder factory is not given or it does not support the 135 // codec type. The caller takes the ownership of the returned object. 136 webrtc::VideoDecoder* CreateExternalDecoder(webrtc::VideoCodecType type); 137 // Releases the decoder instance created by CreateExternalDecoder(). 138 void DestroyExternalDecoder(webrtc::VideoDecoder* decoder); 139 140 // Returns an external encoder for the given codec type. The return value 141 // can be NULL if encoder factory is not given or it does not support the 142 // codec type. The caller takes the ownership of the returned object. 143 webrtc::VideoEncoder* CreateExternalEncoder(webrtc::VideoCodecType type); 144 // Releases the encoder instance created by CreateExternalEncoder(). 145 void DestroyExternalEncoder(webrtc::VideoEncoder* encoder); 146 147 // Returns true if the codec type is supported by the external encoder. 148 bool IsExternalEncoderCodecType(webrtc::VideoCodecType type) const; 149 150 // Functions called by WebRtcVideoMediaChannel. 151 talk_base::Thread* worker_thread() { return worker_thread_; } 152 ViEWrapper* vie() { return vie_wrapper_.get(); } 153 const VideoFormat& default_codec_format() const { 154 return default_codec_format_; 155 } 156 int GetLastEngineError(); 157 bool FindCodec(const VideoCodec& in); 158 bool CanSendCodec(const VideoCodec& in, const VideoCodec& current, 159 VideoCodec* out); 160 void RegisterChannel(WebRtcVideoMediaChannel* channel); 161 void UnregisterChannel(WebRtcVideoMediaChannel* channel); 162 bool ConvertFromCricketVideoCodec(const VideoCodec& in_codec, 163 webrtc::VideoCodec* out_codec); 164 // Check whether the supplied trace should be ignored. 165 bool ShouldIgnoreTrace(const std::string& trace); 166 int GetNumOfChannels(); 167 168 VideoFormat GetStartCaptureFormat() const { return default_codec_format_; } 169 170 talk_base::CpuMonitor* cpu_monitor() { return cpu_monitor_.get(); } 171 172 protected: 173 // When a video processor registers with the engine. 174 // SignalMediaFrame will be invoked for every video frame. 175 // See videoprocessor.h for param reference. 176 sigslot::signal3<uint32, VideoFrame*, bool*> SignalMediaFrame; 177 178 private: 179 typedef std::vector<WebRtcVideoMediaChannel*> VideoChannels; 180 struct VideoCodecPref { 181 const char* name; 182 int payload_type; 183 // For RTX, this field is the payload-type that RTX applies to. 184 // For other codecs, it should be set to -1. 185 int associated_payload_type; 186 int pref; 187 }; 188 189 static const VideoCodecPref kVideoCodecPrefs[]; 190 static const VideoFormatPod kVideoFormats[]; 191 static const VideoFormatPod kDefaultVideoFormat; 192 193 void Construct(ViEWrapper* vie_wrapper, 194 ViETraceWrapper* tracing, 195 WebRtcVoiceEngine* voice_engine, 196 talk_base::CpuMonitor* cpu_monitor); 197 bool SetDefaultCodec(const VideoCodec& codec); 198 bool RebuildCodecList(const VideoCodec& max_codec); 199 void SetTraceFilter(int filter); 200 void SetTraceOptions(const std::string& options); 201 bool InitVideoEngine(); 202 203 // webrtc::TraceCallback implementation. 204 virtual void Print(webrtc::TraceLevel level, const char* trace, int length); 205 206 // WebRtcVideoEncoderFactory::Observer implementation. 207 virtual void OnCodecsAvailable(); 208 209 talk_base::Thread* worker_thread_; 210 talk_base::scoped_ptr<ViEWrapper> vie_wrapper_; 211 bool vie_wrapper_base_initialized_; 212 talk_base::scoped_ptr<ViETraceWrapper> tracing_; 213 WebRtcVoiceEngine* voice_engine_; 214 talk_base::scoped_ptr<webrtc::VideoRender> render_module_; 215 WebRtcVideoEncoderFactory* encoder_factory_; 216 WebRtcVideoDecoderFactory* decoder_factory_; 217 std::vector<VideoCodec> video_codecs_; 218 std::vector<RtpHeaderExtension> rtp_header_extensions_; 219 VideoFormat default_codec_format_; 220 221 bool initialized_; 222 talk_base::CriticalSection channels_crit_; 223 VideoChannels channels_; 224 225 bool capture_started_; 226 int local_renderer_w_; 227 int local_renderer_h_; 228 VideoRenderer* local_renderer_; 229 230 // Critical section to protect the media processor register/unregister 231 // while processing a frame 232 talk_base::CriticalSection signal_media_critical_; 233 234 talk_base::scoped_ptr<talk_base::CpuMonitor> cpu_monitor_; 235 }; 236 237 class WebRtcVideoMediaChannel : public talk_base::MessageHandler, 238 public VideoMediaChannel, 239 public webrtc::Transport { 240 public: 241 WebRtcVideoMediaChannel(WebRtcVideoEngine* engine, 242 VoiceMediaChannel* voice_channel); 243 ~WebRtcVideoMediaChannel(); 244 bool Init(); 245 246 WebRtcVideoEngine* engine() { return engine_; } 247 VoiceMediaChannel* voice_channel() { return voice_channel_; } 248 int video_channel() const { return vie_channel_; } 249 bool sending() const { return sending_; } 250 251 // VideoMediaChannel implementation 252 virtual bool SetRecvCodecs(const std::vector<VideoCodec> &codecs); 253 virtual bool SetSendCodecs(const std::vector<VideoCodec> &codecs); 254 virtual bool GetSendCodec(VideoCodec* send_codec); 255 virtual bool SetSendStreamFormat(uint32 ssrc, const VideoFormat& format); 256 virtual bool SetRender(bool render); 257 virtual bool SetSend(bool send); 258 259 virtual bool AddSendStream(const StreamParams& sp); 260 virtual bool RemoveSendStream(uint32 ssrc); 261 virtual bool AddRecvStream(const StreamParams& sp); 262 virtual bool RemoveRecvStream(uint32 ssrc); 263 virtual bool SetRenderer(uint32 ssrc, VideoRenderer* renderer); 264 virtual bool GetStats(VideoMediaInfo* info); 265 virtual bool SetCapturer(uint32 ssrc, VideoCapturer* capturer); 266 virtual bool SendIntraFrame(); 267 virtual bool RequestIntraFrame(); 268 269 virtual void OnPacketReceived(talk_base::Buffer* packet, 270 const talk_base::PacketTime& packet_time); 271 virtual void OnRtcpReceived(talk_base::Buffer* packet, 272 const talk_base::PacketTime& packet_time); 273 virtual void OnReadyToSend(bool ready); 274 virtual bool MuteStream(uint32 ssrc, bool on); 275 virtual bool SetRecvRtpHeaderExtensions( 276 const std::vector<RtpHeaderExtension>& extensions); 277 virtual bool SetSendRtpHeaderExtensions( 278 const std::vector<RtpHeaderExtension>& extensions); 279 virtual bool SetSendBandwidth(bool autobw, int bps); 280 virtual bool SetOptions(const VideoOptions &options); 281 virtual bool GetOptions(VideoOptions *options) const { 282 *options = options_; 283 return true; 284 } 285 virtual void SetInterface(NetworkInterface* iface); 286 virtual void UpdateAspectRatio(int ratio_w, int ratio_h); 287 288 // Public functions for use by tests and other specialized code. 289 uint32 send_ssrc() const { return 0; } 290 bool GetRenderer(uint32 ssrc, VideoRenderer** renderer); 291 void SendFrame(VideoCapturer* capturer, const VideoFrame* frame); 292 bool SendFrame(WebRtcVideoChannelSendInfo* channel_info, 293 const VideoFrame* frame, bool is_screencast); 294 295 // Thunk functions for use with HybridVideoEngine 296 void OnLocalFrame(VideoCapturer* capturer, const VideoFrame* frame) { 297 SendFrame(0u, frame, capturer->IsScreencast()); 298 } 299 void OnLocalFrameFormat(VideoCapturer* capturer, const VideoFormat* format) { 300 } 301 302 virtual void OnMessage(talk_base::Message* msg); 303 304 protected: 305 int GetLastEngineError() { return engine()->GetLastEngineError(); } 306 virtual int SendPacket(int channel, const void* data, int len); 307 virtual int SendRTCPPacket(int channel, const void* data, int len); 308 309 private: 310 typedef std::map<uint32, WebRtcVideoChannelRecvInfo*> RecvChannelMap; 311 typedef std::map<uint32, WebRtcVideoChannelSendInfo*> SendChannelMap; 312 typedef int (webrtc::ViERTP_RTCP::* ExtensionSetterFunction)(int, bool, int); 313 314 enum MediaDirection { MD_RECV, MD_SEND, MD_SENDRECV }; 315 316 // Creates and initializes a ViE channel. When successful |channel_id| will 317 // contain the new channel's ID. If |receiving| is true |ssrc| is the 318 // remote ssrc. If |sending| is true the ssrc is local ssrc. If both 319 // |receiving| and |sending| is true the ssrc must be 0 and the channel will 320 // be created as a default channel. The ssrc must be different for receive 321 // channels and it must be different for send channels. If the same SSRC is 322 // being used for creating channel more than once, this function will fail 323 // returning false. 324 bool CreateChannel(uint32 ssrc_key, MediaDirection direction, 325 int* channel_id); 326 bool ConfigureChannel(int channel_id, MediaDirection direction, 327 uint32 ssrc_key); 328 bool ConfigureReceiving(int channel_id, uint32 remote_ssrc_key); 329 bool ConfigureSending(int channel_id, uint32 local_ssrc_key); 330 bool SetNackFec(int channel_id, int red_payload_type, int fec_payload_type, 331 bool nack_enabled); 332 bool SetSendCodec(const webrtc::VideoCodec& codec, int min_bitrate, 333 int start_bitrate, int max_bitrate); 334 bool SetSendCodec(WebRtcVideoChannelSendInfo* send_channel, 335 const webrtc::VideoCodec& codec, int min_bitrate, 336 int start_bitrate, int max_bitrate); 337 void LogSendCodecChange(const std::string& reason); 338 // Prepares the channel with channel id |info->channel_id()| to receive all 339 // codecs in |receive_codecs_| and start receive packets. 340 bool SetReceiveCodecs(WebRtcVideoChannelRecvInfo* info); 341 // Returns the channel number that receives the stream with SSRC |ssrc|. 342 int GetRecvChannelNum(uint32 ssrc); 343 // Given captured video frame size, checks if we need to reset vie send codec. 344 // |reset| is set to whether resetting has happened on vie or not. 345 // Returns false on error. 346 bool MaybeResetVieSendCodec(WebRtcVideoChannelSendInfo* send_channel, 347 int new_width, int new_height, bool is_screencast, 348 bool* reset); 349 // Checks the current bitrate estimate and modifies the start bitrate 350 // accordingly. 351 void MaybeChangeStartBitrate(int channel_id, webrtc::VideoCodec* video_codec); 352 // Helper function for starting the sending of media on all channels or 353 // |channel_id|. Note that these two function do not change |sending_|. 354 bool StartSend(); 355 bool StartSend(WebRtcVideoChannelSendInfo* send_channel); 356 // Helper function for stop the sending of media on all channels or 357 // |channel_id|. Note that these two function do not change |sending_|. 358 bool StopSend(); 359 bool StopSend(WebRtcVideoChannelSendInfo* send_channel); 360 bool SendIntraFrame(int channel_id); 361 362 bool HasReadySendChannels(); 363 364 // Send channel key returns the key corresponding to the provided local SSRC 365 // in |key|. The return value is true upon success. 366 // If the local ssrc correspond to that of the default channel the key is 0. 367 // For all other channels the returned key will be the same as the local ssrc. 368 bool GetSendChannelKey(uint32 local_ssrc, uint32* key); 369 WebRtcVideoChannelSendInfo* GetSendChannel(uint32 local_ssrc); 370 // Creates a new unique key that can be used for inserting a new send channel 371 // into |send_channels_| 372 bool CreateSendChannelKey(uint32 local_ssrc, uint32* key); 373 // Get the number of the send channels |capturer| registered with. 374 int GetSendChannelNum(VideoCapturer* capturer); 375 376 bool IsDefaultChannel(int channel_id) const { 377 return channel_id == vie_channel_; 378 } 379 uint32 GetDefaultChannelSsrc(); 380 381 bool DeleteSendChannel(uint32 ssrc_key); 382 383 bool InConferenceMode() const { 384 return options_.conference_mode.GetWithDefaultIfUnset(false); 385 } 386 bool RemoveCapturer(uint32 ssrc); 387 388 389 talk_base::MessageQueue* worker_thread() { return engine_->worker_thread(); } 390 void QueueBlackFrame(uint32 ssrc, int64 timestamp, int framerate); 391 void FlushBlackFrame(uint32 ssrc, int64 timestamp); 392 393 void SetNetworkTransmissionState(bool is_transmitting); 394 395 bool SetHeaderExtension(ExtensionSetterFunction setter, int channel_id, 396 const RtpHeaderExtension* extension); 397 bool SetHeaderExtension(ExtensionSetterFunction setter, int channel_id, 398 const std::vector<RtpHeaderExtension>& extensions, 399 const char header_extension_uri[]); 400 401 // Signal when cpu adaptation has no further scope to adapt. 402 void OnCpuAdaptationUnable(); 403 404 // Set the local (send-side) RTX SSRC corresponding to primary_ssrc. 405 bool SetLocalRtxSsrc(int channel_id, const StreamParams& send_params, 406 uint32 primary_ssrc, int stream_idx); 407 408 // Connect |capturer| to WebRtcVideoMediaChannel if it is only registered 409 // to one send channel, i.e. the first send channel. 410 void MaybeConnectCapturer(VideoCapturer* capturer); 411 // Disconnect |capturer| from WebRtcVideoMediaChannel if it is only registered 412 // to one send channel, i.e. the last send channel. 413 void MaybeDisconnectCapturer(VideoCapturer* capturer); 414 415 // Global state. 416 WebRtcVideoEngine* engine_; 417 VoiceMediaChannel* voice_channel_; 418 int vie_channel_; 419 bool nack_enabled_; 420 // Receiver Estimated Max Bitrate 421 bool remb_enabled_; 422 VideoOptions options_; 423 424 // Global recv side state. 425 // Note the default channel (vie_channel_), i.e. the send channel 426 // corresponding to all the receive channels (this must be done for REMB to 427 // work properly), resides in both recv_channels_ and send_channels_ with the 428 // ssrc key 0. 429 RecvChannelMap recv_channels_; // Contains all receive channels. 430 std::vector<webrtc::VideoCodec> receive_codecs_; 431 bool render_started_; 432 uint32 first_receive_ssrc_; 433 std::vector<RtpHeaderExtension> receive_extensions_; 434 435 // Global send side state. 436 SendChannelMap send_channels_; 437 talk_base::scoped_ptr<webrtc::VideoCodec> send_codec_; 438 int send_rtx_type_; 439 int send_red_type_; 440 int send_fec_type_; 441 int send_min_bitrate_; 442 int send_start_bitrate_; 443 int send_max_bitrate_; 444 bool sending_; 445 std::vector<RtpHeaderExtension> send_extensions_; 446 447 // The aspect ratio that the channel desires. 0 means there is no desired 448 // aspect ratio 449 int ratio_w_; 450 int ratio_h_; 451 }; 452 453 } // namespace cricket 454 455 #endif // TALK_MEDIA_WEBRTCVIDEOENGINE_H_ 456
