1 /* 2 * libjingle 3 * Copyright 2004--2010, Google Inc. 4 * 5 * Redistribution and use in source and binary forms, with or without 6 * modification, are permitted provided that the following conditions are met: 7 * 8 * 1. Redistributions of source code must retain the above copyright notice, 9 * this list of conditions and the following disclaimer. 10 * 2. Redistributions in binary form must reproduce the above copyright notice, 11 * this list of conditions and the following disclaimer in the documentation 12 * and/or other materials provided with the distribution. 13 * 3. The name of the author may not be used to endorse or promote products 14 * derived from this software without specific prior written permission. 15 * 16 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR IMPLIED 17 * WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF 18 * MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO 19 * EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 20 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, 21 * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; 22 * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, 23 * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR 24 * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF 25 * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 26 */ 27 28 #ifndef TALK_SESSION_PHONE_MEDIACHANNEL_H_ 29 #define TALK_SESSION_PHONE_MEDIACHANNEL_H_ 30 31 #include <string> 32 #include <vector> 33 34 #include "talk/base/basictypes.h" 35 #include "talk/base/sigslot.h" 36 #include "talk/base/socket.h" 37 #include "talk/session/phone/codec.h" 38 // TODO: re-evaluate this include 39 #include "talk/session/phone/audiomonitor.h" 40 41 namespace talk_base { 42 class Buffer; 43 } 44 45 namespace flute { 46 class MagicCamVideoRenderer; 47 } 48 49 namespace cricket { 50 51 const size_t kMinRtpPacketLen = 12; 52 const size_t kMinRtcpPacketLen = 4; 53 const size_t kMaxRtpPacketLen = 2048; 54 55 enum VoiceMediaChannelOptions { 56 OPT_CONFERENCE = 0x10000, // tune the audio stream for conference mode 57 OPT_ENERGYLEVEL = 0x20000, // include the energy level in RTP packets, as 58 // defined in https://datatracker.ietf.org/drafts/ 59 // draft-lennox-avt-rtp-audio-level-exthdr/ 60 61 }; 62 63 enum VideoMediaChannelOptions { 64 OPT_INTERPOLATE = 0x10000 // Increase the output framerate by 2x by 65 // interpolating frames 66 }; 67 68 class MediaChannel : public sigslot::has_slots<> { 69 public: 70 class NetworkInterface { 71 public: 72 enum SocketType { ST_RTP, ST_RTCP }; 73 virtual bool SendPacket(talk_base::Buffer* packet) = 0; 74 virtual bool SendRtcp(talk_base::Buffer* packet) = 0; 75 virtual int SetOption(SocketType type, talk_base::Socket::Option opt, 76 int option) = 0; 77 virtual ~NetworkInterface() {} 78 }; 79 80 MediaChannel() : network_interface_(NULL) {} 81 virtual ~MediaChannel() {} 82 83 // Gets/sets the abstract inteface class for sending RTP/RTCP data. 84 NetworkInterface *network_interface() { return network_interface_; } 85 virtual void SetInterface(NetworkInterface *iface) { 86 network_interface_ = iface; 87 } 88 89 // Called when a RTP packet is received. 90 virtual void OnPacketReceived(talk_base::Buffer* packet) = 0; 91 // Called when a RTCP packet is received. 92 virtual void OnRtcpReceived(talk_base::Buffer* packet) = 0; 93 // Sets the SSRC to be used for outgoing data. 94 virtual void SetSendSsrc(uint32 id) = 0; 95 // Set the CNAME of RTCP 96 virtual bool SetRtcpCName(const std::string& cname) = 0; 97 // Mutes the channel. 98 virtual bool Mute(bool on) = 0; 99 100 virtual bool SetRtpExtensionHeaders(bool enable_all) { return true; } 101 virtual bool SetSendBandwidth(bool autobw, int bps) = 0; 102 virtual bool SetOptions(int options) = 0; 103 104 protected: 105 NetworkInterface *network_interface_; 106 }; 107 108 enum SendFlags { 109 SEND_NOTHING, 110 SEND_RINGBACKTONE, 111 SEND_MICROPHONE 112 }; 113 114 struct VoiceSenderInfo { 115 uint32 ssrc; 116 int bytes_sent; 117 int packets_sent; 118 int packets_lost; 119 float fraction_lost; 120 int ext_seqnum; 121 int rtt_ms; 122 int jitter_ms; 123 int audio_level; 124 }; 125 126 struct VoiceReceiverInfo { 127 uint32 ssrc; 128 int bytes_rcvd; 129 int packets_rcvd; 130 int packets_lost; 131 float fraction_lost; 132 int ext_seqnum; 133 int jitter_ms; 134 int audio_level; 135 }; 136 137 struct VideoSenderInfo { 138 uint32 ssrc; 139 int bytes_sent; 140 int packets_sent; 141 int packets_cached; 142 int packets_lost; 143 float fraction_lost; 144 int firs_rcvd; 145 int nacks_rcvd; 146 int rtt_ms; 147 int frame_width; 148 int frame_height; 149 int framerate_input; 150 int framerate_sent; 151 }; 152 153 struct VideoReceiverInfo { 154 uint32 ssrc; 155 int bytes_rcvd; 156 // vector<int> layer_bytes_rcvd; 157 int packets_rcvd; 158 int packets_lost; 159 int packets_concealed; 160 float fraction_lost; 161 int firs_sent; 162 int nacks_sent; 163 int frame_width; 164 int frame_height; 165 int framerate_rcvd; 166 int framerate_decoded; 167 int framerate_output; 168 }; 169 170 struct VoiceMediaInfo { 171 void Clear() { 172 senders.clear(); 173 receivers.clear(); 174 } 175 std::vector<VoiceSenderInfo> senders; 176 std::vector<VoiceReceiverInfo> receivers; 177 }; 178 179 struct VideoMediaInfo { 180 void Clear() { 181 senders.clear(); 182 receivers.clear(); 183 } 184 std::vector<VideoSenderInfo> senders; 185 std::vector<VideoReceiverInfo> receivers; 186 }; 187 188 class VoiceMediaChannel : public MediaChannel { 189 public: 190 enum Error { 191 ERROR_NONE = 0, // No error. 192 ERROR_OTHER, // Other errors. 193 ERROR_REC_DEVICE_OPEN_FAILED = 100, // Could not open mic. 194 ERROR_REC_DEVICE_MUTED, // Mic was muted by OS. 195 ERROR_REC_DEVICE_SILENT, // No background noise picked up. 196 ERROR_REC_DEVICE_SATURATION, // Mic input is clipping. 197 ERROR_REC_DEVICE_REMOVED, // Mic was removed while active. 198 ERROR_REC_RUNTIME_ERROR, // Processing is encountering errors. 199 ERROR_REC_SRTP_ERROR, // Generic SRTP failure. 200 ERROR_PLAY_DEVICE_OPEN_FAILED = 200, // Could not open playout. 201 ERROR_PLAY_DEVICE_MUTED, // Playout muted by OS. 202 ERROR_PLAY_DEVICE_REMOVED, // Playout removed while active. 203 ERROR_PLAY_RUNTIME_ERROR, // Errors in voice processing. 204 ERROR_PLAY_SRTP_ERROR, // Generic SRTP failure. 205 ERROR_PLAY_SRTP_AUTH_FAILED, // Failed to authenticate packets. 206 }; 207 208 VoiceMediaChannel() {} 209 virtual ~VoiceMediaChannel() {} 210 // Sets the codecs/payload types to be used for incoming media. 211 virtual bool SetRecvCodecs(const std::vector<AudioCodec>& codecs) = 0; 212 // Sets the codecs/payload types to be used for outgoing media. 213 virtual bool SetSendCodecs(const std::vector<AudioCodec>& codecs) = 0; 214 // Starts or stops playout of received audio. 215 virtual bool SetPlayout(bool playout) = 0; 216 // Starts or stops sending (and potentially capture) of local audio. 217 virtual bool SetSend(SendFlags flag) = 0; 218 // Adds a new receive-only stream with the specified SSRC. 219 virtual bool AddStream(uint32 ssrc) = 0; 220 // Removes a stream added with AddStream. 221 virtual bool RemoveStream(uint32 ssrc) = 0; 222 // Gets current energy levels for all incoming streams. 223 virtual bool GetActiveStreams(AudioInfo::StreamList* actives) = 0; 224 // Get the current energy level for the outgoing stream. 225 virtual int GetOutputLevel() = 0; 226 // Specifies a ringback tone to be played during call setup. 227 virtual void SetRingbackTone(const char *buf, int len) = 0; 228 // Plays or stops the aforementioned ringback tone 229 virtual bool PlayRingbackTone(bool play, bool loop) = 0; 230 // Sends a out-of-band DTMF signal using the specified event. 231 virtual bool PressDTMF(int event, bool playout) = 0; 232 // Gets quality stats for the channel. 233 virtual bool GetStats(VoiceMediaInfo* info) = 0; 234 // Gets last reported error for this media channel. 235 virtual void GetLastMediaError(uint32* ssrc, 236 VoiceMediaChannel::Error* error) { 237 ASSERT(error != NULL); 238 *error = ERROR_NONE; 239 } 240 241 // Signal errors from MediaChannel. Arguments are: 242 // ssrc(uint32), and error(VoiceMediaChannel::Error). 243 sigslot::signal2<uint32, VoiceMediaChannel::Error> SignalMediaError; 244 }; 245 246 // Represents a YUV420 (a.k.a. I420) video frame. 247 class VideoFrame { 248 friend class flute::MagicCamVideoRenderer; 249 250 public: 251 VideoFrame() : rendered_(false) {} 252 253 virtual ~VideoFrame() {} 254 255 virtual size_t GetWidth() const = 0; 256 virtual size_t GetHeight() const = 0; 257 virtual const uint8 *GetYPlane() const = 0; 258 virtual const uint8 *GetUPlane() const = 0; 259 virtual const uint8 *GetVPlane() const = 0; 260 virtual uint8 *GetYPlane() = 0; 261 virtual uint8 *GetUPlane() = 0; 262 virtual uint8 *GetVPlane() = 0; 263 virtual int32 GetYPitch() const = 0; 264 virtual int32 GetUPitch() const = 0; 265 virtual int32 GetVPitch() const = 0; 266 267 // For retrieving the aspect ratio of each pixel. Usually this is 1x1, but 268 // the aspect_ratio_idc parameter of H.264 can specify non-square pixels. 269 virtual size_t GetPixelWidth() const = 0; 270 virtual size_t GetPixelHeight() const = 0; 271 272 // TODO: Add a fourcc format here and probably combine VideoFrame 273 // with CapturedFrame. 274 virtual int64 GetElapsedTime() const = 0; 275 virtual int64 GetTimeStamp() const = 0; 276 virtual void SetElapsedTime(int64 elapsed_time) = 0; 277 virtual void SetTimeStamp(int64 time_stamp) = 0; 278 279 // Make a copy of the frame. The frame buffer itself may not be copied, 280 // in which case both the current and new VideoFrame will share a single 281 // reference-counted frame buffer. 282 virtual VideoFrame *Copy() const = 0; 283 284 // Writes the frame into the given frame buffer, provided that it is of 285 // sufficient size. Returns the frame's actual size, regardless of whether 286 // it was written or not (like snprintf). If there is insufficient space, 287 // nothing is written. 288 virtual size_t CopyToBuffer(uint8 *buffer, size_t size) const = 0; 289 290 // Converts the I420 data to RGB of a certain type such as ARGB and ABGR. 291 // Returns the frame's actual size, regardless of whether it was written or 292 // not (like snprintf). Parameters size and pitch_rgb are in units of bytes. 293 // If there is insufficient space, nothing is written. 294 virtual size_t ConvertToRgbBuffer(uint32 to_fourcc, uint8 *buffer, 295 size_t size, size_t pitch_rgb) const = 0; 296 297 // Writes the frame into the given planes, stretched to the given width and 298 // height. The parameter "interpolate" controls whether to interpolate or just 299 // take the nearest-point. The parameter "crop" controls whether to crop this 300 // frame to the aspect ratio of the given dimensions before stretching. 301 virtual void StretchToPlanes(uint8 *y, uint8 *u, uint8 *v, 302 int32 pitchY, int32 pitchU, int32 pitchV, 303 size_t width, size_t height, 304 bool interpolate, bool crop) const = 0; 305 306 // Writes the frame into the given frame buffer, stretched to the given width 307 // and height, provided that it is of sufficient size. Returns the frame's 308 // actual size, regardless of whether it was written or not (like snprintf). 309 // If there is insufficient space, nothing is written. The parameter 310 // "interpolate" controls whether to interpolate or just take the 311 // nearest-point. The parameter "crop" controls whether to crop this frame to 312 // the aspect ratio of the given dimensions before stretching. 313 virtual size_t StretchToBuffer(size_t w, size_t h, uint8 *buffer, size_t size, 314 bool interpolate, bool crop) const = 0; 315 316 // Writes the frame into the target VideoFrame, stretched to the size of that 317 // frame. The parameter "interpolate" controls whether to interpolate or just 318 // take the nearest-point. The parameter "crop" controls whether to crop this 319 // frame to the aspect ratio of the target frame before stretching. 320 virtual void StretchToFrame(VideoFrame *target, bool interpolate, 321 bool crop) const = 0; 322 323 // Stretches the frame to the given size, creating a new VideoFrame object to 324 // hold it. The parameter "interpolate" controls whether to interpolate or 325 // just take the nearest-point. The parameter "crop" controls whether to crop 326 // this frame to the aspect ratio of the given dimensions before stretching. 327 virtual VideoFrame *Stretch(size_t w, size_t h, bool interpolate, 328 bool crop) const = 0; 329 330 // Size of an I420 image of given dimensions when stored as a frame buffer. 331 static size_t SizeOf(size_t w, size_t h) { 332 return w * h + ((w + 1) / 2) * ((h + 1) / 2) * 2; 333 } 334 335 protected: 336 // The frame needs to be rendered to magiccam only once. 337 // TODO: Remove this flag once magiccam rendering is fully replaced 338 // by client3d rendering. 339 mutable bool rendered_; 340 }; 341 342 // Simple subclass for use in mocks. 343 class NullVideoFrame : public VideoFrame { 344 public: 345 virtual size_t GetWidth() const { return 0; } 346 virtual size_t GetHeight() const { return 0; } 347 virtual const uint8 *GetYPlane() const { return NULL; } 348 virtual const uint8 *GetUPlane() const { return NULL; } 349 virtual const uint8 *GetVPlane() const { return NULL; } 350 virtual uint8 *GetYPlane() { return NULL; } 351 virtual uint8 *GetUPlane() { return NULL; } 352 virtual uint8 *GetVPlane() { return NULL; } 353 virtual int32 GetYPitch() const { return 0; } 354 virtual int32 GetUPitch() const { return 0; } 355 virtual int32 GetVPitch() const { return 0; } 356 357 virtual size_t GetPixelWidth() const { return 1; } 358 virtual size_t GetPixelHeight() const { return 1; } 359 virtual int64 GetElapsedTime() const { return 0; } 360 virtual int64 GetTimeStamp() const { return 0; } 361 virtual void SetElapsedTime(int64 elapsed_time) {} 362 virtual void SetTimeStamp(int64 time_stamp) {} 363 364 virtual VideoFrame *Copy() const { 365 return NULL; 366 } 367 368 virtual size_t CopyToBuffer(uint8 *buffer, size_t size) const { 369 return 0; 370 } 371 372 virtual size_t ConvertToRgbBuffer(uint32 to_fourcc, uint8 *buffer, 373 size_t size, size_t pitch_rgb) const { 374 return 0; 375 } 376 377 virtual void StretchToPlanes(uint8 *y, uint8 *u, uint8 *v, 378 int32 pitchY, int32 pitchU, int32 pitchV, 379 size_t width, size_t height, 380 bool interpolate, bool crop) const { 381 } 382 383 virtual size_t StretchToBuffer(size_t w, size_t h, uint8 *buffer, size_t size, 384 bool interpolate, bool crop) const { 385 return 0; 386 } 387 388 virtual void StretchToFrame(VideoFrame *target, bool interpolate, 389 bool crop) const { 390 } 391 392 virtual VideoFrame *Stretch(size_t w, size_t h, bool interpolate, 393 bool crop) const { 394 return NULL; 395 } 396 }; 397 398 // Abstract interface for rendering VideoFrames. 399 class VideoRenderer { 400 public: 401 virtual ~VideoRenderer() {} 402 // Called when the video has changed size. 403 virtual bool SetSize(int width, int height, int reserved) = 0; 404 // Called when a new frame is available for display. 405 virtual bool RenderFrame(const VideoFrame *frame) = 0; 406 }; 407 408 // Simple implementation for use in tests. 409 class NullVideoRenderer : public VideoRenderer { 410 virtual bool SetSize(int width, int height, int reserved) { 411 return true; 412 } 413 // Called when a new frame is available for display. 414 virtual bool RenderFrame(const VideoFrame *frame) { 415 return true; 416 } 417 }; 418 419 class VideoMediaChannel : public MediaChannel { 420 public: 421 enum Error { 422 ERROR_NONE = 0, // No error. 423 ERROR_OTHER, // Other errors. 424 ERROR_REC_DEVICE_OPEN_FAILED = 100, // Could not open camera. 425 ERROR_REC_DEVICE_NO_DEVICE, // No camera. 426 ERROR_REC_DEVICE_IN_USE, // Device is in already use. 427 ERROR_REC_DEVICE_REMOVED, // Device is removed. 428 ERROR_REC_SRTP_ERROR, // Generic sender SRTP failure. 429 ERROR_PLAY_SRTP_ERROR = 200, // Generic receiver SRTP failure. 430 ERROR_PLAY_SRTP_AUTH_FAILED, // Failed to authenticate packets. 431 }; 432 433 VideoMediaChannel() { renderer_ = NULL; } 434 virtual ~VideoMediaChannel() {} 435 // Sets the codecs/payload types to be used for incoming media. 436 virtual bool SetRecvCodecs(const std::vector<VideoCodec> &codecs) = 0; 437 // Sets the codecs/payload types to be used for outgoing media. 438 virtual bool SetSendCodecs(const std::vector<VideoCodec> &codecs) = 0; 439 // Starts or stops playout of received video. 440 virtual bool SetRender(bool render) = 0; 441 // Starts or stops transmission (and potentially capture) of local video. 442 virtual bool SetSend(bool send) = 0; 443 // Adds a new receive-only stream with the specified SSRC. 444 virtual bool AddStream(uint32 ssrc, uint32 voice_ssrc) = 0; 445 // Removes a stream added with AddStream. 446 virtual bool RemoveStream(uint32 ssrc) = 0; 447 // Sets the renderer object to be used for the specified stream. 448 // If SSRC is 0, the renderer is used for the 'default' stream. 449 virtual bool SetRenderer(uint32 ssrc, VideoRenderer* renderer) = 0; 450 // Gets quality stats for the channel. 451 virtual bool GetStats(VideoMediaInfo* info) = 0; 452 453 // Send an intra frame to the receivers. 454 virtual bool SendIntraFrame() = 0; 455 // Reuqest each of the remote senders to send an intra frame. 456 virtual bool RequestIntraFrame() = 0; 457 458 sigslot::signal2<uint32, Error> SignalMediaError; 459 460 protected: 461 VideoRenderer *renderer_; 462 }; 463 464 } // namespace cricket 465 466 #endif // TALK_SESSION_PHONE_MEDIACHANNEL_H_ 467
