1 /* 2 * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. 3 * 4 * Use of this source code is governed by a BSD-style license 5 * that can be found in the LICENSE file in the root of the source 6 * tree. An additional intellectual property rights grant can be found 7 * in the file PATENTS. All contributing project authors may 8 * be found in the AUTHORS file in the root of the source tree. 9 */ 10 11 #ifndef WEBRTC_MODULES_INTERFACE_VIDEO_CODING_H_ 12 #define WEBRTC_MODULES_INTERFACE_VIDEO_CODING_H_ 13 14 #include "webrtc/common_video/interface/i420_video_frame.h" 15 #include "webrtc/modules/interface/module.h" 16 #include "webrtc/modules/interface/module_common_types.h" 17 #include "webrtc/modules/video_coding/main/interface/video_coding_defines.h" 18 #include "webrtc/system_wrappers/interface/event_wrapper.h" 19 20 namespace webrtc 21 { 22 23 class Clock; 24 class EncodedImageCallback; 25 class VideoEncoder; 26 class VideoDecoder; 27 struct CodecSpecificInfo; 28 29 class EventFactory { 30 public: 31 virtual ~EventFactory() {} 32 33 virtual EventWrapper* CreateEvent() = 0; 34 }; 35 36 class EventFactoryImpl : public EventFactory { 37 public: 38 virtual ~EventFactoryImpl() {} 39 40 virtual EventWrapper* CreateEvent() { 41 return EventWrapper::Create(); 42 } 43 }; 44 45 // Used to indicate which decode with errors mode should be used. 46 enum VCMDecodeErrorMode { 47 kNoErrors, // Never decode with errors. Video will freeze 48 // if nack is disabled. 49 kSelectiveErrors, // Frames that are determined decodable in 50 // VCMSessionInfo may be decoded with missing 51 // packets. As not all incomplete frames will be 52 // decodable, video will freeze if nack is disabled. 53 kWithErrors // Release frames as needed. Errors may be 54 // introduced as some encoded frames may not be 55 // complete. 56 }; 57 58 class VideoCodingModule : public Module 59 { 60 public: 61 enum SenderNackMode { 62 kNackNone, 63 kNackAll, 64 kNackSelective 65 }; 66 67 enum ReceiverRobustness { 68 kNone, 69 kHardNack, 70 kSoftNack, 71 kDualDecoder, 72 kReferenceSelection 73 }; 74 75 static VideoCodingModule* Create(); 76 77 static VideoCodingModule* Create(Clock* clock, EventFactory* event_factory); 78 79 static void Destroy(VideoCodingModule* module); 80 81 // Get number of supported codecs 82 // 83 // Return value : Number of supported codecs 84 static uint8_t NumberOfCodecs(); 85 86 // Get supported codec settings with using id 87 // 88 // Input: 89 // - listId : Id or index of the codec to look up 90 // - codec : Memory where the codec settings will be stored 91 // 92 // Return value : VCM_OK, on success 93 // VCM_PARAMETER_ERROR if codec not supported or id too high 94 static int32_t Codec(const uint8_t listId, VideoCodec* codec); 95 96 // Get supported codec settings using codec type 97 // 98 // Input: 99 // - codecType : The codec type to get settings for 100 // - codec : Memory where the codec settings will be stored 101 // 102 // Return value : VCM_OK, on success 103 // VCM_PARAMETER_ERROR if codec not supported 104 static int32_t Codec(VideoCodecType codecType, VideoCodec* codec); 105 106 /* 107 * Sender 108 */ 109 110 // Any encoder-related state of VCM will be initialized to the 111 // same state as when the VCM was created. This will not interrupt 112 // or effect decoding functionality of VCM. VCM will lose all the 113 // encoding-related settings by calling this function. 114 // For instance, a send codec has to be registered again. 115 // 116 // Return value : VCM_OK, on success. 117 // < 0, on error. 118 virtual int32_t InitializeSender() = 0; 119 120 // Registers a codec to be used for encoding. Calling this 121 // API multiple times overwrites any previously registered codecs. 122 // 123 // Input: 124 // - sendCodec : Settings for the codec to be registered. 125 // - numberOfCores : The number of cores the codec is allowed 126 // to use. 127 // - maxPayloadSize : The maximum size each payload is allowed 128 // to have. Usually MTU - overhead. 129 // 130 // Return value : VCM_OK, on success. 131 // < 0, on error. 132 virtual int32_t RegisterSendCodec(const VideoCodec* sendCodec, 133 uint32_t numberOfCores, 134 uint32_t maxPayloadSize) = 0; 135 136 // API to get the current send codec in use. 137 // 138 // Input: 139 // - currentSendCodec : Address where the sendCodec will be written. 140 // 141 // Return value : VCM_OK, on success. 142 // < 0, on error. 143 virtual int32_t SendCodec(VideoCodec* currentSendCodec) const = 0; 144 145 // API to get the current send codec type 146 // 147 // Return value : Codec type, on success. 148 // kVideoCodecUnknown, on error or if no send codec is set 149 virtual VideoCodecType SendCodec() const = 0; 150 151 // Register an external encoder object. This can not be used together with 152 // external decoder callbacks. 153 // 154 // Input: 155 // - externalEncoder : Encoder object to be used for encoding frames inserted 156 // with the AddVideoFrame API. 157 // - payloadType : The payload type bound which this encoder is bound to. 158 // 159 // Return value : VCM_OK, on success. 160 // < 0, on error. 161 virtual int32_t RegisterExternalEncoder(VideoEncoder* externalEncoder, 162 uint8_t payloadType, 163 bool internalSource = false) = 0; 164 165 // API to get codec config parameters to be sent out-of-band to a receiver. 166 // 167 // Input: 168 // - buffer : Memory where the codec config parameters should be written. 169 // - size : Size of the memory available. 170 // 171 // Return value : Number of bytes written, on success. 172 // < 0, on error. 173 virtual int32_t CodecConfigParameters(uint8_t* buffer, int32_t size) = 0; 174 175 // API to get currently configured encoder target bitrate in bits/s. 176 // 177 // Return value : 0, on success. 178 // < 0, on error. 179 virtual int Bitrate(unsigned int* bitrate) const = 0; 180 181 // API to get currently configured encoder target frame rate. 182 // 183 // Return value : 0, on success. 184 // < 0, on error. 185 virtual int FrameRate(unsigned int* framerate) const = 0; 186 187 // Sets the parameters describing the send channel. These parameters are inputs to the 188 // Media Optimization inside the VCM and also specifies the target bit rate for the 189 // encoder. Bit rate used by NACK should already be compensated for by the user. 190 // 191 // Input: 192 // - target_bitrate : The target bitrate for VCM in bits/s. 193 // - lossRate : Fractions of lost packets the past second. 194 // (loss rate in percent = 100 * packetLoss / 255) 195 // - rtt : Current round-trip time in ms. 196 // 197 // Return value : VCM_OK, on success. 198 // < 0, on error. 199 virtual int32_t SetChannelParameters(uint32_t target_bitrate, 200 uint8_t lossRate, 201 uint32_t rtt) = 0; 202 203 // Sets the parameters describing the receive channel. These parameters are inputs to the 204 // Media Optimization inside the VCM. 205 // 206 // Input: 207 // - rtt : Current round-trip time in ms. 208 // with the most amount available bandwidth in a conference 209 // scenario 210 // 211 // Return value : VCM_OK, on success. 212 // < 0, on error. 213 virtual int32_t SetReceiveChannelParameters(uint32_t rtt) = 0; 214 215 // Register a transport callback which will be called to deliver the encoded data and 216 // side information. 217 // 218 // Input: 219 // - transport : The callback object to register. 220 // 221 // Return value : VCM_OK, on success. 222 // < 0, on error. 223 virtual int32_t RegisterTransportCallback(VCMPacketizationCallback* transport) = 0; 224 225 // Register video output information callback which will be called to deliver information 226 // about the video stream produced by the encoder, for instance the average frame rate and 227 // bit rate. 228 // 229 // Input: 230 // - outputInformation : The callback object to register. 231 // 232 // Return value : VCM_OK, on success. 233 // < 0, on error. 234 virtual int32_t RegisterSendStatisticsCallback( 235 VCMSendStatisticsCallback* sendStats) = 0; 236 237 // Register a video quality settings callback which will be called when 238 // frame rate/dimensions need to be updated for video quality optimization 239 // 240 // Input: 241 // - videoQMSettings : The callback object to register. 242 // 243 // Return value : VCM_OK, on success. 244 // < 0, on error 245 virtual int32_t RegisterVideoQMCallback(VCMQMSettingsCallback* videoQMSettings) = 0; 246 247 // Register a video protection callback which will be called to deliver 248 // the requested FEC rate and NACK status (on/off). 249 // 250 // Input: 251 // - protection : The callback object to register. 252 // 253 // Return value : VCM_OK, on success. 254 // < 0, on error. 255 virtual int32_t RegisterProtectionCallback(VCMProtectionCallback* protection) = 0; 256 257 // Enable or disable a video protection method. 258 // 259 // Input: 260 // - videoProtection : The method to enable or disable. 261 // - enable : True if the method should be enabled, false if 262 // it should be disabled. 263 // 264 // Return value : VCM_OK, on success. 265 // < 0, on error. 266 virtual int32_t SetVideoProtection(VCMVideoProtection videoProtection, 267 bool enable) = 0; 268 269 // Add one raw video frame to the encoder. This function does all the necessary 270 // processing, then decides what frame type to encode, or if the frame should be 271 // dropped. If the frame should be encoded it passes the frame to the encoder 272 // before it returns. 273 // 274 // Input: 275 // - videoFrame : Video frame to encode. 276 // - codecSpecificInfo : Extra codec information, e.g., pre-parsed in-band signaling. 277 // 278 // Return value : VCM_OK, on success. 279 // < 0, on error. 280 virtual int32_t AddVideoFrame( 281 const I420VideoFrame& videoFrame, 282 const VideoContentMetrics* contentMetrics = NULL, 283 const CodecSpecificInfo* codecSpecificInfo = NULL) = 0; 284 285 // Next frame encoded should be an intra frame (keyframe). 286 // 287 // Return value : VCM_OK, on success. 288 // < 0, on error. 289 virtual int32_t IntraFrameRequest(int stream_index) = 0; 290 291 // Frame Dropper enable. Can be used to disable the frame dropping when the encoder 292 // over-uses its bit rate. This API is designed to be used when the encoded frames 293 // are supposed to be stored to an AVI file, or when the I420 codec is used and the 294 // target bit rate shouldn't affect the frame rate. 295 // 296 // Input: 297 // - enable : True to enable the setting, false to disable it. 298 // 299 // Return value : VCM_OK, on success. 300 // < 0, on error. 301 virtual int32_t EnableFrameDropper(bool enable) = 0; 302 303 // Sent frame counters 304 virtual int32_t SentFrameCount(VCMFrameCount& frameCount) const = 0; 305 306 /* 307 * Receiver 308 */ 309 310 // The receiver state of the VCM will be initialized to the 311 // same state as when the VCM was created. This will not interrupt 312 // or effect the send side functionality of VCM. VCM will lose all the 313 // decoding-related settings by calling this function. All frames 314 // inside the jitter buffer are flushed and the delay is reset. 315 // For instance, a receive codec has to be registered again. 316 // 317 // Return value : VCM_OK, on success. 318 // < 0, on error. 319 virtual int32_t InitializeReceiver() = 0; 320 321 // Register possible receive codecs, can be called multiple times for different codecs. 322 // The module will automatically switch between registered codecs depending on the 323 // payload type of incoming frames. The actual decoder will be created when needed. 324 // 325 // Input: 326 // - receiveCodec : Settings for the codec to be registered. 327 // - numberOfCores : Number of CPU cores that the decoder is allowed to use. 328 // - requireKeyFrame : Set this to true if you don't want any delta frames 329 // to be decoded until the first key frame has been decoded. 330 // 331 // Return value : VCM_OK, on success. 332 // < 0, on error. 333 virtual int32_t RegisterReceiveCodec(const VideoCodec* receiveCodec, 334 int32_t numberOfCores, 335 bool requireKeyFrame = false) = 0; 336 337 // Register an externally defined decoder/renderer object. Can be a decoder only or a 338 // decoder coupled with a renderer. Note that RegisterReceiveCodec must be called to 339 // be used for decoding incoming streams. 340 // 341 // Input: 342 // - externalDecoder : The external decoder/renderer object. 343 // - payloadType : The payload type which this decoder should be 344 // registered to. 345 // - internalRenderTiming : True if the internal renderer (if any) of the decoder 346 // object can make sure to render at a given time in ms. 347 // 348 // Return value : VCM_OK, on success. 349 // < 0, on error. 350 virtual int32_t RegisterExternalDecoder(VideoDecoder* externalDecoder, 351 uint8_t payloadType, 352 bool internalRenderTiming) = 0; 353 354 // Register a receive callback. Will be called whenever there is a new frame ready 355 // for rendering. 356 // 357 // Input: 358 // - receiveCallback : The callback object to be used by the module when a 359 // frame is ready for rendering. 360 // De-register with a NULL pointer. 361 // 362 // Return value : VCM_OK, on success. 363 // < 0, on error. 364 virtual int32_t RegisterReceiveCallback(VCMReceiveCallback* receiveCallback) = 0; 365 366 // Register a receive statistics callback which will be called to deliver information 367 // about the video stream received by the receiving side of the VCM, for instance the 368 // average frame rate and bit rate. 369 // 370 // Input: 371 // - receiveStats : The callback object to register. 372 // 373 // Return value : VCM_OK, on success. 374 // < 0, on error. 375 virtual int32_t RegisterReceiveStatisticsCallback( 376 VCMReceiveStatisticsCallback* receiveStats) = 0; 377 378 // Register a decoder timing callback which will be called to deliver 379 // information about the timing of the decoder in the receiving side of the 380 // VCM, for instance the current and maximum frame decode latency. 381 // 382 // Input: 383 // - decoderTiming : The callback object to register. 384 // 385 // Return value : VCM_OK, on success. 386 // < 0, on error. 387 virtual int32_t RegisterDecoderTimingCallback( 388 VCMDecoderTimingCallback* decoderTiming) = 0; 389 390 // Register a frame type request callback. This callback will be called when the 391 // module needs to request specific frame types from the send side. 392 // 393 // Input: 394 // - frameTypeCallback : The callback object to be used by the module when 395 // requesting a specific type of frame from the send side. 396 // De-register with a NULL pointer. 397 // 398 // Return value : VCM_OK, on success. 399 // < 0, on error. 400 virtual int32_t RegisterFrameTypeCallback( 401 VCMFrameTypeCallback* frameTypeCallback) = 0; 402 403 // Registers a callback which is called whenever the receive side of the VCM 404 // encounters holes in the packet sequence and needs packets to be retransmitted. 405 // 406 // Input: 407 // - callback : The callback to be registered in the VCM. 408 // 409 // Return value : VCM_OK, on success. 410 // <0, on error. 411 virtual int32_t RegisterPacketRequestCallback( 412 VCMPacketRequestCallback* callback) = 0; 413 414 // Waits for the next frame in the jitter buffer to become complete 415 // (waits no longer than maxWaitTimeMs), then passes it to the decoder for decoding. 416 // Should be called as often as possible to get the most out of the decoder. 417 // 418 // Return value : VCM_OK, on success. 419 // < 0, on error. 420 virtual int32_t Decode(uint16_t maxWaitTimeMs = 200) = 0; 421 422 // Registers a callback which conveys the size of the render buffer. 423 virtual int RegisterRenderBufferSizeCallback( 424 VCMRenderBufferSizeCallback* callback) = 0; 425 426 // Waits for the next frame in the dual jitter buffer to become complete 427 // (waits no longer than maxWaitTimeMs), then passes it to the dual decoder 428 // for decoding. This will never trigger a render callback. Should be 429 // called frequently, and as long as it returns 1 it should be called again 430 // as soon as possible. 431 // 432 // Return value : 1, if a frame was decoded 433 // 0, if no frame was decoded 434 // < 0, on error. 435 virtual int32_t DecodeDualFrame(uint16_t maxWaitTimeMs = 200) = 0; 436 437 // Reset the decoder state to the initial state. 438 // 439 // Return value : VCM_OK, on success. 440 // < 0, on error. 441 virtual int32_t ResetDecoder() = 0; 442 443 // API to get the codec which is currently used for decoding by the module. 444 // 445 // Input: 446 // - currentReceiveCodec : Settings for the codec to be registered. 447 // 448 // Return value : VCM_OK, on success. 449 // < 0, on error. 450 virtual int32_t ReceiveCodec(VideoCodec* currentReceiveCodec) const = 0; 451 452 // API to get the codec type currently used for decoding by the module. 453 // 454 // Return value : codecy type, on success. 455 // kVideoCodecUnknown, on error or if no receive codec is registered 456 virtual VideoCodecType ReceiveCodec() const = 0; 457 458 // Insert a parsed packet into the receiver side of the module. Will be placed in the 459 // jitter buffer waiting for the frame to become complete. Returns as soon as the packet 460 // has been placed in the jitter buffer. 461 // 462 // Input: 463 // - incomingPayload : Payload of the packet. 464 // - payloadLength : Length of the payload. 465 // - rtpInfo : The parsed header. 466 // 467 // Return value : VCM_OK, on success. 468 // < 0, on error. 469 virtual int32_t IncomingPacket(const uint8_t* incomingPayload, 470 uint32_t payloadLength, 471 const WebRtcRTPHeader& rtpInfo) = 0; 472 473 // Minimum playout delay (Used for lip-sync). This is the minimum delay required 474 // to sync with audio. Not included in VideoCodingModule::Delay() 475 // Defaults to 0 ms. 476 // 477 // Input: 478 // - minPlayoutDelayMs : Additional delay in ms. 479 // 480 // Return value : VCM_OK, on success. 481 // < 0, on error. 482 virtual int32_t SetMinimumPlayoutDelay(uint32_t minPlayoutDelayMs) = 0; 483 484 // Set the time required by the renderer to render a frame. 485 // 486 // Input: 487 // - timeMS : The time in ms required by the renderer to render a frame. 488 // 489 // Return value : VCM_OK, on success. 490 // < 0, on error. 491 virtual int32_t SetRenderDelay(uint32_t timeMS) = 0; 492 493 // The total delay desired by the VCM. Can be less than the minimum 494 // delay set with SetMinimumPlayoutDelay. 495 // 496 // Return value : Total delay in ms, on success. 497 // < 0, on error. 498 virtual int32_t Delay() const = 0; 499 500 // Get the received frame counters. Keeps track of the number of each frame type 501 // received since the start of the call. 502 // 503 // Output: 504 // - frameCount : Struct to be filled with the number of frames received. 505 // 506 // Return value : VCM_OK, on success. 507 // <0, on error. 508 virtual int32_t ReceivedFrameCount(VCMFrameCount& frameCount) const = 0; 509 510 // Returns the number of packets discarded by the jitter buffer due to being 511 // too late. This can include duplicated packets which arrived after the 512 // frame was sent to the decoder. Therefore packets which were prematurely 513 // NACKed will be counted. 514 virtual uint32_t DiscardedPackets() const = 0; 515 516 517 // Robustness APIs 518 519 // Set the sender RTX/NACK mode. 520 // Input: 521 // - mode : the selected NACK mode. 522 // 523 // Return value : VCM_OK, on success; 524 // < 0, on error. 525 virtual int SetSenderNackMode(SenderNackMode mode) = 0; 526 527 // Set the sender reference picture selection (RPS) mode. 528 // Input: 529 // - enable : true or false, for enable and disable, respectively. 530 // 531 // Return value : VCM_OK, on success; 532 // < 0, on error. 533 virtual int SetSenderReferenceSelection(bool enable) = 0; 534 535 // Set the sender forward error correction (FEC) mode. 536 // Input: 537 // - enable : true or false, for enable and disable, respectively. 538 // 539 // Return value : VCM_OK, on success; 540 // < 0, on error. 541 virtual int SetSenderFEC(bool enable) = 0; 542 543 // Set the key frame period, or disable periodic key frames (I-frames). 544 // Input: 545 // - periodMs : period in ms; <= 0 to disable periodic key frames. 546 // 547 // Return value : VCM_OK, on success; 548 // < 0, on error. 549 virtual int SetSenderKeyFramePeriod(int periodMs) = 0; 550 551 // Set the receiver robustness mode. The mode decides how the receiver 552 // responds to losses in the stream. The type of counter-measure (soft or 553 // hard NACK, dual decoder, RPS, etc.) is selected through the 554 // robustnessMode parameter. The errorMode parameter decides if it is 555 // allowed to display frames corrupted by losses. Note that not all 556 // combinations of the two parameters are feasible. An error will be 557 // returned for invalid combinations. 558 // Input: 559 // - robustnessMode : selected robustness mode. 560 // - errorMode : selected error mode. 561 // 562 // Return value : VCM_OK, on success; 563 // < 0, on error. 564 virtual int SetReceiverRobustnessMode(ReceiverRobustness robustnessMode, 565 VCMDecodeErrorMode errorMode) = 0; 566 567 // Set the decode error mode. The mode decides which errors (if any) are 568 // allowed in decodable frames. Note that setting decode_error_mode to 569 // anything other than kWithErrors without enabling nack will cause 570 // long-term freezes (resulting from frequent key frame requests) if 571 // packet loss occurs. 572 virtual void SetDecodeErrorMode(VCMDecodeErrorMode decode_error_mode) = 0; 573 574 // Sets the maximum number of sequence numbers that we are allowed to NACK 575 // and the oldest sequence number that we will consider to NACK. If a 576 // sequence number older than |max_packet_age_to_nack| is missing 577 // a key frame will be requested. A key frame will also be requested if the 578 // time of incomplete or non-continuous frames in the jitter buffer is above 579 // |max_incomplete_time_ms|. 580 virtual void SetNackSettings(size_t max_nack_list_size, 581 int max_packet_age_to_nack, 582 int max_incomplete_time_ms) = 0; 583 584 // Setting a desired delay to the VCM receiver. Video rendering will be 585 // delayed by at least desired_delay_ms. 586 virtual int SetMinReceiverDelay(int desired_delay_ms) = 0; 587 588 // Enables recording of debugging information. 589 virtual int StartDebugRecording(const char* file_name_utf8) = 0; 590 591 // Disables recording of debugging information. 592 virtual int StopDebugRecording() = 0; 593 594 // Lets the sender suspend video when the rate drops below 595 // |threshold_bps|, and turns back on when the rate goes back up above 596 // |threshold_bps| + |window_bps|. 597 virtual void SuspendBelowMinBitrate() = 0; 598 599 // Returns true if SuspendBelowMinBitrate is engaged and the video has been 600 // suspended due to bandwidth limitations; otherwise false. 601 virtual bool VideoSuspended() const = 0; 602 603 virtual void RegisterPreDecodeImageCallback( 604 EncodedImageCallback* observer) = 0; 605 virtual void RegisterPostEncodeImageCallback( 606 EncodedImageCallback* post_encode_callback) = 0; 607 }; 608 609 } // namespace webrtc 610 611 #endif // WEBRTC_MODULES_INTERFACE_VIDEO_CODING_H_ 612
