Home | History | Annotate | Download | only in video_engine 
      1 /*
      2  *  Copyright (c) 2012 The WebRTC project authors. All Rights Reserved.
      3  *
      4  *  Use of this source code is governed by a BSD-style license
      5  *  that can be found in the LICENSE file in the root of the source
      6  *  tree. An additional intellectual property rights grant can be found
      7  *  in the file PATENTS.  All contributing project authors may
      8  *  be found in the AUTHORS file in the root of the source tree.
      9  */
     10 
     11 #include "webrtc/video_engine/stream_synchronization.h"
     12 
     13 #include <assert.h>
     14 #include <math.h>
     15 #include <stdlib.h>
     16 
     17 #include <algorithm>
     18 
     19 #include "webrtc/system_wrappers/interface/logging.h"
     20 
     21 namespace webrtc {
     22 
     23 static const int kMaxChangeMs = 80;
     24 static const int kMaxDeltaDelayMs = 10000;
     25 static const int kFilterLength = 4;
     26 // Minimum difference between audio and video to warrant a change.
     27 static const int kMinDeltaMs = 30;
     28 
     29 struct ViESyncDelay {
     30   ViESyncDelay() {
     31     extra_video_delay_ms = 0;
     32     last_video_delay_ms = 0;
     33     extra_audio_delay_ms = 0;
     34     last_audio_delay_ms = 0;
     35     network_delay = 120;
     36   }
     37 
     38   int extra_video_delay_ms;
     39   int last_video_delay_ms;
     40   int extra_audio_delay_ms;
     41   int last_audio_delay_ms;
     42   int network_delay;
     43 };
     44 
     45 StreamSynchronization::StreamSynchronization(int audio_channel_id,
     46                                              int video_channel_id)
     47     : channel_delay_(new ViESyncDelay),
     48       audio_channel_id_(audio_channel_id),
     49       video_channel_id_(video_channel_id),
     50       base_target_delay_ms_(0),
     51       avg_diff_ms_(0) {}
     52 
     53 StreamSynchronization::~StreamSynchronization() {
     54   delete channel_delay_;
     55 }
     56 
     57 bool StreamSynchronization::ComputeRelativeDelay(
     58     const Measurements& audio_measurement,
     59     const Measurements& video_measurement,
     60     int* relative_delay_ms) {
     61   assert(relative_delay_ms);
     62   if (audio_measurement.rtcp.size() < 2 || video_measurement.rtcp.size() < 2) {
     63     // We need two RTCP SR reports per stream to do synchronization.
     64     return false;
     65   }
     66   int64_t audio_last_capture_time_ms;
     67   if (!RtpToNtpMs(audio_measurement.latest_timestamp,
     68                   audio_measurement.rtcp,
     69                   &audio_last_capture_time_ms)) {
     70     return false;
     71   }
     72   int64_t video_last_capture_time_ms;
     73   if (!RtpToNtpMs(video_measurement.latest_timestamp,
     74                   video_measurement.rtcp,
     75                   &video_last_capture_time_ms)) {
     76     return false;
     77   }
     78   if (video_last_capture_time_ms < 0) {
     79     return false;
     80   }
     81   // Positive diff means that video_measurement is behind audio_measurement.
     82   *relative_delay_ms = video_measurement.latest_receive_time_ms -
     83       audio_measurement.latest_receive_time_ms -
     84       (video_last_capture_time_ms - audio_last_capture_time_ms);
     85   if (*relative_delay_ms > kMaxDeltaDelayMs ||
     86       *relative_delay_ms < -kMaxDeltaDelayMs) {
     87     return false;
     88   }
     89   return true;
     90 }
     91 
     92 bool StreamSynchronization::ComputeDelays(int relative_delay_ms,
     93                                           int current_audio_delay_ms,
     94                                           int* total_audio_delay_target_ms,
     95                                           int* total_video_delay_target_ms) {
     96   assert(total_audio_delay_target_ms && total_video_delay_target_ms);
     97 
     98   int current_video_delay_ms = *total_video_delay_target_ms;
     99   LOG(LS_VERBOSE) << "Audio delay: " << current_audio_delay_ms
    100                   << ", network delay diff: " << channel_delay_->network_delay
    101                   << " current diff: " << relative_delay_ms
    102                   << " for channel " << audio_channel_id_;
    103   // Calculate the difference between the lowest possible video delay and
    104   // the current audio delay.
    105   int current_diff_ms = current_video_delay_ms - current_audio_delay_ms +
    106       relative_delay_ms;
    107 
    108   avg_diff_ms_ = ((kFilterLength - 1) * avg_diff_ms_ +
    109       current_diff_ms) / kFilterLength;
    110   if (abs(avg_diff_ms_) < kMinDeltaMs) {
    111     // Don't adjust if the diff is within our margin.
    112     return false;
    113   }
    114 
    115   // Make sure we don't move too fast.
    116   int diff_ms = avg_diff_ms_ / 2;
    117   diff_ms = std::min(diff_ms, kMaxChangeMs);
    118   diff_ms = std::max(diff_ms, -kMaxChangeMs);
    119 
    120   // Reset the average after a move to prevent overshooting reaction.
    121   avg_diff_ms_ = 0;
    122 
    123   if (diff_ms > 0) {
    124     // The minimum video delay is longer than the current audio delay.
    125     // We need to decrease extra video delay, or add extra audio delay.
    126     if (channel_delay_->extra_video_delay_ms > base_target_delay_ms_) {
    127       // We have extra delay added to ViE. Reduce this delay before adding
    128       // extra delay to VoE.
    129       channel_delay_->extra_video_delay_ms -= diff_ms;
    130       channel_delay_->extra_audio_delay_ms = base_target_delay_ms_;
    131     } else {  // channel_delay_->extra_video_delay_ms > 0
    132       // We have no extra video delay to remove, increase the audio delay.
    133       channel_delay_->extra_audio_delay_ms += diff_ms;
    134       channel_delay_->extra_video_delay_ms = base_target_delay_ms_;
    135     }
    136   } else {  // if (diff_ms > 0)
    137     // The video delay is lower than the current audio delay.
    138     // We need to decrease extra audio delay, or add extra video delay.
    139     if (channel_delay_->extra_audio_delay_ms > base_target_delay_ms_) {
    140       // We have extra delay in VoiceEngine.
    141       // Start with decreasing the voice delay.
    142       // Note: diff_ms is negative; add the negative difference.
    143       channel_delay_->extra_audio_delay_ms += diff_ms;
    144       channel_delay_->extra_video_delay_ms = base_target_delay_ms_;
    145     } else {  // channel_delay_->extra_audio_delay_ms > base_target_delay_ms_
    146       // We have no extra delay in VoiceEngine, increase the video delay.
    147       // Note: diff_ms is negative; subtract the negative difference.
    148       channel_delay_->extra_video_delay_ms -= diff_ms;  // X - (-Y) = X + Y.
    149       channel_delay_->extra_audio_delay_ms = base_target_delay_ms_;
    150     }
    151   }
    152 
    153   // Make sure that video is never below our target.
    154   channel_delay_->extra_video_delay_ms = std::max(
    155       channel_delay_->extra_video_delay_ms, base_target_delay_ms_);
    156 
    157   int new_video_delay_ms;
    158   if (channel_delay_->extra_video_delay_ms > base_target_delay_ms_) {
    159     new_video_delay_ms = channel_delay_->extra_video_delay_ms;
    160   } else {
    161     // No change to the extra video delay. We are changing audio and we only
    162     // allow to change one at the time.
    163     new_video_delay_ms = channel_delay_->last_video_delay_ms;
    164   }
    165 
    166   // Make sure that we don't go below the extra video delay.
    167   new_video_delay_ms = std::max(
    168       new_video_delay_ms, channel_delay_->extra_video_delay_ms);
    169 
    170   // Verify we don't go above the maximum allowed video delay.
    171   new_video_delay_ms =
    172       std::min(new_video_delay_ms, base_target_delay_ms_ + kMaxDeltaDelayMs);
    173 
    174   int new_audio_delay_ms;
    175   if (channel_delay_->extra_audio_delay_ms > base_target_delay_ms_) {
    176     new_audio_delay_ms = channel_delay_->extra_audio_delay_ms;
    177   } else {
    178     // No change to the audio delay. We are changing video and we only
    179     // allow to change one at the time.
    180     new_audio_delay_ms = channel_delay_->last_audio_delay_ms;
    181   }
    182 
    183   // Make sure that we don't go below the extra audio delay.
    184   new_audio_delay_ms = std::max(
    185       new_audio_delay_ms, channel_delay_->extra_audio_delay_ms);
    186 
    187   // Verify we don't go above the maximum allowed audio delay.
    188   new_audio_delay_ms =
    189       std::min(new_audio_delay_ms, base_target_delay_ms_ + kMaxDeltaDelayMs);
    190 
    191   // Remember our last audio and video delays.
    192   channel_delay_->last_video_delay_ms = new_video_delay_ms;
    193   channel_delay_->last_audio_delay_ms = new_audio_delay_ms;
    194 
    195   LOG(LS_VERBOSE) << "Sync video delay " << new_video_delay_ms
    196                   << " and audio delay " << channel_delay_->extra_audio_delay_ms
    197                   << " for video channel " << video_channel_id_
    198                   << " for audio channel " << audio_channel_id_;
    199 
    200   // Return values.
    201   *total_video_delay_target_ms = new_video_delay_ms;
    202   *total_audio_delay_target_ms = new_audio_delay_ms;
    203   return true;
    204 }
    205 
    206 void StreamSynchronization::SetTargetBufferingDelay(int target_delay_ms) {
    207   // Initial extra delay for audio (accounting for existing extra delay).
    208   channel_delay_->extra_audio_delay_ms +=
    209       target_delay_ms - base_target_delay_ms_;
    210   channel_delay_->last_audio_delay_ms +=
    211       target_delay_ms - base_target_delay_ms_;
    212 
    213   // The video delay is compared to the last value (and how much we can update
    214   // is limited by that as well).
    215   channel_delay_->last_video_delay_ms +=
    216       target_delay_ms - base_target_delay_ms_;
    217 
    218   channel_delay_->extra_video_delay_ms +=
    219       target_delay_ms - base_target_delay_ms_;
    220 
    221   // Video is already delayed by the desired amount.
    222   base_target_delay_ms_ = target_delay_ms;
    223 }
    224 
    225 }  // namespace webrtc
    226