Home | History | Annotate | Download | only in video 
      1 /*
      2  *  Copyright (c) 2012 The WebRTC project authors. All Rights Reserved.
      3  *
      4  *  Use of this source code is governed by a BSD-style license
      5  *  that can be found in the LICENSE file in the root of the source
      6  *  tree. An additional intellectual property rights grant can be found
      7  *  in the file PATENTS.  All contributing project authors may
      8  *  be found in the AUTHORS file in the root of the source tree.
      9  */
     10 
     11 #include "webrtc/video/stream_synchronization.h"
     12 
     13 #include <assert.h>
     14 #include <math.h>
     15 #include <stdlib.h>
     16 
     17 #include <algorithm>
     18 
     19 #include "webrtc/base/logging.h"
     20 
     21 namespace webrtc {
     22 
     23 static const int kMaxChangeMs = 80;
     24 static const int kMaxDeltaDelayMs = 10000;
     25 static const int kFilterLength = 4;
     26 // Minimum difference between audio and video to warrant a change.
     27 static const int kMinDeltaMs = 30;
     28 
     29 struct ViESyncDelay {
     30   ViESyncDelay() {
     31     extra_video_delay_ms = 0;
     32     last_video_delay_ms = 0;
     33     extra_audio_delay_ms = 0;
     34     last_audio_delay_ms = 0;
     35     network_delay = 120;
     36   }
     37 
     38   int extra_video_delay_ms;
     39   int last_video_delay_ms;
     40   int extra_audio_delay_ms;
     41   int last_audio_delay_ms;
     42   int network_delay;
     43 };
     44 
     45 StreamSynchronization::StreamSynchronization(uint32_t video_primary_ssrc,
     46                                              int audio_channel_id)
     47     : channel_delay_(new ViESyncDelay),
     48       video_primary_ssrc_(video_primary_ssrc),
     49       audio_channel_id_(audio_channel_id),
     50       base_target_delay_ms_(0),
     51       avg_diff_ms_(0) {
     52 }
     53 
     54 StreamSynchronization::~StreamSynchronization() {
     55   delete channel_delay_;
     56 }
     57 
     58 bool StreamSynchronization::ComputeRelativeDelay(
     59     const Measurements& audio_measurement,
     60     const Measurements& video_measurement,
     61     int* relative_delay_ms) {
     62   assert(relative_delay_ms);
     63   if (audio_measurement.rtcp.size() < 2 || video_measurement.rtcp.size() < 2) {
     64     // We need two RTCP SR reports per stream to do synchronization.
     65     return false;
     66   }
     67   int64_t audio_last_capture_time_ms;
     68   if (!RtpToNtpMs(audio_measurement.latest_timestamp,
     69                   audio_measurement.rtcp,
     70                   &audio_last_capture_time_ms)) {
     71     return false;
     72   }
     73   int64_t video_last_capture_time_ms;
     74   if (!RtpToNtpMs(video_measurement.latest_timestamp,
     75                   video_measurement.rtcp,
     76                   &video_last_capture_time_ms)) {
     77     return false;
     78   }
     79   if (video_last_capture_time_ms < 0) {
     80     return false;
     81   }
     82   // Positive diff means that video_measurement is behind audio_measurement.
     83   *relative_delay_ms = video_measurement.latest_receive_time_ms -
     84       audio_measurement.latest_receive_time_ms -
     85       (video_last_capture_time_ms - audio_last_capture_time_ms);
     86   if (*relative_delay_ms > kMaxDeltaDelayMs ||
     87       *relative_delay_ms < -kMaxDeltaDelayMs) {
     88     return false;
     89   }
     90   return true;
     91 }
     92 
     93 bool StreamSynchronization::ComputeDelays(int relative_delay_ms,
     94                                           int current_audio_delay_ms,
     95                                           int* total_audio_delay_target_ms,
     96                                           int* total_video_delay_target_ms) {
     97   assert(total_audio_delay_target_ms && total_video_delay_target_ms);
     98 
     99   int current_video_delay_ms = *total_video_delay_target_ms;
    100   LOG(LS_VERBOSE) << "Audio delay: " << current_audio_delay_ms
    101                   << ", network delay diff: " << channel_delay_->network_delay
    102                   << " current diff: " << relative_delay_ms
    103                   << " for channel " << audio_channel_id_;
    104   // Calculate the difference between the lowest possible video delay and
    105   // the current audio delay.
    106   int current_diff_ms = current_video_delay_ms - current_audio_delay_ms +
    107       relative_delay_ms;
    108 
    109   avg_diff_ms_ = ((kFilterLength - 1) * avg_diff_ms_ +
    110       current_diff_ms) / kFilterLength;
    111   if (abs(avg_diff_ms_) < kMinDeltaMs) {
    112     // Don't adjust if the diff is within our margin.
    113     return false;
    114   }
    115 
    116   // Make sure we don't move too fast.
    117   int diff_ms = avg_diff_ms_ / 2;
    118   diff_ms = std::min(diff_ms, kMaxChangeMs);
    119   diff_ms = std::max(diff_ms, -kMaxChangeMs);
    120 
    121   // Reset the average after a move to prevent overshooting reaction.
    122   avg_diff_ms_ = 0;
    123 
    124   if (diff_ms > 0) {
    125     // The minimum video delay is longer than the current audio delay.
    126     // We need to decrease extra video delay, or add extra audio delay.
    127     if (channel_delay_->extra_video_delay_ms > base_target_delay_ms_) {
    128       // We have extra delay added to ViE. Reduce this delay before adding
    129       // extra delay to VoE.
    130       channel_delay_->extra_video_delay_ms -= diff_ms;
    131       channel_delay_->extra_audio_delay_ms = base_target_delay_ms_;
    132     } else {  // channel_delay_->extra_video_delay_ms > 0
    133       // We have no extra video delay to remove, increase the audio delay.
    134       channel_delay_->extra_audio_delay_ms += diff_ms;
    135       channel_delay_->extra_video_delay_ms = base_target_delay_ms_;
    136     }
    137   } else {  // if (diff_ms > 0)
    138     // The video delay is lower than the current audio delay.
    139     // We need to decrease extra audio delay, or add extra video delay.
    140     if (channel_delay_->extra_audio_delay_ms > base_target_delay_ms_) {
    141       // We have extra delay in VoiceEngine.
    142       // Start with decreasing the voice delay.
    143       // Note: diff_ms is negative; add the negative difference.
    144       channel_delay_->extra_audio_delay_ms += diff_ms;
    145       channel_delay_->extra_video_delay_ms = base_target_delay_ms_;
    146     } else {  // channel_delay_->extra_audio_delay_ms > base_target_delay_ms_
    147       // We have no extra delay in VoiceEngine, increase the video delay.
    148       // Note: diff_ms is negative; subtract the negative difference.
    149       channel_delay_->extra_video_delay_ms -= diff_ms;  // X - (-Y) = X + Y.
    150       channel_delay_->extra_audio_delay_ms = base_target_delay_ms_;
    151     }
    152   }
    153 
    154   // Make sure that video is never below our target.
    155   channel_delay_->extra_video_delay_ms = std::max(
    156       channel_delay_->extra_video_delay_ms, base_target_delay_ms_);
    157 
    158   int new_video_delay_ms;
    159   if (channel_delay_->extra_video_delay_ms > base_target_delay_ms_) {
    160     new_video_delay_ms = channel_delay_->extra_video_delay_ms;
    161   } else {
    162     // No change to the extra video delay. We are changing audio and we only
    163     // allow to change one at the time.
    164     new_video_delay_ms = channel_delay_->last_video_delay_ms;
    165   }
    166 
    167   // Make sure that we don't go below the extra video delay.
    168   new_video_delay_ms = std::max(
    169       new_video_delay_ms, channel_delay_->extra_video_delay_ms);
    170 
    171   // Verify we don't go above the maximum allowed video delay.
    172   new_video_delay_ms =
    173       std::min(new_video_delay_ms, base_target_delay_ms_ + kMaxDeltaDelayMs);
    174 
    175   int new_audio_delay_ms;
    176   if (channel_delay_->extra_audio_delay_ms > base_target_delay_ms_) {
    177     new_audio_delay_ms = channel_delay_->extra_audio_delay_ms;
    178   } else {
    179     // No change to the audio delay. We are changing video and we only
    180     // allow to change one at the time.
    181     new_audio_delay_ms = channel_delay_->last_audio_delay_ms;
    182   }
    183 
    184   // Make sure that we don't go below the extra audio delay.
    185   new_audio_delay_ms = std::max(
    186       new_audio_delay_ms, channel_delay_->extra_audio_delay_ms);
    187 
    188   // Verify we don't go above the maximum allowed audio delay.
    189   new_audio_delay_ms =
    190       std::min(new_audio_delay_ms, base_target_delay_ms_ + kMaxDeltaDelayMs);
    191 
    192   // Remember our last audio and video delays.
    193   channel_delay_->last_video_delay_ms = new_video_delay_ms;
    194   channel_delay_->last_audio_delay_ms = new_audio_delay_ms;
    195 
    196   LOG(LS_VERBOSE) << "Sync video delay " << new_video_delay_ms
    197                   << " for video primary SSRC " << video_primary_ssrc_
    198                   << " and audio delay " << channel_delay_->extra_audio_delay_ms
    199                   << " for audio channel " << audio_channel_id_;
    200 
    201   // Return values.
    202   *total_video_delay_target_ms = new_video_delay_ms;
    203   *total_audio_delay_target_ms = new_audio_delay_ms;
    204   return true;
    205 }
    206 
    207 void StreamSynchronization::SetTargetBufferingDelay(int target_delay_ms) {
    208   // Initial extra delay for audio (accounting for existing extra delay).
    209   channel_delay_->extra_audio_delay_ms +=
    210       target_delay_ms - base_target_delay_ms_;
    211   channel_delay_->last_audio_delay_ms +=
    212       target_delay_ms - base_target_delay_ms_;
    213 
    214   // The video delay is compared to the last value (and how much we can update
    215   // is limited by that as well).
    216   channel_delay_->last_video_delay_ms +=
    217       target_delay_ms - base_target_delay_ms_;
    218 
    219   channel_delay_->extra_video_delay_ms +=
    220       target_delay_ms - base_target_delay_ms_;
    221 
    222   // Video is already delayed by the desired amount.
    223   base_target_delay_ms_ = target_delay_ms;
    224 }
    225 
    226 }  // namespace webrtc
    227