1 /* 2 * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. 3 * 4 * Use of this source code is governed by a BSD-style license 5 * that can be found in the LICENSE file in the root of the source 6 * tree. An additional intellectual property rights grant can be found 7 * in the file PATENTS. All contributing project authors may 8 * be found in the AUTHORS file in the root of the source tree. 9 */ 10 #include "webrtc/modules/video_processing/main/source/content_analysis.h" 11 12 #include <math.h> 13 #include <stdlib.h> 14 15 #include "webrtc/system_wrappers/interface/cpu_features_wrapper.h" 16 #include "webrtc/system_wrappers/interface/tick_util.h" 17 18 namespace webrtc { 19 20 VPMContentAnalysis::VPMContentAnalysis(bool runtime_cpu_detection) 21 : orig_frame_(NULL), 22 prev_frame_(NULL), 23 width_(0), 24 height_(0), 25 skip_num_(1), 26 border_(8), 27 motion_magnitude_(0.0f), 28 spatial_pred_err_(0.0f), 29 spatial_pred_err_h_(0.0f), 30 spatial_pred_err_v_(0.0f), 31 first_frame_(true), 32 ca_Init_(false), 33 content_metrics_(NULL) { 34 ComputeSpatialMetrics = &VPMContentAnalysis::ComputeSpatialMetrics_C; 35 TemporalDiffMetric = &VPMContentAnalysis::TemporalDiffMetric_C; 36 37 if (runtime_cpu_detection) { 38 #if defined(WEBRTC_ARCH_X86_FAMILY) 39 if (WebRtc_GetCPUInfo(kSSE2)) { 40 ComputeSpatialMetrics = &VPMContentAnalysis::ComputeSpatialMetrics_SSE2; 41 TemporalDiffMetric = &VPMContentAnalysis::TemporalDiffMetric_SSE2; 42 } 43 #endif 44 } 45 Release(); 46 } 47 48 VPMContentAnalysis::~VPMContentAnalysis() { 49 Release(); 50 } 51 52 53 VideoContentMetrics* VPMContentAnalysis::ComputeContentMetrics( 54 const I420VideoFrame& inputFrame) { 55 if (inputFrame.IsZeroSize()) 56 return NULL; 57 58 // Init if needed (native dimension change). 59 if (width_ != inputFrame.width() || height_ != inputFrame.height()) { 60 if (VPM_OK != Initialize(inputFrame.width(), inputFrame.height())) 61 return NULL; 62 } 63 // Only interested in the Y plane. 64 orig_frame_ = inputFrame.buffer(kYPlane); 65 66 // Compute spatial metrics: 3 spatial prediction errors. 67 (this->*ComputeSpatialMetrics)(); 68 69 // Compute motion metrics 70 if (first_frame_ == false) 71 ComputeMotionMetrics(); 72 73 // Saving current frame as previous one: Y only. 74 memcpy(prev_frame_, orig_frame_, width_ * height_); 75 76 first_frame_ = false; 77 ca_Init_ = true; 78 79 return ContentMetrics(); 80 } 81 82 int32_t VPMContentAnalysis::Release() { 83 if (content_metrics_ != NULL) { 84 delete content_metrics_; 85 content_metrics_ = NULL; 86 } 87 88 if (prev_frame_ != NULL) { 89 delete [] prev_frame_; 90 prev_frame_ = NULL; 91 } 92 93 width_ = 0; 94 height_ = 0; 95 first_frame_ = true; 96 97 return VPM_OK; 98 } 99 100 int32_t VPMContentAnalysis::Initialize(int width, int height) { 101 width_ = width; 102 height_ = height; 103 first_frame_ = true; 104 105 // skip parameter: # of skipped rows: for complexity reduction 106 // temporal also currently uses it for column reduction. 107 skip_num_ = 1; 108 109 // use skipNum = 2 for 4CIF, WHD 110 if ( (height_ >= 576) && (width_ >= 704) ) { 111 skip_num_ = 2; 112 } 113 // use skipNum = 4 for FULLL_HD images 114 if ( (height_ >= 1080) && (width_ >= 1920) ) { 115 skip_num_ = 4; 116 } 117 118 if (content_metrics_ != NULL) { 119 delete content_metrics_; 120 } 121 122 if (prev_frame_ != NULL) { 123 delete [] prev_frame_; 124 } 125 126 // Spatial Metrics don't work on a border of 8. Minimum processing 127 // block size is 16 pixels. So make sure the width and height support this. 128 if (width_ <= 32 || height_ <= 32) { 129 ca_Init_ = false; 130 return VPM_PARAMETER_ERROR; 131 } 132 133 content_metrics_ = new VideoContentMetrics(); 134 if (content_metrics_ == NULL) { 135 return VPM_MEMORY; 136 } 137 138 prev_frame_ = new uint8_t[width_ * height_]; // Y only. 139 if (prev_frame_ == NULL) return VPM_MEMORY; 140 141 return VPM_OK; 142 } 143 144 145 // Compute motion metrics: magnitude over non-zero motion vectors, 146 // and size of zero cluster 147 int32_t VPMContentAnalysis::ComputeMotionMetrics() { 148 // Motion metrics: only one is derived from normalized 149 // (MAD) temporal difference 150 (this->*TemporalDiffMetric)(); 151 return VPM_OK; 152 } 153 154 // Normalized temporal difference (MAD): used as a motion level metric 155 // Normalize MAD by spatial contrast: images with more contrast 156 // (pixel variance) likely have larger temporal difference 157 // To reduce complexity, we compute the metric for a reduced set of points. 158 int32_t VPMContentAnalysis::TemporalDiffMetric_C() { 159 // size of original frame 160 int sizei = height_; 161 int sizej = width_; 162 uint32_t tempDiffSum = 0; 163 uint32_t pixelSum = 0; 164 uint64_t pixelSqSum = 0; 165 166 uint32_t num_pixels = 0; // Counter for # of pixels. 167 const int width_end = ((width_ - 2*border_) & -16) + border_; 168 169 for (int i = border_; i < sizei - border_; i += skip_num_) { 170 for (int j = border_; j < width_end; j++) { 171 num_pixels += 1; 172 int ssn = i * sizej + j; 173 174 uint8_t currPixel = orig_frame_[ssn]; 175 uint8_t prevPixel = prev_frame_[ssn]; 176 177 tempDiffSum += (uint32_t)abs((int16_t)(currPixel - prevPixel)); 178 pixelSum += (uint32_t) currPixel; 179 pixelSqSum += (uint64_t) (currPixel * currPixel); 180 } 181 } 182 183 // Default. 184 motion_magnitude_ = 0.0f; 185 186 if (tempDiffSum == 0) return VPM_OK; 187 188 // Normalize over all pixels. 189 float const tempDiffAvg = (float)tempDiffSum / (float)(num_pixels); 190 float const pixelSumAvg = (float)pixelSum / (float)(num_pixels); 191 float const pixelSqSumAvg = (float)pixelSqSum / (float)(num_pixels); 192 float contrast = pixelSqSumAvg - (pixelSumAvg * pixelSumAvg); 193 194 if (contrast > 0.0) { 195 contrast = sqrt(contrast); 196 motion_magnitude_ = tempDiffAvg/contrast; 197 } 198 return VPM_OK; 199 } 200 201 // Compute spatial metrics: 202 // To reduce complexity, we compute the metric for a reduced set of points. 203 // The spatial metrics are rough estimates of the prediction error cost for 204 // each QM spatial mode: 2x2,1x2,2x1 205 // The metrics are a simple estimate of the up-sampling prediction error, 206 // estimated assuming sub-sampling for decimation (no filtering), 207 // and up-sampling back up with simple bilinear interpolation. 208 int32_t VPMContentAnalysis::ComputeSpatialMetrics_C() { 209 const int sizei = height_; 210 const int sizej = width_; 211 212 // Pixel mean square average: used to normalize the spatial metrics. 213 uint32_t pixelMSA = 0; 214 215 uint32_t spatialErrSum = 0; 216 uint32_t spatialErrVSum = 0; 217 uint32_t spatialErrHSum = 0; 218 219 // make sure work section is a multiple of 16 220 const int width_end = ((sizej - 2*border_) & -16) + border_; 221 222 for (int i = border_; i < sizei - border_; i += skip_num_) { 223 for (int j = border_; j < width_end; j++) { 224 int ssn1= i * sizej + j; 225 int ssn2 = (i + 1) * sizej + j; // bottom 226 int ssn3 = (i - 1) * sizej + j; // top 227 int ssn4 = i * sizej + j + 1; // right 228 int ssn5 = i * sizej + j - 1; // left 229 230 uint16_t refPixel1 = orig_frame_[ssn1] << 1; 231 uint16_t refPixel2 = orig_frame_[ssn1] << 2; 232 233 uint8_t bottPixel = orig_frame_[ssn2]; 234 uint8_t topPixel = orig_frame_[ssn3]; 235 uint8_t rightPixel = orig_frame_[ssn4]; 236 uint8_t leftPixel = orig_frame_[ssn5]; 237 238 spatialErrSum += (uint32_t) abs((int16_t)(refPixel2 239 - (uint16_t)(bottPixel + topPixel + leftPixel + rightPixel))); 240 spatialErrVSum += (uint32_t) abs((int16_t)(refPixel1 241 - (uint16_t)(bottPixel + topPixel))); 242 spatialErrHSum += (uint32_t) abs((int16_t)(refPixel1 243 - (uint16_t)(leftPixel + rightPixel))); 244 pixelMSA += orig_frame_[ssn1]; 245 } 246 } 247 248 // Normalize over all pixels. 249 const float spatialErr = (float)(spatialErrSum >> 2); 250 const float spatialErrH = (float)(spatialErrHSum >> 1); 251 const float spatialErrV = (float)(spatialErrVSum >> 1); 252 const float norm = (float)pixelMSA; 253 254 // 2X2: 255 spatial_pred_err_ = spatialErr / norm; 256 // 1X2: 257 spatial_pred_err_h_ = spatialErrH / norm; 258 // 2X1: 259 spatial_pred_err_v_ = spatialErrV / norm; 260 return VPM_OK; 261 } 262 263 VideoContentMetrics* VPMContentAnalysis::ContentMetrics() { 264 if (ca_Init_ == false) return NULL; 265 266 content_metrics_->spatial_pred_err = spatial_pred_err_; 267 content_metrics_->spatial_pred_err_h = spatial_pred_err_h_; 268 content_metrics_->spatial_pred_err_v = spatial_pred_err_v_; 269 // Motion metric: normalized temporal difference (MAD). 270 content_metrics_->motion_magnitude = motion_magnitude_; 271 272 return content_metrics_; 273 } 274 275 } // namespace webrtc 276