1 /* 2 * Copyright (c) 2015 The WebM project authors. All Rights Reserved. 3 * 4 * Use of this source code is governed by a BSD-style license 5 * that can be found in the LICENSE file in the root of the source 6 * tree. An additional intellectual property rights grant can be found 7 * in the file PATENTS. All contributing project authors may 8 * be found in the AUTHORS file in the root of the source tree. 9 */ 10 11 #include <assert.h> 12 #include <limits.h> 13 #include <math.h> 14 15 #include "./vpx_dsp_rtcd.h" 16 #include "vpx_dsp/vpx_dsp_common.h" 17 #include "vpx_scale/yv12config.h" 18 #include "vpx/vpx_integer.h" 19 #include "vp9/common/vp9_reconinter.h" 20 #include "vp9/encoder/vp9_context_tree.h" 21 #include "vp9/encoder/vp9_noise_estimate.h" 22 #include "vp9/encoder/vp9_encoder.h" 23 24 #if CONFIG_VP9_TEMPORAL_DENOISING 25 // For SVC: only do noise estimation on top spatial layer. 26 static INLINE int noise_est_svc(const struct VP9_COMP *const cpi) { 27 return (!cpi->use_svc || 28 (cpi->use_svc && 29 cpi->svc.spatial_layer_id == cpi->svc.number_spatial_layers - 1)); 30 } 31 #endif 32 33 void vp9_noise_estimate_init(NOISE_ESTIMATE *const ne, int width, int height) { 34 ne->enabled = 0; 35 ne->level = kLowLow; 36 ne->value = 0; 37 ne->count = 0; 38 ne->thresh = 90; 39 ne->last_w = 0; 40 ne->last_h = 0; 41 if (width * height >= 1920 * 1080) { 42 ne->thresh = 200; 43 } else if (width * height >= 1280 * 720) { 44 ne->thresh = 140; 45 } else if (width * height >= 640 * 360) { 46 ne->thresh = 115; 47 } 48 ne->num_frames_estimate = 15; 49 } 50 51 static int enable_noise_estimation(VP9_COMP *const cpi) { 52 #if CONFIG_VP9_HIGHBITDEPTH 53 if (cpi->common.use_highbitdepth) return 0; 54 #endif 55 // Enable noise estimation if denoising is on. 56 #if CONFIG_VP9_TEMPORAL_DENOISING 57 if (cpi->oxcf.noise_sensitivity > 0 && noise_est_svc(cpi) && 58 cpi->common.width >= 320 && cpi->common.height >= 180) 59 return 1; 60 #endif 61 // Only allow noise estimate under certain encoding mode. 62 // Enabled for 1 pass CBR, speed >=5, and if resolution is same as original. 63 // Not enabled for SVC mode and screen_content_mode. 64 // Not enabled for low resolutions. 65 if (cpi->oxcf.pass == 0 && cpi->oxcf.rc_mode == VPX_CBR && 66 cpi->oxcf.aq_mode == CYCLIC_REFRESH_AQ && cpi->oxcf.speed >= 5 && 67 cpi->resize_state == ORIG && cpi->resize_pending == 0 && !cpi->use_svc && 68 cpi->oxcf.content != VP9E_CONTENT_SCREEN && 69 cpi->common.width * cpi->common.height >= 640 * 360) 70 return 1; 71 else 72 return 0; 73 } 74 75 #if CONFIG_VP9_TEMPORAL_DENOISING 76 static void copy_frame(YV12_BUFFER_CONFIG *const dest, 77 const YV12_BUFFER_CONFIG *const src) { 78 int r; 79 const uint8_t *srcbuf = src->y_buffer; 80 uint8_t *destbuf = dest->y_buffer; 81 82 assert(dest->y_width == src->y_width); 83 assert(dest->y_height == src->y_height); 84 85 for (r = 0; r < dest->y_height; ++r) { 86 memcpy(destbuf, srcbuf, dest->y_width); 87 destbuf += dest->y_stride; 88 srcbuf += src->y_stride; 89 } 90 } 91 #endif // CONFIG_VP9_TEMPORAL_DENOISING 92 93 NOISE_LEVEL vp9_noise_estimate_extract_level(NOISE_ESTIMATE *const ne) { 94 int noise_level = kLowLow; 95 if (ne->value > (ne->thresh << 1)) { 96 noise_level = kHigh; 97 } else { 98 if (ne->value > ne->thresh) 99 noise_level = kMedium; 100 else if (ne->value > ((9 * ne->thresh) >> 4)) 101 noise_level = kLow; 102 else 103 noise_level = kLowLow; 104 } 105 return noise_level; 106 } 107 108 void vp9_update_noise_estimate(VP9_COMP *const cpi) { 109 const VP9_COMMON *const cm = &cpi->common; 110 NOISE_ESTIMATE *const ne = &cpi->noise_estimate; 111 const int low_res = (cm->width <= 352 && cm->height <= 288); 112 // Estimate of noise level every frame_period frames. 113 int frame_period = 8; 114 int thresh_consec_zeromv = 6; 115 unsigned int thresh_sum_diff = 100; 116 unsigned int thresh_sum_spatial = (200 * 200) << 8; 117 unsigned int thresh_spatial_var = (32 * 32) << 8; 118 int min_blocks_estimate = cm->mi_rows * cm->mi_cols >> 7; 119 int frame_counter = cm->current_video_frame; 120 // Estimate is between current source and last source. 121 YV12_BUFFER_CONFIG *last_source = cpi->Last_Source; 122 #if CONFIG_VP9_TEMPORAL_DENOISING 123 if (cpi->oxcf.noise_sensitivity > 0 && noise_est_svc(cpi)) { 124 last_source = &cpi->denoiser.last_source; 125 // Tune these thresholds for different resolutions when denoising is 126 // enabled. 127 if (cm->width > 640 && cm->width < 1920) { 128 thresh_consec_zeromv = 4; 129 thresh_sum_diff = 200; 130 thresh_sum_spatial = (120 * 120) << 8; 131 thresh_spatial_var = (48 * 48) << 8; 132 } 133 } 134 #endif 135 ne->enabled = enable_noise_estimation(cpi); 136 if (cpi->svc.number_spatial_layers > 1) 137 frame_counter = cpi->svc.current_superframe; 138 if (!ne->enabled || frame_counter % frame_period != 0 || 139 last_source == NULL || 140 (cpi->svc.number_spatial_layers == 1 && 141 (ne->last_w != cm->width || ne->last_h != cm->height))) { 142 #if CONFIG_VP9_TEMPORAL_DENOISING 143 if (cpi->oxcf.noise_sensitivity > 0 && noise_est_svc(cpi)) 144 copy_frame(&cpi->denoiser.last_source, cpi->Source); 145 #endif 146 if (last_source != NULL) { 147 ne->last_w = cm->width; 148 ne->last_h = cm->height; 149 } 150 return; 151 } else if (cm->current_video_frame > 60 && 152 cpi->rc.avg_frame_low_motion < (low_res ? 70 : 50)) { 153 // Force noise estimation to 0 and denoiser off if content has high motion. 154 ne->level = kLowLow; 155 ne->count = 0; 156 ne->num_frames_estimate = 10; 157 #if CONFIG_VP9_TEMPORAL_DENOISING 158 if (cpi->oxcf.noise_sensitivity > 0 && noise_est_svc(cpi) && 159 cpi->svc.current_superframe > 1) { 160 vp9_denoiser_set_noise_level(&cpi->denoiser, ne->level); 161 copy_frame(&cpi->denoiser.last_source, cpi->Source); 162 } 163 #endif 164 return; 165 } else { 166 int num_samples = 0; 167 uint64_t avg_est = 0; 168 int bsize = BLOCK_16X16; 169 static const unsigned char const_source[16] = { 0, 0, 0, 0, 0, 0, 0, 0, 170 0, 0, 0, 0, 0, 0, 0, 0 }; 171 // Loop over sub-sample of 16x16 blocks of frame, and for blocks that have 172 // been encoded as zero/small mv at least x consecutive frames, compute 173 // the variance to update estimate of noise in the source. 174 const uint8_t *src_y = cpi->Source->y_buffer; 175 const int src_ystride = cpi->Source->y_stride; 176 const uint8_t *last_src_y = last_source->y_buffer; 177 const int last_src_ystride = last_source->y_stride; 178 const uint8_t *src_u = cpi->Source->u_buffer; 179 const uint8_t *src_v = cpi->Source->v_buffer; 180 const int src_uvstride = cpi->Source->uv_stride; 181 int mi_row, mi_col; 182 int num_low_motion = 0; 183 int frame_low_motion = 1; 184 for (mi_row = 0; mi_row < cm->mi_rows; mi_row++) { 185 for (mi_col = 0; mi_col < cm->mi_cols; mi_col++) { 186 int bl_index = mi_row * cm->mi_cols + mi_col; 187 if (cpi->consec_zero_mv[bl_index] > thresh_consec_zeromv) 188 num_low_motion++; 189 } 190 } 191 if (num_low_motion < ((3 * cm->mi_rows * cm->mi_cols) >> 3)) 192 frame_low_motion = 0; 193 for (mi_row = 0; mi_row < cm->mi_rows; mi_row++) { 194 for (mi_col = 0; mi_col < cm->mi_cols; mi_col++) { 195 // 16x16 blocks, 1/4 sample of frame. 196 if (mi_row % 4 == 0 && mi_col % 4 == 0 && mi_row < cm->mi_rows - 1 && 197 mi_col < cm->mi_cols - 1) { 198 int bl_index = mi_row * cm->mi_cols + mi_col; 199 int bl_index1 = bl_index + 1; 200 int bl_index2 = bl_index + cm->mi_cols; 201 int bl_index3 = bl_index2 + 1; 202 int consec_zeromv = 203 VPXMIN(cpi->consec_zero_mv[bl_index], 204 VPXMIN(cpi->consec_zero_mv[bl_index1], 205 VPXMIN(cpi->consec_zero_mv[bl_index2], 206 cpi->consec_zero_mv[bl_index3]))); 207 // Only consider blocks that are likely steady background. i.e, have 208 // been encoded as zero/low motion x (= thresh_consec_zeromv) frames 209 // in a row. consec_zero_mv[] defined for 8x8 blocks, so consider all 210 // 4 sub-blocks for 16x16 block. Also, avoid skin blocks. 211 if (frame_low_motion && consec_zeromv > thresh_consec_zeromv) { 212 int is_skin = 0; 213 if (cpi->use_skin_detection) { 214 is_skin = 215 vp9_compute_skin_block(src_y, src_u, src_v, src_ystride, 216 src_uvstride, bsize, consec_zeromv, 0); 217 } 218 if (!is_skin) { 219 unsigned int sse; 220 // Compute variance. 221 unsigned int variance = cpi->fn_ptr[bsize].vf( 222 src_y, src_ystride, last_src_y, last_src_ystride, &sse); 223 // Only consider this block as valid for noise measurement if the 224 // average term (sse - variance = N * avg^{2}, N = 16X16) of the 225 // temporal residual is small (avoid effects from lighting 226 // change). 227 if ((sse - variance) < thresh_sum_diff) { 228 unsigned int sse2; 229 const unsigned int spatial_variance = cpi->fn_ptr[bsize].vf( 230 src_y, src_ystride, const_source, 0, &sse2); 231 // Avoid blocks with high brightness and high spatial variance. 232 if ((sse2 - spatial_variance) < thresh_sum_spatial && 233 spatial_variance < thresh_spatial_var) { 234 avg_est += low_res ? variance >> 4 235 : variance / ((spatial_variance >> 9) + 1); 236 num_samples++; 237 } 238 } 239 } 240 } 241 } 242 src_y += 8; 243 last_src_y += 8; 244 src_u += 4; 245 src_v += 4; 246 } 247 src_y += (src_ystride << 3) - (cm->mi_cols << 3); 248 last_src_y += (last_src_ystride << 3) - (cm->mi_cols << 3); 249 src_u += (src_uvstride << 2) - (cm->mi_cols << 2); 250 src_v += (src_uvstride << 2) - (cm->mi_cols << 2); 251 } 252 ne->last_w = cm->width; 253 ne->last_h = cm->height; 254 // Update noise estimate if we have at a minimum number of block samples, 255 // and avg_est > 0 (avg_est == 0 can happen if the application inputs 256 // duplicate frames). 257 if (num_samples > min_blocks_estimate && avg_est > 0) { 258 // Normalize. 259 avg_est = avg_est / num_samples; 260 // Update noise estimate. 261 ne->value = (int)((15 * ne->value + avg_est) >> 4); 262 ne->count++; 263 if (ne->count == ne->num_frames_estimate) { 264 // Reset counter and check noise level condition. 265 ne->num_frames_estimate = 30; 266 ne->count = 0; 267 ne->level = vp9_noise_estimate_extract_level(ne); 268 #if CONFIG_VP9_TEMPORAL_DENOISING 269 if (cpi->oxcf.noise_sensitivity > 0 && noise_est_svc(cpi)) 270 vp9_denoiser_set_noise_level(&cpi->denoiser, ne->level); 271 #endif 272 } 273 } 274 } 275 #if CONFIG_VP9_TEMPORAL_DENOISING 276 if (cpi->oxcf.noise_sensitivity > 0 && noise_est_svc(cpi)) 277 copy_frame(&cpi->denoiser.last_source, cpi->Source); 278 #endif 279 } 280
