1 /* 2 * Copyright (c) 2010 The WebM project authors. All Rights Reserved. 3 * 4 * Use of this source code is governed by a BSD-style license 5 * that can be found in the LICENSE file in the root of the source 6 * tree. An additional intellectual property rights grant can be found 7 * in the file PATENTS. All contributing project authors may 8 * be found in the AUTHORS file in the root of the source tree. 9 */ 10 11 #include <assert.h> 12 #include <math.h> 13 #include "./vpx_dsp_rtcd.h" 14 #include "vpx_dsp/ssim.h" 15 #include "vpx_ports/mem.h" 16 #include "vpx_ports/system_state.h" 17 18 void vpx_ssim_parms_16x16_c(const uint8_t *s, int sp, const uint8_t *r, int rp, 19 uint32_t *sum_s, uint32_t *sum_r, 20 uint32_t *sum_sq_s, uint32_t *sum_sq_r, 21 uint32_t *sum_sxr) { 22 int i, j; 23 for (i = 0; i < 16; i++, s += sp, r += rp) { 24 for (j = 0; j < 16; j++) { 25 *sum_s += s[j]; 26 *sum_r += r[j]; 27 *sum_sq_s += s[j] * s[j]; 28 *sum_sq_r += r[j] * r[j]; 29 *sum_sxr += s[j] * r[j]; 30 } 31 } 32 } 33 void vpx_ssim_parms_8x8_c(const uint8_t *s, int sp, const uint8_t *r, int rp, 34 uint32_t *sum_s, uint32_t *sum_r, uint32_t *sum_sq_s, 35 uint32_t *sum_sq_r, uint32_t *sum_sxr) { 36 int i, j; 37 for (i = 0; i < 8; i++, s += sp, r += rp) { 38 for (j = 0; j < 8; j++) { 39 *sum_s += s[j]; 40 *sum_r += r[j]; 41 *sum_sq_s += s[j] * s[j]; 42 *sum_sq_r += r[j] * r[j]; 43 *sum_sxr += s[j] * r[j]; 44 } 45 } 46 } 47 48 #if CONFIG_VP9_HIGHBITDEPTH 49 void vpx_highbd_ssim_parms_8x8_c(const uint16_t *s, int sp, const uint16_t *r, 50 int rp, uint32_t *sum_s, uint32_t *sum_r, 51 uint32_t *sum_sq_s, uint32_t *sum_sq_r, 52 uint32_t *sum_sxr) { 53 int i, j; 54 for (i = 0; i < 8; i++, s += sp, r += rp) { 55 for (j = 0; j < 8; j++) { 56 *sum_s += s[j]; 57 *sum_r += r[j]; 58 *sum_sq_s += s[j] * s[j]; 59 *sum_sq_r += r[j] * r[j]; 60 *sum_sxr += s[j] * r[j]; 61 } 62 } 63 } 64 #endif // CONFIG_VP9_HIGHBITDEPTH 65 66 static const int64_t cc1 = 26634; // (64^2*(.01*255)^2 67 static const int64_t cc2 = 239708; // (64^2*(.03*255)^2 68 static const int64_t cc1_10 = 428658; // (64^2*(.01*1023)^2 69 static const int64_t cc2_10 = 3857925; // (64^2*(.03*1023)^2 70 static const int64_t cc1_12 = 6868593; // (64^2*(.01*4095)^2 71 static const int64_t cc2_12 = 61817334; // (64^2*(.03*4095)^2 72 73 static double similarity(uint32_t sum_s, uint32_t sum_r, uint32_t sum_sq_s, 74 uint32_t sum_sq_r, uint32_t sum_sxr, int count, 75 uint32_t bd) { 76 int64_t ssim_n, ssim_d; 77 int64_t c1, c2; 78 if (bd == 8) { 79 // scale the constants by number of pixels 80 c1 = (cc1 * count * count) >> 12; 81 c2 = (cc2 * count * count) >> 12; 82 } else if (bd == 10) { 83 c1 = (cc1_10 * count * count) >> 12; 84 c2 = (cc2_10 * count * count) >> 12; 85 } else if (bd == 12) { 86 c1 = (cc1_12 * count * count) >> 12; 87 c2 = (cc2_12 * count * count) >> 12; 88 } else { 89 c1 = c2 = 0; 90 assert(0); 91 } 92 93 ssim_n = (2 * sum_s * sum_r + c1) * 94 ((int64_t)2 * count * sum_sxr - (int64_t)2 * sum_s * sum_r + c2); 95 96 ssim_d = (sum_s * sum_s + sum_r * sum_r + c1) * 97 ((int64_t)count * sum_sq_s - (int64_t)sum_s * sum_s + 98 (int64_t)count * sum_sq_r - (int64_t)sum_r * sum_r + c2); 99 100 return ssim_n * 1.0 / ssim_d; 101 } 102 103 static double ssim_8x8(const uint8_t *s, int sp, const uint8_t *r, int rp) { 104 uint32_t sum_s = 0, sum_r = 0, sum_sq_s = 0, sum_sq_r = 0, sum_sxr = 0; 105 vpx_ssim_parms_8x8(s, sp, r, rp, &sum_s, &sum_r, &sum_sq_s, &sum_sq_r, 106 &sum_sxr); 107 return similarity(sum_s, sum_r, sum_sq_s, sum_sq_r, sum_sxr, 64, 8); 108 } 109 110 #if CONFIG_VP9_HIGHBITDEPTH 111 static double highbd_ssim_8x8(const uint16_t *s, int sp, const uint16_t *r, 112 int rp, uint32_t bd, uint32_t shift) { 113 uint32_t sum_s = 0, sum_r = 0, sum_sq_s = 0, sum_sq_r = 0, sum_sxr = 0; 114 vpx_highbd_ssim_parms_8x8(s, sp, r, rp, &sum_s, &sum_r, &sum_sq_s, &sum_sq_r, 115 &sum_sxr); 116 return similarity(sum_s >> shift, sum_r >> shift, sum_sq_s >> (2 * shift), 117 sum_sq_r >> (2 * shift), sum_sxr >> (2 * shift), 64, bd); 118 } 119 #endif // CONFIG_VP9_HIGHBITDEPTH 120 121 // We are using a 8x8 moving window with starting location of each 8x8 window 122 // on the 4x4 pixel grid. Such arrangement allows the windows to overlap 123 // block boundaries to penalize blocking artifacts. 124 static double vpx_ssim2(const uint8_t *img1, const uint8_t *img2, 125 int stride_img1, int stride_img2, int width, 126 int height) { 127 int i, j; 128 int samples = 0; 129 double ssim_total = 0; 130 131 // sample point start with each 4x4 location 132 for (i = 0; i <= height - 8; 133 i += 4, img1 += stride_img1 * 4, img2 += stride_img2 * 4) { 134 for (j = 0; j <= width - 8; j += 4) { 135 double v = ssim_8x8(img1 + j, stride_img1, img2 + j, stride_img2); 136 ssim_total += v; 137 samples++; 138 } 139 } 140 ssim_total /= samples; 141 return ssim_total; 142 } 143 144 #if CONFIG_VP9_HIGHBITDEPTH 145 static double vpx_highbd_ssim2(const uint8_t *img1, const uint8_t *img2, 146 int stride_img1, int stride_img2, int width, 147 int height, uint32_t bd, uint32_t shift) { 148 int i, j; 149 int samples = 0; 150 double ssim_total = 0; 151 152 // sample point start with each 4x4 location 153 for (i = 0; i <= height - 8; 154 i += 4, img1 += stride_img1 * 4, img2 += stride_img2 * 4) { 155 for (j = 0; j <= width - 8; j += 4) { 156 double v = highbd_ssim_8x8(CONVERT_TO_SHORTPTR(img1 + j), stride_img1, 157 CONVERT_TO_SHORTPTR(img2 + j), stride_img2, bd, 158 shift); 159 ssim_total += v; 160 samples++; 161 } 162 } 163 ssim_total /= samples; 164 return ssim_total; 165 } 166 #endif // CONFIG_VP9_HIGHBITDEPTH 167 168 double vpx_calc_ssim(const YV12_BUFFER_CONFIG *source, 169 const YV12_BUFFER_CONFIG *dest, double *weight) { 170 double a, b, c; 171 double ssimv; 172 173 a = vpx_ssim2(source->y_buffer, dest->y_buffer, source->y_stride, 174 dest->y_stride, source->y_crop_width, source->y_crop_height); 175 176 b = vpx_ssim2(source->u_buffer, dest->u_buffer, source->uv_stride, 177 dest->uv_stride, source->uv_crop_width, source->uv_crop_height); 178 179 c = vpx_ssim2(source->v_buffer, dest->v_buffer, source->uv_stride, 180 dest->uv_stride, source->uv_crop_width, source->uv_crop_height); 181 182 ssimv = a * .8 + .1 * (b + c); 183 184 *weight = 1; 185 186 return ssimv; 187 } 188 189 // traditional ssim as per: http://en.wikipedia.org/wiki/Structural_similarity 190 // 191 // Re working out the math -> 192 // 193 // ssim(x,y) = (2*mean(x)*mean(y) + c1)*(2*cov(x,y)+c2) / 194 // ((mean(x)^2+mean(y)^2+c1)*(var(x)+var(y)+c2)) 195 // 196 // mean(x) = sum(x) / n 197 // 198 // cov(x,y) = (n*sum(xi*yi)-sum(x)*sum(y))/(n*n) 199 // 200 // var(x) = (n*sum(xi*xi)-sum(xi)*sum(xi))/(n*n) 201 // 202 // ssim(x,y) = 203 // (2*sum(x)*sum(y)/(n*n) + c1)*(2*(n*sum(xi*yi)-sum(x)*sum(y))/(n*n)+c2) / 204 // (((sum(x)*sum(x)+sum(y)*sum(y))/(n*n) +c1) * 205 // ((n*sum(xi*xi) - sum(xi)*sum(xi))/(n*n)+ 206 // (n*sum(yi*yi) - sum(yi)*sum(yi))/(n*n)+c2))) 207 // 208 // factoring out n*n 209 // 210 // ssim(x,y) = 211 // (2*sum(x)*sum(y) + n*n*c1)*(2*(n*sum(xi*yi)-sum(x)*sum(y))+n*n*c2) / 212 // (((sum(x)*sum(x)+sum(y)*sum(y)) + n*n*c1) * 213 // (n*sum(xi*xi)-sum(xi)*sum(xi)+n*sum(yi*yi)-sum(yi)*sum(yi)+n*n*c2)) 214 // 215 // Replace c1 with n*n * c1 for the final step that leads to this code: 216 // The final step scales by 12 bits so we don't lose precision in the constants. 217 218 static double ssimv_similarity(const Ssimv *sv, int64_t n) { 219 // Scale the constants by number of pixels. 220 const int64_t c1 = (cc1 * n * n) >> 12; 221 const int64_t c2 = (cc2 * n * n) >> 12; 222 223 const double l = 1.0 * (2 * sv->sum_s * sv->sum_r + c1) / 224 (sv->sum_s * sv->sum_s + sv->sum_r * sv->sum_r + c1); 225 226 // Since these variables are unsigned sums, convert to double so 227 // math is done in double arithmetic. 228 const double v = (2.0 * n * sv->sum_sxr - 2 * sv->sum_s * sv->sum_r + c2) / 229 (n * sv->sum_sq_s - sv->sum_s * sv->sum_s + 230 n * sv->sum_sq_r - sv->sum_r * sv->sum_r + c2); 231 232 return l * v; 233 } 234 235 // The first term of the ssim metric is a luminance factor. 236 // 237 // (2*mean(x)*mean(y) + c1)/ (mean(x)^2+mean(y)^2+c1) 238 // 239 // This luminance factor is super sensitive to the dark side of luminance 240 // values and completely insensitive on the white side. check out 2 sets 241 // (1,3) and (250,252) the term gives ( 2*1*3/(1+9) = .60 242 // 2*250*252/ (250^2+252^2) => .99999997 243 // 244 // As a result in this tweaked version of the calculation in which the 245 // luminance is taken as percentage off from peak possible. 246 // 247 // 255 * 255 - (sum_s - sum_r) / count * (sum_s - sum_r) / count 248 // 249 static double ssimv_similarity2(const Ssimv *sv, int64_t n) { 250 // Scale the constants by number of pixels. 251 const int64_t c1 = (cc1 * n * n) >> 12; 252 const int64_t c2 = (cc2 * n * n) >> 12; 253 254 const double mean_diff = (1.0 * sv->sum_s - sv->sum_r) / n; 255 const double l = (255 * 255 - mean_diff * mean_diff + c1) / (255 * 255 + c1); 256 257 // Since these variables are unsigned, sums convert to double so 258 // math is done in double arithmetic. 259 const double v = (2.0 * n * sv->sum_sxr - 2 * sv->sum_s * sv->sum_r + c2) / 260 (n * sv->sum_sq_s - sv->sum_s * sv->sum_s + 261 n * sv->sum_sq_r - sv->sum_r * sv->sum_r + c2); 262 263 return l * v; 264 } 265 static void ssimv_parms(uint8_t *img1, int img1_pitch, uint8_t *img2, 266 int img2_pitch, Ssimv *sv) { 267 vpx_ssim_parms_8x8(img1, img1_pitch, img2, img2_pitch, &sv->sum_s, &sv->sum_r, 268 &sv->sum_sq_s, &sv->sum_sq_r, &sv->sum_sxr); 269 } 270 271 double vpx_get_ssim_metrics(uint8_t *img1, int img1_pitch, uint8_t *img2, 272 int img2_pitch, int width, int height, Ssimv *sv2, 273 Metrics *m, int do_inconsistency) { 274 double dssim_total = 0; 275 double ssim_total = 0; 276 double ssim2_total = 0; 277 double inconsistency_total = 0; 278 int i, j; 279 int c = 0; 280 double norm; 281 double old_ssim_total = 0; 282 vpx_clear_system_state(); 283 // We can sample points as frequently as we like start with 1 per 4x4. 284 for (i = 0; i < height; 285 i += 4, img1 += img1_pitch * 4, img2 += img2_pitch * 4) { 286 for (j = 0; j < width; j += 4, ++c) { 287 Ssimv sv = { 0 }; 288 double ssim; 289 double ssim2; 290 double dssim; 291 uint32_t var_new; 292 uint32_t var_old; 293 uint32_t mean_new; 294 uint32_t mean_old; 295 double ssim_new; 296 double ssim_old; 297 298 // Not sure there's a great way to handle the edge pixels 299 // in ssim when using a window. Seems biased against edge pixels 300 // however you handle this. This uses only samples that are 301 // fully in the frame. 302 if (j + 8 <= width && i + 8 <= height) { 303 ssimv_parms(img1 + j, img1_pitch, img2 + j, img2_pitch, &sv); 304 } 305 306 ssim = ssimv_similarity(&sv, 64); 307 ssim2 = ssimv_similarity2(&sv, 64); 308 309 sv.ssim = ssim2; 310 311 // dssim is calculated to use as an actual error metric and 312 // is scaled up to the same range as sum square error. 313 // Since we are subsampling every 16th point maybe this should be 314 // *16 ? 315 dssim = 255 * 255 * (1 - ssim2) / 2; 316 317 // Here I introduce a new error metric: consistency-weighted 318 // SSIM-inconsistency. This metric isolates frames where the 319 // SSIM 'suddenly' changes, e.g. if one frame in every 8 is much 320 // sharper or blurrier than the others. Higher values indicate a 321 // temporally inconsistent SSIM. There are two ideas at work: 322 // 323 // 1) 'SSIM-inconsistency': the total inconsistency value 324 // reflects how much SSIM values are changing between this 325 // source / reference frame pair and the previous pair. 326 // 327 // 2) 'consistency-weighted': weights de-emphasize areas in the 328 // frame where the scene content has changed. Changes in scene 329 // content are detected via changes in local variance and local 330 // mean. 331 // 332 // Thus the overall measure reflects how inconsistent the SSIM 333 // values are, over consistent regions of the frame. 334 // 335 // The metric has three terms: 336 // 337 // term 1 -> uses change in scene Variance to weight error score 338 // 2 * var(Fi)*var(Fi-1) / (var(Fi)^2+var(Fi-1)^2) 339 // larger changes from one frame to the next mean we care 340 // less about consistency. 341 // 342 // term 2 -> uses change in local scene luminance to weight error 343 // 2 * avg(Fi)*avg(Fi-1) / (avg(Fi)^2+avg(Fi-1)^2) 344 // larger changes from one frame to the next mean we care 345 // less about consistency. 346 // 347 // term3 -> measures inconsistency in ssim scores between frames 348 // 1 - ( 2 * ssim(Fi)*ssim(Fi-1)/(ssim(Fi)^2+sssim(Fi-1)^2). 349 // 350 // This term compares the ssim score for the same location in 2 351 // subsequent frames. 352 var_new = sv.sum_sq_s - sv.sum_s * sv.sum_s / 64; 353 var_old = sv2[c].sum_sq_s - sv2[c].sum_s * sv2[c].sum_s / 64; 354 mean_new = sv.sum_s; 355 mean_old = sv2[c].sum_s; 356 ssim_new = sv.ssim; 357 ssim_old = sv2[c].ssim; 358 359 if (do_inconsistency) { 360 // We do the metric once for every 4x4 block in the image. Since 361 // we are scaling the error to SSE for use in a psnr calculation 362 // 1.0 = 4x4x255x255 the worst error we can possibly have. 363 static const double kScaling = 4. * 4 * 255 * 255; 364 365 // The constants have to be non 0 to avoid potential divide by 0 366 // issues other than that they affect kind of a weighting between 367 // the terms. No testing of what the right terms should be has been 368 // done. 369 static const double c1 = 1, c2 = 1, c3 = 1; 370 371 // This measures how much consistent variance is in two consecutive 372 // source frames. 1.0 means they have exactly the same variance. 373 const double variance_term = 374 (2.0 * var_old * var_new + c1) / 375 (1.0 * var_old * var_old + 1.0 * var_new * var_new + c1); 376 377 // This measures how consistent the local mean are between two 378 // consecutive frames. 1.0 means they have exactly the same mean. 379 const double mean_term = 380 (2.0 * mean_old * mean_new + c2) / 381 (1.0 * mean_old * mean_old + 1.0 * mean_new * mean_new + c2); 382 383 // This measures how consistent the ssims of two 384 // consecutive frames is. 1.0 means they are exactly the same. 385 double ssim_term = 386 pow((2.0 * ssim_old * ssim_new + c3) / 387 (ssim_old * ssim_old + ssim_new * ssim_new + c3), 388 5); 389 390 double this_inconsistency; 391 392 // Floating point math sometimes makes this > 1 by a tiny bit. 393 // We want the metric to scale between 0 and 1.0 so we can convert 394 // it to an snr scaled value. 395 if (ssim_term > 1) ssim_term = 1; 396 397 // This converts the consistency metric to an inconsistency metric 398 // ( so we can scale it like psnr to something like sum square error. 399 // The reason for the variance and mean terms is the assumption that 400 // if there are big changes in the source we shouldn't penalize 401 // inconsistency in ssim scores a bit less as it will be less visible 402 // to the user. 403 this_inconsistency = (1 - ssim_term) * variance_term * mean_term; 404 405 this_inconsistency *= kScaling; 406 inconsistency_total += this_inconsistency; 407 } 408 sv2[c] = sv; 409 ssim_total += ssim; 410 ssim2_total += ssim2; 411 dssim_total += dssim; 412 413 old_ssim_total += ssim_old; 414 } 415 old_ssim_total += 0; 416 } 417 418 norm = 1. / (width / 4) / (height / 4); 419 ssim_total *= norm; 420 ssim2_total *= norm; 421 m->ssim2 = ssim2_total; 422 m->ssim = ssim_total; 423 if (old_ssim_total == 0) inconsistency_total = 0; 424 425 m->ssimc = inconsistency_total; 426 427 m->dssim = dssim_total; 428 return inconsistency_total; 429 } 430 431 #if CONFIG_VP9_HIGHBITDEPTH 432 double vpx_highbd_calc_ssim(const YV12_BUFFER_CONFIG *source, 433 const YV12_BUFFER_CONFIG *dest, double *weight, 434 uint32_t bd, uint32_t in_bd) { 435 double a, b, c; 436 double ssimv; 437 uint32_t shift = 0; 438 439 assert(bd >= in_bd); 440 shift = bd - in_bd; 441 442 a = vpx_highbd_ssim2(source->y_buffer, dest->y_buffer, source->y_stride, 443 dest->y_stride, source->y_crop_width, 444 source->y_crop_height, in_bd, shift); 445 446 b = vpx_highbd_ssim2(source->u_buffer, dest->u_buffer, source->uv_stride, 447 dest->uv_stride, source->uv_crop_width, 448 source->uv_crop_height, in_bd, shift); 449 450 c = vpx_highbd_ssim2(source->v_buffer, dest->v_buffer, source->uv_stride, 451 dest->uv_stride, source->uv_crop_width, 452 source->uv_crop_height, in_bd, shift); 453 454 ssimv = a * .8 + .1 * (b + c); 455 456 *weight = 1; 457 458 return ssimv; 459 } 460 461 #endif // CONFIG_VP9_HIGHBITDEPTH 462
