1 // Copyright 2010 Google Inc. 2 // 3 // This code is licensed under the same terms as WebM: 4 // Software License Agreement: http://www.webmproject.org/license/software/ 5 // Additional IP Rights Grant: http://www.webmproject.org/license/additional/ 6 // ----------------------------------------------------------------------------- 7 // 8 // speed-critical functions. 9 // 10 // Author: Skal (pascal.massimino (at) gmail.com) 11 12 #include "vp8i.h" 13 14 #if defined(__cplusplus) || defined(c_plusplus) 15 extern "C" { 16 #endif 17 18 //----------------------------------------------------------------------------- 19 // run-time tables (~4k) 20 21 static uint8_t abs0[255 + 255 + 1]; // abs(i) 22 static uint8_t abs1[255 + 255 + 1]; // abs(i)>>1 23 static int8_t sclip1[1020 + 1020 + 1]; // clips [-1020, 1020] to [-128, 127] 24 static int8_t sclip2[112 + 112 + 1]; // clips [-112, 112] to [-16, 15] 25 static uint8_t clip1[255 + 510 + 1]; // clips [-255,510] to [0,255] 26 27 // We declare this variable 'volatile' to prevent instruction reordering 28 // and make sure it's set to true _last_ (so as to be thread-safe) 29 static volatile int tables_ok = 0; 30 31 void VP8DspInitTables(void) { 32 if (!tables_ok) { 33 int i; 34 for (i = -255; i <= 255; ++i) { 35 abs0[255 + i] = (i < 0) ? -i : i; 36 abs1[255 + i] = abs0[255 + i] >> 1; 37 } 38 for (i = -1020; i <= 1020; ++i) { 39 sclip1[1020 + i] = (i < -128) ? -128 : (i > 127) ? 127 : i; 40 } 41 for (i = -112; i <= 112; ++i) { 42 sclip2[112 + i] = (i < -16) ? -16 : (i > 15) ? 15 : i; 43 } 44 for (i = -255; i <= 255 + 255; ++i) { 45 clip1[255 + i] = (i < 0) ? 0 : (i > 255) ? 255 : i; 46 } 47 tables_ok = 1; 48 } 49 } 50 51 static inline uint8_t clip_8b(int v) { 52 return (!(v & ~0xff)) ? v : (v < 0) ? 0 : 255; 53 } 54 55 //----------------------------------------------------------------------------- 56 // Transforms (Paragraph 14.4) 57 58 #define STORE(x, y, v) \ 59 dst[x + y * BPS] = clip_8b(dst[x + y * BPS] + ((v) >> 3)) 60 61 static const int kC1 = 20091 + (1 << 16); 62 static const int kC2 = 35468; 63 #define MUL(a, b) (((a) * (b)) >> 16) 64 65 static void TransformOne(const int16_t* in, uint8_t* dst) { 66 int C[4 * 4], *tmp; 67 int i; 68 tmp = C; 69 for (i = 0; i < 4; ++i) { // vertical pass 70 const int a = in[0] + in[8]; // [-4096, 4094] 71 const int b = in[0] - in[8]; // [-4095, 4095] 72 const int c = MUL(in[4], kC2) - MUL(in[12], kC1); // [-3783, 3783] 73 const int d = MUL(in[4], kC1) + MUL(in[12], kC2); // [-3785, 3781] 74 tmp[0] = a + d; // [-7881, 7875] 75 tmp[1] = b + c; // [-7878, 7878] 76 tmp[2] = b - c; // [-7878, 7878] 77 tmp[3] = a - d; // [-7877, 7879] 78 tmp += 4; 79 in++; 80 } 81 // Each pass is expanding the dynamic range by ~3.85 (upper bound). 82 // The exact value is (2. + (kC1 + kC2) / 65536). 83 // After the second pass, maximum interval is [-3794, 3794], assuming 84 // an input in [-2048, 2047] interval. We then need to add a dst value 85 // in the [0, 255] range. 86 // In the worst case scenario, the input to clip_8b() can be as large as 87 // [-60713, 60968]. 88 tmp = C; 89 for (i = 0; i < 4; ++i) { // horizontal pass 90 const int dc = tmp[0] + 4; 91 const int a = dc + tmp[8]; 92 const int b = dc - tmp[8]; 93 const int c = MUL(tmp[4], kC2) - MUL(tmp[12], kC1); 94 const int d = MUL(tmp[4], kC1) + MUL(tmp[12], kC2); 95 STORE(0, 0, a + d); 96 STORE(1, 0, b + c); 97 STORE(2, 0, b - c); 98 STORE(3, 0, a - d); 99 tmp++; 100 dst += BPS; 101 } 102 } 103 #undef MUL 104 105 static void TransformTwo(const int16_t* in, uint8_t* dst, int do_two) { 106 TransformOne(in, dst); 107 if (do_two) { 108 TransformOne(in + 16, dst + 4); 109 } 110 } 111 112 static void TransformUV(const int16_t* in, uint8_t* dst) { 113 VP8Transform(in + 0 * 16, dst, 1); 114 VP8Transform(in + 2 * 16, dst + 4 * BPS, 1); 115 } 116 117 static void TransformDC(const int16_t *in, uint8_t* dst) { 118 const int DC = in[0] + 4; 119 int i, j; 120 for (j = 0; j < 4; ++j) { 121 for (i = 0; i < 4; ++i) { 122 STORE(i, j, DC); 123 } 124 } 125 } 126 127 static void TransformDCUV(const int16_t* in, uint8_t* dst) { 128 if (in[0 * 16]) TransformDC(in + 0 * 16, dst); 129 if (in[1 * 16]) TransformDC(in + 1 * 16, dst + 4); 130 if (in[2 * 16]) TransformDC(in + 2 * 16, dst + 4 * BPS); 131 if (in[3 * 16]) TransformDC(in + 3 * 16, dst + 4 * BPS + 4); 132 } 133 134 #undef STORE 135 136 // default C implementations: 137 VP8Idct2 VP8Transform = TransformTwo; 138 VP8Idct VP8TransformUV = TransformUV; 139 VP8Idct VP8TransformDC = TransformDC; 140 VP8Idct VP8TransformDCUV = TransformDCUV; 141 142 //----------------------------------------------------------------------------- 143 // Paragraph 14.3 144 145 static void TransformWHT(const int16_t* in, int16_t* out) { 146 int tmp[16]; 147 int i; 148 for (i = 0; i < 4; ++i) { 149 const int a0 = in[0 + i] + in[12 + i]; 150 const int a1 = in[4 + i] + in[ 8 + i]; 151 const int a2 = in[4 + i] - in[ 8 + i]; 152 const int a3 = in[0 + i] - in[12 + i]; 153 tmp[0 + i] = a0 + a1; 154 tmp[8 + i] = a0 - a1; 155 tmp[4 + i] = a3 + a2; 156 tmp[12 + i] = a3 - a2; 157 } 158 for (i = 0; i < 4; ++i) { 159 const int dc = tmp[0 + i * 4] + 3; // w/ rounder 160 const int a0 = dc + tmp[3 + i * 4]; 161 const int a1 = tmp[1 + i * 4] + tmp[2 + i * 4]; 162 const int a2 = tmp[1 + i * 4] - tmp[2 + i * 4]; 163 const int a3 = dc - tmp[3 + i * 4]; 164 out[ 0] = (a0 + a1) >> 3; 165 out[16] = (a3 + a2) >> 3; 166 out[32] = (a0 - a1) >> 3; 167 out[48] = (a3 - a2) >> 3; 168 out += 64; 169 } 170 } 171 172 void (*VP8TransformWHT)(const int16_t* in, int16_t* out) = TransformWHT; 173 174 //----------------------------------------------------------------------------- 175 // Intra predictions 176 177 #define OUT(x, y) dst[(x) + (y) * BPS] 178 179 static inline void TrueMotion(uint8_t *dst, int size) { 180 const uint8_t* top = dst - BPS; 181 const uint8_t* const clip0 = clip1 + 255 - top[-1]; 182 int y; 183 for (y = 0; y < size; ++y) { 184 const uint8_t* const clip = clip0 + dst[-1]; 185 int x; 186 for (x = 0; x < size; ++x) { 187 dst[x] = clip[top[x]]; 188 } 189 dst += BPS; 190 } 191 } 192 static void TM4(uint8_t *dst) { TrueMotion(dst, 4); } 193 static void TM8uv(uint8_t *dst) { TrueMotion(dst, 8); } 194 static void TM16(uint8_t *dst) { TrueMotion(dst, 16); } 195 196 //----------------------------------------------------------------------------- 197 // 16x16 198 199 static void VE16(uint8_t *dst) { // vertical 200 int j; 201 for (j = 0; j < 16; ++j) { 202 memcpy(dst + j * BPS, dst - BPS, 16); 203 } 204 } 205 206 static void HE16(uint8_t *dst) { // horizontal 207 int j; 208 for (j = 16; j > 0; --j) { 209 memset(dst, dst[-1], 16); 210 dst += BPS; 211 } 212 } 213 214 static inline void Put16(int v, uint8_t* dst) { 215 int j; 216 for (j = 0; j < 16; ++j) { 217 memset(dst + j * BPS, v, 16); 218 } 219 } 220 221 static void DC16(uint8_t *dst) { // DC 222 int DC = 16; 223 int j; 224 for (j = 0; j < 16; ++j) { 225 DC += dst[-1 + j * BPS] + dst[j - BPS]; 226 } 227 Put16(DC >> 5, dst); 228 } 229 230 static void DC16NoTop(uint8_t *dst) { // DC with top samples not available 231 int DC = 8; 232 int j; 233 for (j = 0; j < 16; ++j) { 234 DC += dst[-1 + j * BPS]; 235 } 236 Put16(DC >> 4, dst); 237 } 238 239 static void DC16NoLeft(uint8_t *dst) { // DC with left samples not available 240 int DC = 8; 241 int i; 242 for (i = 0; i < 16; ++i) { 243 DC += dst[i - BPS]; 244 } 245 Put16(DC >> 4, dst); 246 } 247 248 static void DC16NoTopLeft(uint8_t *dst) { // DC with no top and left samples 249 Put16(0x80, dst); 250 } 251 252 //----------------------------------------------------------------------------- 253 // 4x4 254 255 #define AVG3(a, b, c) (((a) + 2 * (b) + (c) + 2) >> 2) 256 #define AVG2(a, b) (((a) + (b) + 1) >> 1) 257 258 static void VE4(uint8_t *dst) { // vertical 259 const uint8_t* top = dst - BPS; 260 const uint8_t vals[4] = { 261 AVG3(top[-1], top[0], top[1]), 262 AVG3(top[ 0], top[1], top[2]), 263 AVG3(top[ 1], top[2], top[3]), 264 AVG3(top[ 2], top[3], top[4]) 265 }; 266 int i; 267 for (i = 0; i < 4; ++i) { 268 memcpy(dst + i * BPS, vals, sizeof(vals)); 269 } 270 } 271 272 static void HE4(uint8_t *dst) { // horizontal 273 const int A = dst[-1 - BPS]; 274 const int B = dst[-1]; 275 const int C = dst[-1 + BPS]; 276 const int D = dst[-1 + 2 * BPS]; 277 const int E = dst[-1 + 3 * BPS]; 278 *(uint32_t*)(dst + 0 * BPS) = 0x01010101U * AVG3(A, B, C); 279 *(uint32_t*)(dst + 1 * BPS) = 0x01010101U * AVG3(B, C, D); 280 *(uint32_t*)(dst + 2 * BPS) = 0x01010101U * AVG3(C, D, E); 281 *(uint32_t*)(dst + 3 * BPS) = 0x01010101U * AVG3(D, E, E); 282 } 283 284 static void DC4(uint8_t *dst) { // DC 285 uint32_t dc = 4; 286 int i; 287 for (i = 0; i < 4; ++i) dc += dst[i - BPS] + dst[-1 + i * BPS]; 288 dc >>= 3; 289 for (i = 0; i < 4; ++i) memset(dst + i * BPS, dc, 4); 290 } 291 292 static void RD4(uint8_t *dst) { // Down-right 293 const int I = dst[-1 + 0 * BPS]; 294 const int J = dst[-1 + 1 * BPS]; 295 const int K = dst[-1 + 2 * BPS]; 296 const int L = dst[-1 + 3 * BPS]; 297 const int X = dst[-1 - BPS]; 298 const int A = dst[0 - BPS]; 299 const int B = dst[1 - BPS]; 300 const int C = dst[2 - BPS]; 301 const int D = dst[3 - BPS]; 302 OUT(0, 3) = AVG3(J, K, L); 303 OUT(0, 2) = OUT(1, 3) = AVG3(I, J, K); 304 OUT(0, 1) = OUT(1, 2) = OUT(2, 3) = AVG3(X, I, J); 305 OUT(0, 0) = OUT(1, 1) = OUT(2, 2) = OUT(3, 3) = AVG3(A, X, I); 306 OUT(1, 0) = OUT(2, 1) = OUT(3, 2) = AVG3(B, A, X); 307 OUT(2, 0) = OUT(3, 1) = AVG3(C, B, A); 308 OUT(3, 0) = AVG3(D, C, B); 309 } 310 311 static void LD4(uint8_t *dst) { // Down-Left 312 const int A = dst[0 - BPS]; 313 const int B = dst[1 - BPS]; 314 const int C = dst[2 - BPS]; 315 const int D = dst[3 - BPS]; 316 const int E = dst[4 - BPS]; 317 const int F = dst[5 - BPS]; 318 const int G = dst[6 - BPS]; 319 const int H = dst[7 - BPS]; 320 OUT(0, 0) = AVG3(A, B, C); 321 OUT(1, 0) = OUT(0, 1) = AVG3(B, C, D); 322 OUT(2, 0) = OUT(1, 1) = OUT(0, 2) = AVG3(C, D, E); 323 OUT(3, 0) = OUT(2, 1) = OUT(1, 2) = OUT(0, 3) = AVG3(D, E, F); 324 OUT(3, 1) = OUT(2, 2) = OUT(1, 3) = AVG3(E, F, G); 325 OUT(3, 2) = OUT(2, 3) = AVG3(F, G, H); 326 OUT(3, 3) = AVG3(G, H, H); 327 } 328 329 static void VR4(uint8_t *dst) { // Vertical-Right 330 const int I = dst[-1 + 0 * BPS]; 331 const int J = dst[-1 + 1 * BPS]; 332 const int K = dst[-1 + 2 * BPS]; 333 const int X = dst[-1 - BPS]; 334 const int A = dst[0 - BPS]; 335 const int B = dst[1 - BPS]; 336 const int C = dst[2 - BPS]; 337 const int D = dst[3 - BPS]; 338 OUT(0, 0) = OUT(1, 2) = AVG2(X, A); 339 OUT(1, 0) = OUT(2, 2) = AVG2(A, B); 340 OUT(2, 0) = OUT(3, 2) = AVG2(B, C); 341 OUT(3, 0) = AVG2(C, D); 342 343 OUT(0, 3) = AVG3(K, J, I); 344 OUT(0, 2) = AVG3(J, I, X); 345 OUT(0, 1) = OUT(1, 3) = AVG3(I, X, A); 346 OUT(1, 1) = OUT(2, 3) = AVG3(X, A, B); 347 OUT(2, 1) = OUT(3, 3) = AVG3(A, B, C); 348 OUT(3, 1) = AVG3(B, C, D); 349 } 350 351 static void VL4(uint8_t *dst) { // Vertical-Left 352 const int A = dst[0 - BPS]; 353 const int B = dst[1 - BPS]; 354 const int C = dst[2 - BPS]; 355 const int D = dst[3 - BPS]; 356 const int E = dst[4 - BPS]; 357 const int F = dst[5 - BPS]; 358 const int G = dst[6 - BPS]; 359 const int H = dst[7 - BPS]; 360 OUT(0, 0) = AVG2(A, B); 361 OUT(1, 0) = OUT(0, 2) = AVG2(B, C); 362 OUT(2, 0) = OUT(1, 2) = AVG2(C, D); 363 OUT(3, 0) = OUT(2, 2) = AVG2(D, E); 364 365 OUT(0, 1) = AVG3(A, B, C); 366 OUT(1, 1) = OUT(0, 3) = AVG3(B, C, D); 367 OUT(2, 1) = OUT(1, 3) = AVG3(C, D, E); 368 OUT(3, 1) = OUT(2, 3) = AVG3(D, E, F); 369 OUT(3, 2) = AVG3(E, F, G); 370 OUT(3, 3) = AVG3(F, G, H); 371 } 372 373 static void HU4(uint8_t *dst) { // Horizontal-Up 374 const int I = dst[-1 + 0 * BPS]; 375 const int J = dst[-1 + 1 * BPS]; 376 const int K = dst[-1 + 2 * BPS]; 377 const int L = dst[-1 + 3 * BPS]; 378 OUT(0, 0) = AVG2(I, J); 379 OUT(2, 0) = OUT(0, 1) = AVG2(J, K); 380 OUT(2, 1) = OUT(0, 2) = AVG2(K, L); 381 OUT(1, 0) = AVG3(I, J, K); 382 OUT(3, 0) = OUT(1, 1) = AVG3(J, K, L); 383 OUT(3, 1) = OUT(1, 2) = AVG3(K, L, L); 384 OUT(3, 2) = OUT(2, 2) = 385 OUT(0, 3) = OUT(1, 3) = OUT(2, 3) = OUT(3, 3) = L; 386 } 387 388 static void HD4(uint8_t *dst) { // Horizontal-Down 389 const int I = dst[-1 + 0 * BPS]; 390 const int J = dst[-1 + 1 * BPS]; 391 const int K = dst[-1 + 2 * BPS]; 392 const int L = dst[-1 + 3 * BPS]; 393 const int X = dst[-1 - BPS]; 394 const int A = dst[0 - BPS]; 395 const int B = dst[1 - BPS]; 396 const int C = dst[2 - BPS]; 397 398 OUT(0, 0) = OUT(2, 1) = AVG2(I, X); 399 OUT(0, 1) = OUT(2, 2) = AVG2(J, I); 400 OUT(0, 2) = OUT(2, 3) = AVG2(K, J); 401 OUT(0, 3) = AVG2(L, K); 402 403 OUT(3, 0) = AVG3(A, B, C); 404 OUT(2, 0) = AVG3(X, A, B); 405 OUT(1, 0) = OUT(3, 1) = AVG3(I, X, A); 406 OUT(1, 1) = OUT(3, 2) = AVG3(J, I, X); 407 OUT(1, 2) = OUT(3, 3) = AVG3(K, J, I); 408 OUT(1, 3) = AVG3(L, K, J); 409 } 410 411 #undef AVG3 412 #undef AVG2 413 414 //----------------------------------------------------------------------------- 415 // Chroma 416 417 static void VE8uv(uint8_t *dst) { // vertical 418 int j; 419 for (j = 0; j < 8; ++j) { 420 memcpy(dst + j * BPS, dst - BPS, 8); 421 } 422 } 423 424 static void HE8uv(uint8_t *dst) { // horizontal 425 int j; 426 for (j = 0; j < 8; ++j) { 427 memset(dst, dst[-1], 8); 428 dst += BPS; 429 } 430 } 431 432 // helper for chroma-DC predictions 433 static inline void Put8x8uv(uint64_t v, uint8_t* dst) { 434 int j; 435 for (j = 0; j < 8; ++j) { 436 *(uint64_t*)(dst + j * BPS) = v; 437 } 438 } 439 440 static void DC8uv(uint8_t *dst) { // DC 441 int dc0 = 8; 442 int i; 443 for (i = 0; i < 8; ++i) { 444 dc0 += dst[i - BPS] + dst[-1 + i * BPS]; 445 } 446 Put8x8uv((uint64_t)((dc0 >> 4) * 0x0101010101010101ULL), dst); 447 } 448 449 static void DC8uvNoLeft(uint8_t *dst) { // DC with no left samples 450 int dc0 = 4; 451 int i; 452 for (i = 0; i < 8; ++i) { 453 dc0 += dst[i - BPS]; 454 } 455 Put8x8uv((uint64_t)((dc0 >> 3) * 0x0101010101010101ULL), dst); 456 } 457 458 static void DC8uvNoTop(uint8_t *dst) { // DC with no top samples 459 int dc0 = 4; 460 int i; 461 for (i = 0; i < 8; ++i) { 462 dc0 += dst[-1 + i * BPS]; 463 } 464 Put8x8uv((uint64_t)((dc0 >> 3) * 0x0101010101010101ULL), dst); 465 } 466 467 static void DC8uvNoTopLeft(uint8_t *dst) { // DC with nothing 468 Put8x8uv(0x8080808080808080ULL, dst); 469 } 470 471 //----------------------------------------------------------------------------- 472 // default C implementations 473 474 VP8PredFunc VP8PredLuma4[NUM_BMODES] = { 475 DC4, TM4, VE4, HE4, RD4, VR4, LD4, VL4, HD4, HU4 476 }; 477 478 VP8PredFunc VP8PredLuma16[NUM_B_DC_MODES] = { 479 DC16, TM16, VE16, HE16, 480 DC16NoTop, DC16NoLeft, DC16NoTopLeft 481 }; 482 483 VP8PredFunc VP8PredChroma8[NUM_B_DC_MODES] = { 484 DC8uv, TM8uv, VE8uv, HE8uv, 485 DC8uvNoTop, DC8uvNoLeft, DC8uvNoTopLeft 486 }; 487 488 //----------------------------------------------------------------------------- 489 // Edge filtering functions 490 491 // 4 pixels in, 2 pixels out 492 static inline void do_filter2(uint8_t* p, int step) { 493 const int p1 = p[-2*step], p0 = p[-step], q0 = p[0], q1 = p[step]; 494 const int a = 3 * (q0 - p0) + sclip1[1020 + p1 - q1]; 495 const int a1 = sclip2[112 + ((a + 4) >> 3)]; 496 const int a2 = sclip2[112 + ((a + 3) >> 3)]; 497 p[-step] = clip1[255 + p0 + a2]; 498 p[ 0] = clip1[255 + q0 - a1]; 499 } 500 501 // 4 pixels in, 4 pixels out 502 static inline void do_filter4(uint8_t* p, int step) { 503 const int p1 = p[-2*step], p0 = p[-step], q0 = p[0], q1 = p[step]; 504 const int a = 3 * (q0 - p0); 505 const int a1 = sclip2[112 + ((a + 4) >> 3)]; 506 const int a2 = sclip2[112 + ((a + 3) >> 3)]; 507 const int a3 = (a1 + 1) >> 1; 508 p[-2*step] = clip1[255 + p1 + a3]; 509 p[- step] = clip1[255 + p0 + a2]; 510 p[ 0] = clip1[255 + q0 - a1]; 511 p[ step] = clip1[255 + q1 - a3]; 512 } 513 514 // 6 pixels in, 6 pixels out 515 static inline void do_filter6(uint8_t* p, int step) { 516 const int p2 = p[-3*step], p1 = p[-2*step], p0 = p[-step]; 517 const int q0 = p[0], q1 = p[step], q2 = p[2*step]; 518 const int a = sclip1[1020 + 3 * (q0 - p0) + sclip1[1020 + p1 - q1]]; 519 const int a1 = (27 * a + 63) >> 7; // eq. to ((3 * a + 7) * 9) >> 7 520 const int a2 = (18 * a + 63) >> 7; // eq. to ((2 * a + 7) * 9) >> 7 521 const int a3 = (9 * a + 63) >> 7; // eq. to ((1 * a + 7) * 9) >> 7 522 p[-3*step] = clip1[255 + p2 + a3]; 523 p[-2*step] = clip1[255 + p1 + a2]; 524 p[- step] = clip1[255 + p0 + a1]; 525 p[ 0] = clip1[255 + q0 - a1]; 526 p[ step] = clip1[255 + q1 - a2]; 527 p[ 2*step] = clip1[255 + q2 - a3]; 528 } 529 530 static inline int hev(const uint8_t* p, int step, int thresh) { 531 const int p1 = p[-2*step], p0 = p[-step], q0 = p[0], q1 = p[step]; 532 return (abs0[255 + p1 - p0] > thresh) || (abs0[255 + q1 - q0] > thresh); 533 } 534 535 static inline int needs_filter(const uint8_t* p, int step, int thresh) { 536 const int p1 = p[-2*step], p0 = p[-step], q0 = p[0], q1 = p[step]; 537 return (2 * abs0[255 + p0 - q0] + abs1[255 + p1 - q1]) <= thresh; 538 } 539 540 static inline int needs_filter2(const uint8_t* p, int step, int t, int it) { 541 const int p3 = p[-4*step], p2 = p[-3*step], p1 = p[-2*step], p0 = p[-step]; 542 const int q0 = p[0], q1 = p[step], q2 = p[2*step], q3 = p[3*step]; 543 if ((2 * abs0[255 + p0 - q0] + abs1[255 + p1 - q1]) > t) 544 return 0; 545 return abs0[255 + p3 - p2] <= it && abs0[255 + p2 - p1] <= it && 546 abs0[255 + p1 - p0] <= it && abs0[255 + q3 - q2] <= it && 547 abs0[255 + q2 - q1] <= it && abs0[255 + q1 - q0] <= it; 548 } 549 550 //----------------------------------------------------------------------------- 551 // Simple In-loop filtering (Paragraph 15.2) 552 553 static void SimpleVFilter16(uint8_t* p, int stride, int thresh) { 554 int i; 555 for (i = 0; i < 16; ++i) { 556 if (needs_filter(p + i, stride, thresh)) { 557 do_filter2(p + i, stride); 558 } 559 } 560 } 561 562 static void SimpleHFilter16(uint8_t* p, int stride, int thresh) { 563 int i; 564 for (i = 0; i < 16; ++i) { 565 if (needs_filter(p + i * stride, 1, thresh)) { 566 do_filter2(p + i * stride, 1); 567 } 568 } 569 } 570 571 static void SimpleVFilter16i(uint8_t* p, int stride, int thresh) { 572 int k; 573 for (k = 3; k > 0; --k) { 574 p += 4 * stride; 575 SimpleVFilter16(p, stride, thresh); 576 } 577 } 578 579 static void SimpleHFilter16i(uint8_t* p, int stride, int thresh) { 580 int k; 581 for (k = 3; k > 0; --k) { 582 p += 4; 583 SimpleHFilter16(p, stride, thresh); 584 } 585 } 586 587 //----------------------------------------------------------------------------- 588 // Complex In-loop filtering (Paragraph 15.3) 589 590 static inline void FilterLoop26(uint8_t* p, int hstride, int vstride, int size, 591 int thresh, int ithresh, int hev_thresh) { 592 while (size-- > 0) { 593 if (needs_filter2(p, hstride, thresh, ithresh)) { 594 if (hev(p, hstride, hev_thresh)) { 595 do_filter2(p, hstride); 596 } else { 597 do_filter6(p, hstride); 598 } 599 } 600 p += vstride; 601 } 602 } 603 604 static inline void FilterLoop24(uint8_t* p, int hstride, int vstride, int size, 605 int thresh, int ithresh, int hev_thresh) { 606 while (size-- > 0) { 607 if (needs_filter2(p, hstride, thresh, ithresh)) { 608 if (hev(p, hstride, hev_thresh)) { 609 do_filter2(p, hstride); 610 } else { 611 do_filter4(p, hstride); 612 } 613 } 614 p += vstride; 615 } 616 } 617 618 // on macroblock edges 619 static void VFilter16(uint8_t* p, int stride, 620 int thresh, int ithresh, int hev_thresh) { 621 FilterLoop26(p, stride, 1, 16, thresh, ithresh, hev_thresh); 622 } 623 624 static void HFilter16(uint8_t* p, int stride, 625 int thresh, int ithresh, int hev_thresh) { 626 FilterLoop26(p, 1, stride, 16, thresh, ithresh, hev_thresh); 627 } 628 629 // on three inner edges 630 static void VFilter16i(uint8_t* p, int stride, 631 int thresh, int ithresh, int hev_thresh) { 632 int k; 633 for (k = 3; k > 0; --k) { 634 p += 4 * stride; 635 FilterLoop24(p, stride, 1, 16, thresh, ithresh, hev_thresh); 636 } 637 } 638 639 static void HFilter16i(uint8_t* p, int stride, 640 int thresh, int ithresh, int hev_thresh) { 641 int k; 642 for (k = 3; k > 0; --k) { 643 p += 4; 644 FilterLoop24(p, 1, stride, 16, thresh, ithresh, hev_thresh); 645 } 646 } 647 648 // 8-pixels wide variant, for chroma filtering 649 static void VFilter8(uint8_t* u, uint8_t* v, int stride, 650 int thresh, int ithresh, int hev_thresh) { 651 FilterLoop26(u, stride, 1, 8, thresh, ithresh, hev_thresh); 652 FilterLoop26(v, stride, 1, 8, thresh, ithresh, hev_thresh); 653 } 654 655 static void HFilter8(uint8_t* u, uint8_t* v, int stride, 656 int thresh, int ithresh, int hev_thresh) { 657 FilterLoop26(u, 1, stride, 8, thresh, ithresh, hev_thresh); 658 FilterLoop26(v, 1, stride, 8, thresh, ithresh, hev_thresh); 659 } 660 661 static void VFilter8i(uint8_t* u, uint8_t* v, int stride, 662 int thresh, int ithresh, int hev_thresh) { 663 FilterLoop24(u + 4 * stride, stride, 1, 8, thresh, ithresh, hev_thresh); 664 FilterLoop24(v + 4 * stride, stride, 1, 8, thresh, ithresh, hev_thresh); 665 } 666 667 static void HFilter8i(uint8_t* u, uint8_t* v, int stride, 668 int thresh, int ithresh, int hev_thresh) { 669 FilterLoop24(u + 4, 1, stride, 8, thresh, ithresh, hev_thresh); 670 FilterLoop24(v + 4, 1, stride, 8, thresh, ithresh, hev_thresh); 671 } 672 673 //----------------------------------------------------------------------------- 674 675 void (*VP8VFilter16)(uint8_t*, int, int, int, int) = VFilter16; 676 void (*VP8HFilter16)(uint8_t*, int, int, int, int) = HFilter16; 677 void (*VP8VFilter8)(uint8_t*, uint8_t*, int, int, int, int) = VFilter8; 678 void (*VP8HFilter8)(uint8_t*, uint8_t*, int, int, int, int) = HFilter8; 679 void (*VP8VFilter16i)(uint8_t*, int, int, int, int) = VFilter16i; 680 void (*VP8HFilter16i)(uint8_t*, int, int, int, int) = HFilter16i; 681 void (*VP8VFilter8i)(uint8_t*, uint8_t*, int, int, int, int) = VFilter8i; 682 void (*VP8HFilter8i)(uint8_t*, uint8_t*, int, int, int, int) = HFilter8i; 683 684 void (*VP8SimpleVFilter16)(uint8_t*, int, int) = SimpleVFilter16; 685 void (*VP8SimpleHFilter16)(uint8_t*, int, int) = SimpleHFilter16; 686 void (*VP8SimpleVFilter16i)(uint8_t*, int, int) = SimpleVFilter16i; 687 void (*VP8SimpleHFilter16i)(uint8_t*, int, int) = SimpleHFilter16i; 688 689 //----------------------------------------------------------------------------- 690 // SSE2 detection. 691 // 692 693 #if defined(__pic__) && defined(__i386__) 694 static inline void GetCPUInfo(int cpu_info[4], int info_type) { 695 __asm__ volatile ( 696 "mov %%ebx, %%edi\n" 697 "cpuid\n" 698 "xchg %%edi, %%ebx\n" 699 : "=a"(cpu_info[0]), "=D"(cpu_info[1]), "=c"(cpu_info[2]), "=d"(cpu_info[3]) 700 : "a"(info_type)); 701 } 702 #elif defined(__i386__) || defined(__x86_64__) 703 static inline void GetCPUInfo(int cpu_info[4], int info_type) { 704 __asm__ volatile ( 705 "cpuid\n" 706 : "=a"(cpu_info[0]), "=b"(cpu_info[1]), "=c"(cpu_info[2]), "=d"(cpu_info[3]) 707 : "a"(info_type)); 708 } 709 #elif defined(_MSC_VER) // Visual C++ 710 #define GetCPUInfo __cpuid 711 #endif 712 713 #if defined(__i386__) || defined(__x86_64__) || defined(_MSC_VER) 714 static int x86CPUInfo(CPUFeature feature) { 715 int cpu_info[4]; 716 GetCPUInfo(cpu_info, 1); 717 if (feature == kSSE2) { 718 return 0 != (cpu_info[3] & 0x04000000); 719 } 720 if (feature == kSSE3) { 721 return 0 != (cpu_info[2] & 0x00000001); 722 } 723 return 0; 724 } 725 VP8CPUInfo VP8DecGetCPUInfo = x86CPUInfo; 726 #else 727 VP8CPUInfo VP8DecGetCPUInfo = NULL; 728 #endif 729 730 //----------------------------------------------------------------------------- 731 732 extern void VP8DspInitSSE2(void); 733 734 void VP8DspInit(void) { 735 // If defined, use CPUInfo() to overwrite some pointers with faster versions. 736 if (VP8DecGetCPUInfo) { 737 if (VP8DecGetCPUInfo(kSSE2)) { 738 #if defined(__SSE2__) || defined(_MSC_VER) 739 VP8DspInitSSE2(); 740 #endif 741 } 742 if (VP8DecGetCPUInfo(kSSE3)) { 743 // later we'll plug some SSE3 variant here 744 } 745 } 746 } 747 748 #if defined(__cplusplus) || defined(c_plusplus) 749 } // extern "C" 750 #endif 751
