1 // Copyright 2015 Google Inc. All Rights Reserved. 2 // 3 // Use of this source code is governed by a BSD-style license 4 // that can be found in the COPYING file in the root of the source 5 // tree. An additional intellectual property rights grant can be found 6 // in the file PATENTS. All contributing project authors may 7 // be found in the AUTHORS file in the root of the source tree. 8 // ----------------------------------------------------------------------------- 9 // 10 // SSE2 variant of methods for lossless encoder 11 // 12 // Author: Skal (pascal.massimino (at) gmail.com) 13 14 #include "src/dsp/dsp.h" 15 16 #if defined(WEBP_USE_SSE2) 17 #include <assert.h> 18 #include <emmintrin.h> 19 #include "src/dsp/lossless.h" 20 #include "src/dsp/common_sse2.h" 21 #include "src/dsp/lossless_common.h" 22 23 // For sign-extended multiplying constants, pre-shifted by 5: 24 #define CST_5b(X) (((int16_t)((uint16_t)(X) << 8)) >> 5) 25 26 //------------------------------------------------------------------------------ 27 // Subtract-Green Transform 28 29 static void SubtractGreenFromBlueAndRed_SSE2(uint32_t* argb_data, 30 int num_pixels) { 31 int i; 32 for (i = 0; i + 4 <= num_pixels; i += 4) { 33 const __m128i in = _mm_loadu_si128((__m128i*)&argb_data[i]); // argb 34 const __m128i A = _mm_srli_epi16(in, 8); // 0 a 0 g 35 const __m128i B = _mm_shufflelo_epi16(A, _MM_SHUFFLE(2, 2, 0, 0)); 36 const __m128i C = _mm_shufflehi_epi16(B, _MM_SHUFFLE(2, 2, 0, 0)); // 0g0g 37 const __m128i out = _mm_sub_epi8(in, C); 38 _mm_storeu_si128((__m128i*)&argb_data[i], out); 39 } 40 // fallthrough and finish off with plain-C 41 if (i != num_pixels) { 42 VP8LSubtractGreenFromBlueAndRed_C(argb_data + i, num_pixels - i); 43 } 44 } 45 46 //------------------------------------------------------------------------------ 47 // Color Transform 48 49 static void TransformColor_SSE2(const VP8LMultipliers* const m, 50 uint32_t* argb_data, int num_pixels) { 51 const __m128i mults_rb = _mm_set_epi16( 52 CST_5b(m->green_to_red_), CST_5b(m->green_to_blue_), 53 CST_5b(m->green_to_red_), CST_5b(m->green_to_blue_), 54 CST_5b(m->green_to_red_), CST_5b(m->green_to_blue_), 55 CST_5b(m->green_to_red_), CST_5b(m->green_to_blue_)); 56 const __m128i mults_b2 = _mm_set_epi16( 57 CST_5b(m->red_to_blue_), 0, CST_5b(m->red_to_blue_), 0, 58 CST_5b(m->red_to_blue_), 0, CST_5b(m->red_to_blue_), 0); 59 const __m128i mask_ag = _mm_set1_epi32(0xff00ff00); // alpha-green masks 60 const __m128i mask_rb = _mm_set1_epi32(0x00ff00ff); // red-blue masks 61 int i; 62 for (i = 0; i + 4 <= num_pixels; i += 4) { 63 const __m128i in = _mm_loadu_si128((__m128i*)&argb_data[i]); // argb 64 const __m128i A = _mm_and_si128(in, mask_ag); // a 0 g 0 65 const __m128i B = _mm_shufflelo_epi16(A, _MM_SHUFFLE(2, 2, 0, 0)); 66 const __m128i C = _mm_shufflehi_epi16(B, _MM_SHUFFLE(2, 2, 0, 0)); // g0g0 67 const __m128i D = _mm_mulhi_epi16(C, mults_rb); // x dr x db1 68 const __m128i E = _mm_slli_epi16(in, 8); // r 0 b 0 69 const __m128i F = _mm_mulhi_epi16(E, mults_b2); // x db2 0 0 70 const __m128i G = _mm_srli_epi32(F, 16); // 0 0 x db2 71 const __m128i H = _mm_add_epi8(G, D); // x dr x db 72 const __m128i I = _mm_and_si128(H, mask_rb); // 0 dr 0 db 73 const __m128i out = _mm_sub_epi8(in, I); 74 _mm_storeu_si128((__m128i*)&argb_data[i], out); 75 } 76 // fallthrough and finish off with plain-C 77 if (i != num_pixels) { 78 VP8LTransformColor_C(m, argb_data + i, num_pixels - i); 79 } 80 } 81 82 //------------------------------------------------------------------------------ 83 #define SPAN 8 84 static void CollectColorBlueTransforms_SSE2(const uint32_t* argb, int stride, 85 int tile_width, int tile_height, 86 int green_to_blue, int red_to_blue, 87 int histo[]) { 88 const __m128i mults_r = _mm_set_epi16( 89 CST_5b(red_to_blue), 0, CST_5b(red_to_blue), 0, 90 CST_5b(red_to_blue), 0, CST_5b(red_to_blue), 0); 91 const __m128i mults_g = _mm_set_epi16( 92 0, CST_5b(green_to_blue), 0, CST_5b(green_to_blue), 93 0, CST_5b(green_to_blue), 0, CST_5b(green_to_blue)); 94 const __m128i mask_g = _mm_set1_epi32(0x00ff00); // green mask 95 const __m128i mask_b = _mm_set1_epi32(0x0000ff); // blue mask 96 int y; 97 for (y = 0; y < tile_height; ++y) { 98 const uint32_t* const src = argb + y * stride; 99 int i, x; 100 for (x = 0; x + SPAN <= tile_width; x += SPAN) { 101 uint16_t values[SPAN]; 102 const __m128i in0 = _mm_loadu_si128((__m128i*)&src[x + 0]); 103 const __m128i in1 = _mm_loadu_si128((__m128i*)&src[x + SPAN / 2]); 104 const __m128i A0 = _mm_slli_epi16(in0, 8); // r 0 | b 0 105 const __m128i A1 = _mm_slli_epi16(in1, 8); 106 const __m128i B0 = _mm_and_si128(in0, mask_g); // 0 0 | g 0 107 const __m128i B1 = _mm_and_si128(in1, mask_g); 108 const __m128i C0 = _mm_mulhi_epi16(A0, mults_r); // x db | 0 0 109 const __m128i C1 = _mm_mulhi_epi16(A1, mults_r); 110 const __m128i D0 = _mm_mulhi_epi16(B0, mults_g); // 0 0 | x db 111 const __m128i D1 = _mm_mulhi_epi16(B1, mults_g); 112 const __m128i E0 = _mm_sub_epi8(in0, D0); // x x | x b' 113 const __m128i E1 = _mm_sub_epi8(in1, D1); 114 const __m128i F0 = _mm_srli_epi32(C0, 16); // 0 0 | x db 115 const __m128i F1 = _mm_srli_epi32(C1, 16); 116 const __m128i G0 = _mm_sub_epi8(E0, F0); // 0 0 | x b' 117 const __m128i G1 = _mm_sub_epi8(E1, F1); 118 const __m128i H0 = _mm_and_si128(G0, mask_b); // 0 0 | 0 b 119 const __m128i H1 = _mm_and_si128(G1, mask_b); 120 const __m128i I = _mm_packs_epi32(H0, H1); // 0 b' | 0 b' 121 _mm_storeu_si128((__m128i*)values, I); 122 for (i = 0; i < SPAN; ++i) ++histo[values[i]]; 123 } 124 } 125 { 126 const int left_over = tile_width & (SPAN - 1); 127 if (left_over > 0) { 128 VP8LCollectColorBlueTransforms_C(argb + tile_width - left_over, stride, 129 left_over, tile_height, 130 green_to_blue, red_to_blue, histo); 131 } 132 } 133 } 134 135 static void CollectColorRedTransforms_SSE2(const uint32_t* argb, int stride, 136 int tile_width, int tile_height, 137 int green_to_red, int histo[]) { 138 const __m128i mults_g = _mm_set_epi16( 139 0, CST_5b(green_to_red), 0, CST_5b(green_to_red), 140 0, CST_5b(green_to_red), 0, CST_5b(green_to_red)); 141 const __m128i mask_g = _mm_set1_epi32(0x00ff00); // green mask 142 const __m128i mask = _mm_set1_epi32(0xff); 143 144 int y; 145 for (y = 0; y < tile_height; ++y) { 146 const uint32_t* const src = argb + y * stride; 147 int i, x; 148 for (x = 0; x + SPAN <= tile_width; x += SPAN) { 149 uint16_t values[SPAN]; 150 const __m128i in0 = _mm_loadu_si128((__m128i*)&src[x + 0]); 151 const __m128i in1 = _mm_loadu_si128((__m128i*)&src[x + SPAN / 2]); 152 const __m128i A0 = _mm_and_si128(in0, mask_g); // 0 0 | g 0 153 const __m128i A1 = _mm_and_si128(in1, mask_g); 154 const __m128i B0 = _mm_srli_epi32(in0, 16); // 0 0 | x r 155 const __m128i B1 = _mm_srli_epi32(in1, 16); 156 const __m128i C0 = _mm_mulhi_epi16(A0, mults_g); // 0 0 | x dr 157 const __m128i C1 = _mm_mulhi_epi16(A1, mults_g); 158 const __m128i E0 = _mm_sub_epi8(B0, C0); // x x | x r' 159 const __m128i E1 = _mm_sub_epi8(B1, C1); 160 const __m128i F0 = _mm_and_si128(E0, mask); // 0 0 | 0 r' 161 const __m128i F1 = _mm_and_si128(E1, mask); 162 const __m128i I = _mm_packs_epi32(F0, F1); 163 _mm_storeu_si128((__m128i*)values, I); 164 for (i = 0; i < SPAN; ++i) ++histo[values[i]]; 165 } 166 } 167 { 168 const int left_over = tile_width & (SPAN - 1); 169 if (left_over > 0) { 170 VP8LCollectColorRedTransforms_C(argb + tile_width - left_over, stride, 171 left_over, tile_height, 172 green_to_red, histo); 173 } 174 } 175 } 176 #undef SPAN 177 178 //------------------------------------------------------------------------------ 179 180 #define LINE_SIZE 16 // 8 or 16 181 static void AddVector_SSE2(const uint32_t* a, const uint32_t* b, uint32_t* out, 182 int size) { 183 int i; 184 assert(size % LINE_SIZE == 0); 185 for (i = 0; i < size; i += LINE_SIZE) { 186 const __m128i a0 = _mm_loadu_si128((const __m128i*)&a[i + 0]); 187 const __m128i a1 = _mm_loadu_si128((const __m128i*)&a[i + 4]); 188 #if (LINE_SIZE == 16) 189 const __m128i a2 = _mm_loadu_si128((const __m128i*)&a[i + 8]); 190 const __m128i a3 = _mm_loadu_si128((const __m128i*)&a[i + 12]); 191 #endif 192 const __m128i b0 = _mm_loadu_si128((const __m128i*)&b[i + 0]); 193 const __m128i b1 = _mm_loadu_si128((const __m128i*)&b[i + 4]); 194 #if (LINE_SIZE == 16) 195 const __m128i b2 = _mm_loadu_si128((const __m128i*)&b[i + 8]); 196 const __m128i b3 = _mm_loadu_si128((const __m128i*)&b[i + 12]); 197 #endif 198 _mm_storeu_si128((__m128i*)&out[i + 0], _mm_add_epi32(a0, b0)); 199 _mm_storeu_si128((__m128i*)&out[i + 4], _mm_add_epi32(a1, b1)); 200 #if (LINE_SIZE == 16) 201 _mm_storeu_si128((__m128i*)&out[i + 8], _mm_add_epi32(a2, b2)); 202 _mm_storeu_si128((__m128i*)&out[i + 12], _mm_add_epi32(a3, b3)); 203 #endif 204 } 205 } 206 207 static void AddVectorEq_SSE2(const uint32_t* a, uint32_t* out, int size) { 208 int i; 209 assert(size % LINE_SIZE == 0); 210 for (i = 0; i < size; i += LINE_SIZE) { 211 const __m128i a0 = _mm_loadu_si128((const __m128i*)&a[i + 0]); 212 const __m128i a1 = _mm_loadu_si128((const __m128i*)&a[i + 4]); 213 #if (LINE_SIZE == 16) 214 const __m128i a2 = _mm_loadu_si128((const __m128i*)&a[i + 8]); 215 const __m128i a3 = _mm_loadu_si128((const __m128i*)&a[i + 12]); 216 #endif 217 const __m128i b0 = _mm_loadu_si128((const __m128i*)&out[i + 0]); 218 const __m128i b1 = _mm_loadu_si128((const __m128i*)&out[i + 4]); 219 #if (LINE_SIZE == 16) 220 const __m128i b2 = _mm_loadu_si128((const __m128i*)&out[i + 8]); 221 const __m128i b3 = _mm_loadu_si128((const __m128i*)&out[i + 12]); 222 #endif 223 _mm_storeu_si128((__m128i*)&out[i + 0], _mm_add_epi32(a0, b0)); 224 _mm_storeu_si128((__m128i*)&out[i + 4], _mm_add_epi32(a1, b1)); 225 #if (LINE_SIZE == 16) 226 _mm_storeu_si128((__m128i*)&out[i + 8], _mm_add_epi32(a2, b2)); 227 _mm_storeu_si128((__m128i*)&out[i + 12], _mm_add_epi32(a3, b3)); 228 #endif 229 } 230 } 231 #undef LINE_SIZE 232 233 // Note we are adding uint32_t's as *signed* int32's (using _mm_add_epi32). But 234 // that's ok since the histogram values are less than 1<<28 (max picture size). 235 static void HistogramAdd_SSE2(const VP8LHistogram* const a, 236 const VP8LHistogram* const b, 237 VP8LHistogram* const out) { 238 int i; 239 const int literal_size = VP8LHistogramNumCodes(a->palette_code_bits_); 240 assert(a->palette_code_bits_ == b->palette_code_bits_); 241 if (b != out) { 242 AddVector_SSE2(a->literal_, b->literal_, out->literal_, NUM_LITERAL_CODES); 243 AddVector_SSE2(a->red_, b->red_, out->red_, NUM_LITERAL_CODES); 244 AddVector_SSE2(a->blue_, b->blue_, out->blue_, NUM_LITERAL_CODES); 245 AddVector_SSE2(a->alpha_, b->alpha_, out->alpha_, NUM_LITERAL_CODES); 246 } else { 247 AddVectorEq_SSE2(a->literal_, out->literal_, NUM_LITERAL_CODES); 248 AddVectorEq_SSE2(a->red_, out->red_, NUM_LITERAL_CODES); 249 AddVectorEq_SSE2(a->blue_, out->blue_, NUM_LITERAL_CODES); 250 AddVectorEq_SSE2(a->alpha_, out->alpha_, NUM_LITERAL_CODES); 251 } 252 for (i = NUM_LITERAL_CODES; i < literal_size; ++i) { 253 out->literal_[i] = a->literal_[i] + b->literal_[i]; 254 } 255 for (i = 0; i < NUM_DISTANCE_CODES; ++i) { 256 out->distance_[i] = a->distance_[i] + b->distance_[i]; 257 } 258 } 259 260 //------------------------------------------------------------------------------ 261 // Entropy 262 263 // Checks whether the X or Y contribution is worth computing and adding. 264 // Used in loop unrolling. 265 #define ANALYZE_X_OR_Y(x_or_y, j) \ 266 do { \ 267 if ((x_or_y)[i + (j)] != 0) retval -= VP8LFastSLog2((x_or_y)[i + (j)]); \ 268 } while (0) 269 270 // Checks whether the X + Y contribution is worth computing and adding. 271 // Used in loop unrolling. 272 #define ANALYZE_XY(j) \ 273 do { \ 274 if (tmp[j] != 0) { \ 275 retval -= VP8LFastSLog2(tmp[j]); \ 276 ANALYZE_X_OR_Y(X, j); \ 277 } \ 278 } while (0) 279 280 static float CombinedShannonEntropy_SSE2(const int X[256], const int Y[256]) { 281 int i; 282 double retval = 0.; 283 int sumX, sumXY; 284 int32_t tmp[4]; 285 __m128i zero = _mm_setzero_si128(); 286 // Sums up X + Y, 4 ints at a time (and will merge it at the end for sumXY). 287 __m128i sumXY_128 = zero; 288 __m128i sumX_128 = zero; 289 290 for (i = 0; i < 256; i += 4) { 291 const __m128i x = _mm_loadu_si128((const __m128i*)(X + i)); 292 const __m128i y = _mm_loadu_si128((const __m128i*)(Y + i)); 293 294 // Check if any X is non-zero: this actually provides a speedup as X is 295 // usually sparse. 296 if (_mm_movemask_epi8(_mm_cmpeq_epi32(x, zero)) != 0xFFFF) { 297 const __m128i xy_128 = _mm_add_epi32(x, y); 298 sumXY_128 = _mm_add_epi32(sumXY_128, xy_128); 299 300 sumX_128 = _mm_add_epi32(sumX_128, x); 301 302 // Analyze the different X + Y. 303 _mm_storeu_si128((__m128i*)tmp, xy_128); 304 305 ANALYZE_XY(0); 306 ANALYZE_XY(1); 307 ANALYZE_XY(2); 308 ANALYZE_XY(3); 309 } else { 310 // X is fully 0, so only deal with Y. 311 sumXY_128 = _mm_add_epi32(sumXY_128, y); 312 313 ANALYZE_X_OR_Y(Y, 0); 314 ANALYZE_X_OR_Y(Y, 1); 315 ANALYZE_X_OR_Y(Y, 2); 316 ANALYZE_X_OR_Y(Y, 3); 317 } 318 } 319 320 // Sum up sumX_128 to get sumX. 321 _mm_storeu_si128((__m128i*)tmp, sumX_128); 322 sumX = tmp[3] + tmp[2] + tmp[1] + tmp[0]; 323 324 // Sum up sumXY_128 to get sumXY. 325 _mm_storeu_si128((__m128i*)tmp, sumXY_128); 326 sumXY = tmp[3] + tmp[2] + tmp[1] + tmp[0]; 327 328 retval += VP8LFastSLog2(sumX) + VP8LFastSLog2(sumXY); 329 return (float)retval; 330 } 331 #undef ANALYZE_X_OR_Y 332 #undef ANALYZE_XY 333 334 //------------------------------------------------------------------------------ 335 336 static int VectorMismatch_SSE2(const uint32_t* const array1, 337 const uint32_t* const array2, int length) { 338 int match_len; 339 340 if (length >= 12) { 341 __m128i A0 = _mm_loadu_si128((const __m128i*)&array1[0]); 342 __m128i A1 = _mm_loadu_si128((const __m128i*)&array2[0]); 343 match_len = 0; 344 do { 345 // Loop unrolling and early load both provide a speedup of 10% for the 346 // current function. Also, max_limit can be MAX_LENGTH=4096 at most. 347 const __m128i cmpA = _mm_cmpeq_epi32(A0, A1); 348 const __m128i B0 = 349 _mm_loadu_si128((const __m128i*)&array1[match_len + 4]); 350 const __m128i B1 = 351 _mm_loadu_si128((const __m128i*)&array2[match_len + 4]); 352 if (_mm_movemask_epi8(cmpA) != 0xffff) break; 353 match_len += 4; 354 355 { 356 const __m128i cmpB = _mm_cmpeq_epi32(B0, B1); 357 A0 = _mm_loadu_si128((const __m128i*)&array1[match_len + 4]); 358 A1 = _mm_loadu_si128((const __m128i*)&array2[match_len + 4]); 359 if (_mm_movemask_epi8(cmpB) != 0xffff) break; 360 match_len += 4; 361 } 362 } while (match_len + 12 < length); 363 } else { 364 match_len = 0; 365 // Unroll the potential first two loops. 366 if (length >= 4 && 367 _mm_movemask_epi8(_mm_cmpeq_epi32( 368 _mm_loadu_si128((const __m128i*)&array1[0]), 369 _mm_loadu_si128((const __m128i*)&array2[0]))) == 0xffff) { 370 match_len = 4; 371 if (length >= 8 && 372 _mm_movemask_epi8(_mm_cmpeq_epi32( 373 _mm_loadu_si128((const __m128i*)&array1[4]), 374 _mm_loadu_si128((const __m128i*)&array2[4]))) == 0xffff) { 375 match_len = 8; 376 } 377 } 378 } 379 380 while (match_len < length && array1[match_len] == array2[match_len]) { 381 ++match_len; 382 } 383 return match_len; 384 } 385 386 // Bundles multiple (1, 2, 4 or 8) pixels into a single pixel. 387 static void BundleColorMap_SSE2(const uint8_t* const row, int width, int xbits, 388 uint32_t* dst) { 389 int x; 390 assert(xbits >= 0); 391 assert(xbits <= 3); 392 switch (xbits) { 393 case 0: { 394 const __m128i ff = _mm_set1_epi16(0xff00); 395 const __m128i zero = _mm_setzero_si128(); 396 // Store 0xff000000 | (row[x] << 8). 397 for (x = 0; x + 16 <= width; x += 16, dst += 16) { 398 const __m128i in = _mm_loadu_si128((const __m128i*)&row[x]); 399 const __m128i in_lo = _mm_unpacklo_epi8(zero, in); 400 const __m128i dst0 = _mm_unpacklo_epi16(in_lo, ff); 401 const __m128i dst1 = _mm_unpackhi_epi16(in_lo, ff); 402 const __m128i in_hi = _mm_unpackhi_epi8(zero, in); 403 const __m128i dst2 = _mm_unpacklo_epi16(in_hi, ff); 404 const __m128i dst3 = _mm_unpackhi_epi16(in_hi, ff); 405 _mm_storeu_si128((__m128i*)&dst[0], dst0); 406 _mm_storeu_si128((__m128i*)&dst[4], dst1); 407 _mm_storeu_si128((__m128i*)&dst[8], dst2); 408 _mm_storeu_si128((__m128i*)&dst[12], dst3); 409 } 410 break; 411 } 412 case 1: { 413 const __m128i ff = _mm_set1_epi16(0xff00); 414 const __m128i mul = _mm_set1_epi16(0x110); 415 for (x = 0; x + 16 <= width; x += 16, dst += 8) { 416 // 0a0b | (where a/b are 4 bits). 417 const __m128i in = _mm_loadu_si128((const __m128i*)&row[x]); 418 const __m128i tmp = _mm_mullo_epi16(in, mul); // aba0 419 const __m128i pack = _mm_and_si128(tmp, ff); // ab00 420 const __m128i dst0 = _mm_unpacklo_epi16(pack, ff); 421 const __m128i dst1 = _mm_unpackhi_epi16(pack, ff); 422 _mm_storeu_si128((__m128i*)&dst[0], dst0); 423 _mm_storeu_si128((__m128i*)&dst[4], dst1); 424 } 425 break; 426 } 427 case 2: { 428 const __m128i mask_or = _mm_set1_epi32(0xff000000); 429 const __m128i mul_cst = _mm_set1_epi16(0x0104); 430 const __m128i mask_mul = _mm_set1_epi16(0x0f00); 431 for (x = 0; x + 16 <= width; x += 16, dst += 4) { 432 // 000a000b000c000d | (where a/b/c/d are 2 bits). 433 const __m128i in = _mm_loadu_si128((const __m128i*)&row[x]); 434 const __m128i mul = _mm_mullo_epi16(in, mul_cst); // 00ab00b000cd00d0 435 const __m128i tmp = _mm_and_si128(mul, mask_mul); // 00ab000000cd0000 436 const __m128i shift = _mm_srli_epi32(tmp, 12); // 00000000ab000000 437 const __m128i pack = _mm_or_si128(shift, tmp); // 00000000abcd0000 438 // Convert to 0xff00**00. 439 const __m128i res = _mm_or_si128(pack, mask_or); 440 _mm_storeu_si128((__m128i*)dst, res); 441 } 442 break; 443 } 444 default: { 445 assert(xbits == 3); 446 for (x = 0; x + 16 <= width; x += 16, dst += 2) { 447 // 0000000a00000000b... | (where a/b are 1 bit). 448 const __m128i in = _mm_loadu_si128((const __m128i*)&row[x]); 449 const __m128i shift = _mm_slli_epi64(in, 7); 450 const uint32_t move = _mm_movemask_epi8(shift); 451 dst[0] = 0xff000000 | ((move & 0xff) << 8); 452 dst[1] = 0xff000000 | (move & 0xff00); 453 } 454 break; 455 } 456 } 457 if (x != width) { 458 VP8LBundleColorMap_C(row + x, width - x, xbits, dst); 459 } 460 } 461 462 //------------------------------------------------------------------------------ 463 // Batch version of Predictor Transform subtraction 464 465 static WEBP_INLINE void Average2_m128i(const __m128i* const a0, 466 const __m128i* const a1, 467 __m128i* const avg) { 468 // (a + b) >> 1 = ((a + b + 1) >> 1) - ((a ^ b) & 1) 469 const __m128i ones = _mm_set1_epi8(1); 470 const __m128i avg1 = _mm_avg_epu8(*a0, *a1); 471 const __m128i one = _mm_and_si128(_mm_xor_si128(*a0, *a1), ones); 472 *avg = _mm_sub_epi8(avg1, one); 473 } 474 475 // Predictor0: ARGB_BLACK. 476 static void PredictorSub0_SSE2(const uint32_t* in, const uint32_t* upper, 477 int num_pixels, uint32_t* out) { 478 int i; 479 const __m128i black = _mm_set1_epi32(ARGB_BLACK); 480 for (i = 0; i + 4 <= num_pixels; i += 4) { 481 const __m128i src = _mm_loadu_si128((const __m128i*)&in[i]); 482 const __m128i res = _mm_sub_epi8(src, black); 483 _mm_storeu_si128((__m128i*)&out[i], res); 484 } 485 if (i != num_pixels) { 486 VP8LPredictorsSub_C[0](in + i, upper + i, num_pixels - i, out + i); 487 } 488 } 489 490 #define GENERATE_PREDICTOR_1(X, IN) \ 491 static void PredictorSub##X##_SSE2(const uint32_t* in, const uint32_t* upper, \ 492 int num_pixels, uint32_t* out) { \ 493 int i; \ 494 for (i = 0; i + 4 <= num_pixels; i += 4) { \ 495 const __m128i src = _mm_loadu_si128((const __m128i*)&in[i]); \ 496 const __m128i pred = _mm_loadu_si128((const __m128i*)&(IN)); \ 497 const __m128i res = _mm_sub_epi8(src, pred); \ 498 _mm_storeu_si128((__m128i*)&out[i], res); \ 499 } \ 500 if (i != num_pixels) { \ 501 VP8LPredictorsSub_C[(X)](in + i, upper + i, num_pixels - i, out + i); \ 502 } \ 503 } 504 505 GENERATE_PREDICTOR_1(1, in[i - 1]) // Predictor1: L 506 GENERATE_PREDICTOR_1(2, upper[i]) // Predictor2: T 507 GENERATE_PREDICTOR_1(3, upper[i + 1]) // Predictor3: TR 508 GENERATE_PREDICTOR_1(4, upper[i - 1]) // Predictor4: TL 509 #undef GENERATE_PREDICTOR_1 510 511 // Predictor5: avg2(avg2(L, TR), T) 512 static void PredictorSub5_SSE2(const uint32_t* in, const uint32_t* upper, 513 int num_pixels, uint32_t* out) { 514 int i; 515 for (i = 0; i + 4 <= num_pixels; i += 4) { 516 const __m128i L = _mm_loadu_si128((const __m128i*)&in[i - 1]); 517 const __m128i T = _mm_loadu_si128((const __m128i*)&upper[i]); 518 const __m128i TR = _mm_loadu_si128((const __m128i*)&upper[i + 1]); 519 const __m128i src = _mm_loadu_si128((const __m128i*)&in[i]); 520 __m128i avg, pred, res; 521 Average2_m128i(&L, &TR, &avg); 522 Average2_m128i(&avg, &T, &pred); 523 res = _mm_sub_epi8(src, pred); 524 _mm_storeu_si128((__m128i*)&out[i], res); 525 } 526 if (i != num_pixels) { 527 VP8LPredictorsSub_C[5](in + i, upper + i, num_pixels - i, out + i); 528 } 529 } 530 531 #define GENERATE_PREDICTOR_2(X, A, B) \ 532 static void PredictorSub##X##_SSE2(const uint32_t* in, const uint32_t* upper, \ 533 int num_pixels, uint32_t* out) { \ 534 int i; \ 535 for (i = 0; i + 4 <= num_pixels; i += 4) { \ 536 const __m128i tA = _mm_loadu_si128((const __m128i*)&(A)); \ 537 const __m128i tB = _mm_loadu_si128((const __m128i*)&(B)); \ 538 const __m128i src = _mm_loadu_si128((const __m128i*)&in[i]); \ 539 __m128i pred, res; \ 540 Average2_m128i(&tA, &tB, &pred); \ 541 res = _mm_sub_epi8(src, pred); \ 542 _mm_storeu_si128((__m128i*)&out[i], res); \ 543 } \ 544 if (i != num_pixels) { \ 545 VP8LPredictorsSub_C[(X)](in + i, upper + i, num_pixels - i, out + i); \ 546 } \ 547 } 548 549 GENERATE_PREDICTOR_2(6, in[i - 1], upper[i - 1]) // Predictor6: avg(L, TL) 550 GENERATE_PREDICTOR_2(7, in[i - 1], upper[i]) // Predictor7: avg(L, T) 551 GENERATE_PREDICTOR_2(8, upper[i - 1], upper[i]) // Predictor8: avg(TL, T) 552 GENERATE_PREDICTOR_2(9, upper[i], upper[i + 1]) // Predictor9: average(T, TR) 553 #undef GENERATE_PREDICTOR_2 554 555 // Predictor10: avg(avg(L,TL), avg(T, TR)). 556 static void PredictorSub10_SSE2(const uint32_t* in, const uint32_t* upper, 557 int num_pixels, uint32_t* out) { 558 int i; 559 for (i = 0; i + 4 <= num_pixels; i += 4) { 560 const __m128i L = _mm_loadu_si128((const __m128i*)&in[i - 1]); 561 const __m128i src = _mm_loadu_si128((const __m128i*)&in[i]); 562 const __m128i TL = _mm_loadu_si128((const __m128i*)&upper[i - 1]); 563 const __m128i T = _mm_loadu_si128((const __m128i*)&upper[i]); 564 const __m128i TR = _mm_loadu_si128((const __m128i*)&upper[i + 1]); 565 __m128i avgTTR, avgLTL, avg, res; 566 Average2_m128i(&T, &TR, &avgTTR); 567 Average2_m128i(&L, &TL, &avgLTL); 568 Average2_m128i(&avgTTR, &avgLTL, &avg); 569 res = _mm_sub_epi8(src, avg); 570 _mm_storeu_si128((__m128i*)&out[i], res); 571 } 572 if (i != num_pixels) { 573 VP8LPredictorsSub_C[10](in + i, upper + i, num_pixels - i, out + i); 574 } 575 } 576 577 // Predictor11: select. 578 static void GetSumAbsDiff32_SSE2(const __m128i* const A, const __m128i* const B, 579 __m128i* const out) { 580 // We can unpack with any value on the upper 32 bits, provided it's the same 581 // on both operands (to that their sum of abs diff is zero). Here we use *A. 582 const __m128i A_lo = _mm_unpacklo_epi32(*A, *A); 583 const __m128i B_lo = _mm_unpacklo_epi32(*B, *A); 584 const __m128i A_hi = _mm_unpackhi_epi32(*A, *A); 585 const __m128i B_hi = _mm_unpackhi_epi32(*B, *A); 586 const __m128i s_lo = _mm_sad_epu8(A_lo, B_lo); 587 const __m128i s_hi = _mm_sad_epu8(A_hi, B_hi); 588 *out = _mm_packs_epi32(s_lo, s_hi); 589 } 590 591 static void PredictorSub11_SSE2(const uint32_t* in, const uint32_t* upper, 592 int num_pixels, uint32_t* out) { 593 int i; 594 for (i = 0; i + 4 <= num_pixels; i += 4) { 595 const __m128i L = _mm_loadu_si128((const __m128i*)&in[i - 1]); 596 const __m128i T = _mm_loadu_si128((const __m128i*)&upper[i]); 597 const __m128i TL = _mm_loadu_si128((const __m128i*)&upper[i - 1]); 598 const __m128i src = _mm_loadu_si128((const __m128i*)&in[i]); 599 __m128i pa, pb; 600 GetSumAbsDiff32_SSE2(&T, &TL, &pa); // pa = sum |T-TL| 601 GetSumAbsDiff32_SSE2(&L, &TL, &pb); // pb = sum |L-TL| 602 { 603 const __m128i mask = _mm_cmpgt_epi32(pb, pa); 604 const __m128i A = _mm_and_si128(mask, L); 605 const __m128i B = _mm_andnot_si128(mask, T); 606 const __m128i pred = _mm_or_si128(A, B); // pred = (L > T)? L : T 607 const __m128i res = _mm_sub_epi8(src, pred); 608 _mm_storeu_si128((__m128i*)&out[i], res); 609 } 610 } 611 if (i != num_pixels) { 612 VP8LPredictorsSub_C[11](in + i, upper + i, num_pixels - i, out + i); 613 } 614 } 615 616 // Predictor12: ClampedSubSubtractFull. 617 static void PredictorSub12_SSE2(const uint32_t* in, const uint32_t* upper, 618 int num_pixels, uint32_t* out) { 619 int i; 620 const __m128i zero = _mm_setzero_si128(); 621 for (i = 0; i + 4 <= num_pixels; i += 4) { 622 const __m128i src = _mm_loadu_si128((const __m128i*)&in[i]); 623 const __m128i L = _mm_loadu_si128((const __m128i*)&in[i - 1]); 624 const __m128i L_lo = _mm_unpacklo_epi8(L, zero); 625 const __m128i L_hi = _mm_unpackhi_epi8(L, zero); 626 const __m128i T = _mm_loadu_si128((const __m128i*)&upper[i]); 627 const __m128i T_lo = _mm_unpacklo_epi8(T, zero); 628 const __m128i T_hi = _mm_unpackhi_epi8(T, zero); 629 const __m128i TL = _mm_loadu_si128((const __m128i*)&upper[i - 1]); 630 const __m128i TL_lo = _mm_unpacklo_epi8(TL, zero); 631 const __m128i TL_hi = _mm_unpackhi_epi8(TL, zero); 632 const __m128i diff_lo = _mm_sub_epi16(T_lo, TL_lo); 633 const __m128i diff_hi = _mm_sub_epi16(T_hi, TL_hi); 634 const __m128i pred_lo = _mm_add_epi16(L_lo, diff_lo); 635 const __m128i pred_hi = _mm_add_epi16(L_hi, diff_hi); 636 const __m128i pred = _mm_packus_epi16(pred_lo, pred_hi); 637 const __m128i res = _mm_sub_epi8(src, pred); 638 _mm_storeu_si128((__m128i*)&out[i], res); 639 } 640 if (i != num_pixels) { 641 VP8LPredictorsSub_C[12](in + i, upper + i, num_pixels - i, out + i); 642 } 643 } 644 645 // Predictors13: ClampedAddSubtractHalf 646 static void PredictorSub13_SSE2(const uint32_t* in, const uint32_t* upper, 647 int num_pixels, uint32_t* out) { 648 int i; 649 const __m128i zero = _mm_setzero_si128(); 650 for (i = 0; i + 2 <= num_pixels; i += 2) { 651 // we can only process two pixels at a time 652 const __m128i L = _mm_loadl_epi64((const __m128i*)&in[i - 1]); 653 const __m128i src = _mm_loadl_epi64((const __m128i*)&in[i]); 654 const __m128i T = _mm_loadl_epi64((const __m128i*)&upper[i]); 655 const __m128i TL = _mm_loadl_epi64((const __m128i*)&upper[i - 1]); 656 const __m128i L_lo = _mm_unpacklo_epi8(L, zero); 657 const __m128i T_lo = _mm_unpacklo_epi8(T, zero); 658 const __m128i TL_lo = _mm_unpacklo_epi8(TL, zero); 659 const __m128i sum = _mm_add_epi16(T_lo, L_lo); 660 const __m128i avg = _mm_srli_epi16(sum, 1); 661 const __m128i A1 = _mm_sub_epi16(avg, TL_lo); 662 const __m128i bit_fix = _mm_cmpgt_epi16(TL_lo, avg); 663 const __m128i A2 = _mm_sub_epi16(A1, bit_fix); 664 const __m128i A3 = _mm_srai_epi16(A2, 1); 665 const __m128i A4 = _mm_add_epi16(avg, A3); 666 const __m128i pred = _mm_packus_epi16(A4, A4); 667 const __m128i res = _mm_sub_epi8(src, pred); 668 _mm_storel_epi64((__m128i*)&out[i], res); 669 } 670 if (i != num_pixels) { 671 VP8LPredictorsSub_C[13](in + i, upper + i, num_pixels - i, out + i); 672 } 673 } 674 675 //------------------------------------------------------------------------------ 676 // Entry point 677 678 extern void VP8LEncDspInitSSE2(void); 679 680 WEBP_TSAN_IGNORE_FUNCTION void VP8LEncDspInitSSE2(void) { 681 VP8LSubtractGreenFromBlueAndRed = SubtractGreenFromBlueAndRed_SSE2; 682 VP8LTransformColor = TransformColor_SSE2; 683 VP8LCollectColorBlueTransforms = CollectColorBlueTransforms_SSE2; 684 VP8LCollectColorRedTransforms = CollectColorRedTransforms_SSE2; 685 VP8LHistogramAdd = HistogramAdd_SSE2; 686 VP8LCombinedShannonEntropy = CombinedShannonEntropy_SSE2; 687 VP8LVectorMismatch = VectorMismatch_SSE2; 688 VP8LBundleColorMap = BundleColorMap_SSE2; 689 690 VP8LPredictorsSub[0] = PredictorSub0_SSE2; 691 VP8LPredictorsSub[1] = PredictorSub1_SSE2; 692 VP8LPredictorsSub[2] = PredictorSub2_SSE2; 693 VP8LPredictorsSub[3] = PredictorSub3_SSE2; 694 VP8LPredictorsSub[4] = PredictorSub4_SSE2; 695 VP8LPredictorsSub[5] = PredictorSub5_SSE2; 696 VP8LPredictorsSub[6] = PredictorSub6_SSE2; 697 VP8LPredictorsSub[7] = PredictorSub7_SSE2; 698 VP8LPredictorsSub[8] = PredictorSub8_SSE2; 699 VP8LPredictorsSub[9] = PredictorSub9_SSE2; 700 VP8LPredictorsSub[10] = PredictorSub10_SSE2; 701 VP8LPredictorsSub[11] = PredictorSub11_SSE2; 702 VP8LPredictorsSub[12] = PredictorSub12_SSE2; 703 VP8LPredictorsSub[13] = PredictorSub13_SSE2; 704 VP8LPredictorsSub[14] = PredictorSub0_SSE2; // <- padding security sentinels 705 VP8LPredictorsSub[15] = PredictorSub0_SSE2; 706 } 707 708 #else // !WEBP_USE_SSE2 709 710 WEBP_DSP_INIT_STUB(VP8LEncDspInitSSE2) 711 712 #endif // WEBP_USE_SSE2 713
