1 /* 2 * Copyright 2012 The Android Open Source Project 3 * 4 * Use of this source code is governed by a BSD-style license that can be 5 * found in the LICENSE file. 6 */ 7 8 #include "SkBitmapProcState_opts_SSSE3.h" 9 #include "SkColorPriv.h" 10 #include "SkPaint.h" 11 #include "SkUtils.h" 12 13 /* With the exception of the compilers that don't support it, we always build the 14 * SSSE3 functions and enable the caller to determine SSSE3 support. However for 15 * compilers that do not support SSSE3 we provide a stub implementation. 16 */ 17 #if SK_CPU_SSE_LEVEL >= SK_CPU_SSE_LEVEL_SSSE3 18 19 #include <tmmintrin.h> // SSSE3 20 21 // adding anonymous namespace seemed to force gcc to inline directly the 22 // instantiation, instead of creating the functions 23 // S32_generic_D32_filter_DX_SSSE3<true> and 24 // S32_generic_D32_filter_DX_SSSE3<false> which were then called by the 25 // external functions. 26 namespace { 27 // In this file, variations for alpha and non alpha versions are implemented 28 // with a template, as it makes the code more compact and a bit easier to 29 // maintain, while making the compiler generate the same exact code as with 30 // two functions that only differ by a few lines. 31 32 33 // Prepare all necessary constants for a round of processing for two pixel 34 // pairs. 35 // @param xy is the location where the xy parameters for four pixels should be 36 // read from. It is identical in concept with argument two of 37 // S32_{opaque}_D32_filter_DX methods. 38 // @param mask_3FFF vector of 32 bit constants containing 3FFF, 39 // suitable to mask the bottom 14 bits of a XY value. 40 // @param mask_000F vector of 32 bit constants containing 000F, 41 // suitable to mask the bottom 4 bits of a XY value. 42 // @param sixteen_8bit vector of 8 bit components containing the value 16. 43 // @param mask_dist_select vector of 8 bit components containing the shuffling 44 // parameters to reorder x[0-3] parameters. 45 // @param all_x_result vector of 8 bit components that will contain the 46 // (4x(x3), 4x(x2), 4x(x1), 4x(x0)) upon return. 47 // @param sixteen_minus_x vector of 8 bit components, containing 48 // (4x(16 - x3), 4x(16 - x2), 4x(16 - x1), 4x(16 - x0)) 49 inline void PrepareConstantsTwoPixelPairs(const uint32_t* xy, 50 const __m128i& mask_3FFF, 51 const __m128i& mask_000F, 52 const __m128i& sixteen_8bit, 53 const __m128i& mask_dist_select, 54 __m128i* all_x_result, 55 __m128i* sixteen_minus_x, 56 int* x0, 57 int* x1) { 58 const __m128i xx = _mm_loadu_si128(reinterpret_cast<const __m128i *>(xy)); 59 60 // 4 delta X 61 // (x03, x02, x01, x00) 62 const __m128i x0_wide = _mm_srli_epi32(xx, 18); 63 // (x13, x12, x11, x10) 64 const __m128i x1_wide = _mm_and_si128(xx, mask_3FFF); 65 66 _mm_storeu_si128(reinterpret_cast<__m128i *>(x0), x0_wide); 67 _mm_storeu_si128(reinterpret_cast<__m128i *>(x1), x1_wide); 68 69 __m128i all_x = _mm_and_si128(_mm_srli_epi32(xx, 14), mask_000F); 70 71 // (4x(x3), 4x(x2), 4x(x1), 4x(x0)) 72 all_x = _mm_shuffle_epi8(all_x, mask_dist_select); 73 74 *all_x_result = all_x; 75 // (4x(16-x3), 4x(16-x2), 4x(16-x1), 4x(16-x0)) 76 *sixteen_minus_x = _mm_sub_epi8(sixteen_8bit, all_x); 77 } 78 79 // Prepare all necessary constants for a round of processing for two pixel 80 // pairs. 81 // @param xy is the location where the xy parameters for four pixels should be 82 // read from. It is identical in concept with argument two of 83 // S32_{opaque}_D32_filter_DXDY methods. 84 // @param mask_3FFF vector of 32 bit constants containing 3FFF, 85 // suitable to mask the bottom 14 bits of a XY value. 86 // @param mask_000F vector of 32 bit constants containing 000F, 87 // suitable to mask the bottom 4 bits of a XY value. 88 // @param sixteen_8bit vector of 8 bit components containing the value 16. 89 // @param mask_dist_select vector of 8 bit components containing the shuffling 90 // parameters to reorder x[0-3] parameters. 91 // @param all_xy_result vector of 8 bit components that will contain the 92 // (4x(y1), 4x(y0), 4x(x1), 4x(x0)) upon return. 93 // @param sixteen_minus_x vector of 8 bit components, containing 94 // (4x(16-y1), 4x(16-y0), 4x(16-x1), 4x(16-x0)). 95 inline void PrepareConstantsTwoPixelPairsDXDY(const uint32_t* xy, 96 const __m128i& mask_3FFF, 97 const __m128i& mask_000F, 98 const __m128i& sixteen_8bit, 99 const __m128i& mask_dist_select, 100 __m128i* all_xy_result, 101 __m128i* sixteen_minus_xy, 102 int* xy0, int* xy1) { 103 const __m128i xy_wide = 104 _mm_loadu_si128(reinterpret_cast<const __m128i *>(xy)); 105 106 // (x10, y10, x00, y00) 107 __m128i xy0_wide = _mm_srli_epi32(xy_wide, 18); 108 // (y10, y00, x10, x00) 109 xy0_wide = _mm_shuffle_epi32(xy0_wide, _MM_SHUFFLE(2, 0, 3, 1)); 110 // (x11, y11, x01, y01) 111 __m128i xy1_wide = _mm_and_si128(xy_wide, mask_3FFF); 112 // (y11, y01, x11, x01) 113 xy1_wide = _mm_shuffle_epi32(xy1_wide, _MM_SHUFFLE(2, 0, 3, 1)); 114 115 _mm_storeu_si128(reinterpret_cast<__m128i *>(xy0), xy0_wide); 116 _mm_storeu_si128(reinterpret_cast<__m128i *>(xy1), xy1_wide); 117 118 // (x1, y1, x0, y0) 119 __m128i all_xy = _mm_and_si128(_mm_srli_epi32(xy_wide, 14), mask_000F); 120 // (y1, y0, x1, x0) 121 all_xy = _mm_shuffle_epi32(all_xy, _MM_SHUFFLE(2, 0, 3, 1)); 122 // (4x(y1), 4x(y0), 4x(x1), 4x(x0)) 123 all_xy = _mm_shuffle_epi8(all_xy, mask_dist_select); 124 125 *all_xy_result = all_xy; 126 // (4x(16-y1), 4x(16-y0), 4x(16-x1), 4x(16-x0)) 127 *sixteen_minus_xy = _mm_sub_epi8(sixteen_8bit, all_xy); 128 } 129 130 // Helper function used when processing one pixel pair. 131 // @param pixel0..3 are the four input pixels 132 // @param scale_x vector of 8 bit components to multiply the pixel[0:3]. This 133 // will contain (4x(x1, 16-x1), 4x(x0, 16-x0)) 134 // or (4x(x3, 16-x3), 4x(x2, 16-x2)) 135 // @return a vector of 16 bit components containing: 136 // (Aa2 * (16 - x1) + Aa3 * x1, ... , Ra0 * (16 - x0) + Ra1 * x0) 137 inline __m128i ProcessPixelPairHelper(uint32_t pixel0, 138 uint32_t pixel1, 139 uint32_t pixel2, 140 uint32_t pixel3, 141 const __m128i& scale_x) { 142 __m128i a0, a1, a2, a3; 143 // Load 2 pairs of pixels 144 a0 = _mm_cvtsi32_si128(pixel0); 145 a1 = _mm_cvtsi32_si128(pixel1); 146 147 // Interleave pixels. 148 // (0, 0, 0, 0, 0, 0, 0, 0, Aa1, Aa0, Ba1, Ba0, Ga1, Ga0, Ra1, Ra0) 149 a0 = _mm_unpacklo_epi8(a0, a1); 150 151 a2 = _mm_cvtsi32_si128(pixel2); 152 a3 = _mm_cvtsi32_si128(pixel3); 153 // (0, 0, 0, 0, 0, 0, 0, 0, Aa3, Aa2, Ba3, Ba2, Ga3, Ga2, Ra3, Ra2) 154 a2 = _mm_unpacklo_epi8(a2, a3); 155 156 // two pairs of pixel pairs, interleaved. 157 // (Aa3, Aa2, Ba3, Ba2, Ga3, Ga2, Ra3, Ra2, 158 // Aa1, Aa0, Ba1, Ba0, Ga1, Ga0, Ra1, Ra0) 159 a0 = _mm_unpacklo_epi64(a0, a2); 160 161 // multiply and sum to 16 bit components. 162 // (Aa2 * (16 - x1) + Aa3 * x1, ... , Ra0 * (16 - x0) + Ra1 * x0) 163 // At that point, we use up a bit less than 12 bits for each 16 bit 164 // component: 165 // All components are less than 255. So, 166 // C0 * (16 - x) + C1 * x <= 255 * (16 - x) + 255 * x = 255 * 16. 167 return _mm_maddubs_epi16(a0, scale_x); 168 } 169 170 // Scale back the results after multiplications to the [0:255] range, and scale 171 // by alpha when has_alpha is true. 172 // Depending on whether one set or two sets of multiplications had been applied, 173 // the results have to be shifted by four places (dividing by 16), or shifted 174 // by eight places (dividing by 256), since each multiplication is by a quantity 175 // in the range [0:16]. 176 template<bool has_alpha, int scale> 177 inline __m128i ScaleFourPixels(__m128i* pixels, 178 const __m128i& alpha) { 179 // Divide each 16 bit component by 16 (or 256 depending on scale). 180 *pixels = _mm_srli_epi16(*pixels, scale); 181 182 if (has_alpha) { 183 // Multiply by alpha. 184 *pixels = _mm_mullo_epi16(*pixels, alpha); 185 186 // Divide each 16 bit component by 256. 187 *pixels = _mm_srli_epi16(*pixels, 8); 188 } 189 return *pixels; 190 } 191 192 // Wrapper to calculate two output pixels from four input pixels. The 193 // arguments are the same as ProcessPixelPairHelper. Technically, there are 194 // eight input pixels, but since sub_y == 0, the factors applied to half of the 195 // pixels is zero (sub_y), and are therefore omitted here to save on some 196 // processing. 197 // @param alpha when has_alpha is true, scale all resulting components by this 198 // value. 199 // @return a vector of 16 bit components containing: 200 // ((Aa2 * (16 - x1) + Aa3 * x1) * alpha, ..., 201 // (Ra0 * (16 - x0) + Ra1 * x0) * alpha) (when has_alpha is true) 202 // otherwise 203 // (Aa2 * (16 - x1) + Aa3 * x1, ... , Ra0 * (16 - x0) + Ra1 * x0) 204 // In both cases, the results are renormalized (divided by 16) to match the 205 // expected formats when storing back the results into memory. 206 template<bool has_alpha> 207 inline __m128i ProcessPixelPairZeroSubY(uint32_t pixel0, 208 uint32_t pixel1, 209 uint32_t pixel2, 210 uint32_t pixel3, 211 const __m128i& scale_x, 212 const __m128i& alpha) { 213 __m128i sum = ProcessPixelPairHelper(pixel0, pixel1, pixel2, pixel3, 214 scale_x); 215 return ScaleFourPixels<has_alpha, 4>(&sum, alpha); 216 } 217 218 // Same as ProcessPixelPairZeroSubY, expect processing one output pixel at a 219 // time instead of two. As in the above function, only two pixels are needed 220 // to generate a single pixel since sub_y == 0. 221 // @return same as ProcessPixelPairZeroSubY, except that only the bottom 4 222 // 16 bit components are set. 223 template<bool has_alpha> 224 inline __m128i ProcessOnePixelZeroSubY(uint32_t pixel0, 225 uint32_t pixel1, 226 __m128i scale_x, 227 __m128i alpha) { 228 __m128i a0 = _mm_cvtsi32_si128(pixel0); 229 __m128i a1 = _mm_cvtsi32_si128(pixel1); 230 231 // Interleave 232 a0 = _mm_unpacklo_epi8(a0, a1); 233 234 // (a0 * (16-x) + a1 * x) 235 __m128i sum = _mm_maddubs_epi16(a0, scale_x); 236 237 return ScaleFourPixels<has_alpha, 4>(&sum, alpha); 238 } 239 240 // Methods when sub_y != 0 241 242 243 // Same as ProcessPixelPairHelper, except that the values are scaled by y. 244 // @param y vector of 16 bit components containing 'y' values. There are two 245 // cases in practice, where y will contain the sub_y constant, or will 246 // contain the 16 - sub_y constant. 247 // @return vector of 16 bit components containing: 248 // (y * (Aa2 * (16 - x1) + Aa3 * x1), ... , y * (Ra0 * (16 - x0) + Ra1 * x0)) 249 inline __m128i ProcessPixelPair(uint32_t pixel0, 250 uint32_t pixel1, 251 uint32_t pixel2, 252 uint32_t pixel3, 253 const __m128i& scale_x, 254 const __m128i& y) { 255 __m128i sum = ProcessPixelPairHelper(pixel0, pixel1, pixel2, pixel3, 256 scale_x); 257 258 // first row times 16-y or y depending on whether 'y' represents one or 259 // the other. 260 // Values will be up to 255 * 16 * 16 = 65280. 261 // (y * (Aa2 * (16 - x1) + Aa3 * x1), ... , 262 // y * (Ra0 * (16 - x0) + Ra1 * x0)) 263 sum = _mm_mullo_epi16(sum, y); 264 265 return sum; 266 } 267 268 // Process two pixel pairs out of eight input pixels. 269 // In other methods, the distinct pixels are passed one by one, but in this 270 // case, the rows, and index offsets to the pixels into the row are passed 271 // to generate the 8 pixels. 272 // @param row0..1 top and bottom row where to find input pixels. 273 // @param x0..1 offsets into the row for all eight input pixels. 274 // @param all_y vector of 16 bit components containing the constant sub_y 275 // @param neg_y vector of 16 bit components containing the constant 16 - sub_y 276 // @param alpha vector of 16 bit components containing the alpha value to scale 277 // the results by, when has_alpha is true. 278 // @return 279 // (alpha * ((16-y) * (Aa2 * (16-x1) + Aa3 * x1) + 280 // y * (Aa2' * (16-x1) + Aa3' * x1)), 281 // ... 282 // alpha * ((16-y) * (Ra0 * (16-x0) + Ra1 * x0) + 283 // y * (Ra0' * (16-x0) + Ra1' * x0)) 284 // With the factor alpha removed when has_alpha is false. 285 // The values are scaled back to 16 bit components, but with only the bottom 286 // 8 bits being set. 287 template<bool has_alpha> 288 inline __m128i ProcessTwoPixelPairs(const uint32_t* row0, 289 const uint32_t* row1, 290 const int* x0, 291 const int* x1, 292 const __m128i& scale_x, 293 const __m128i& all_y, 294 const __m128i& neg_y, 295 const __m128i& alpha) { 296 __m128i sum0 = ProcessPixelPair( 297 row0[x0[0]], row0[x1[0]], row0[x0[1]], row0[x1[1]], 298 scale_x, neg_y); 299 __m128i sum1 = ProcessPixelPair( 300 row1[x0[0]], row1[x1[0]], row1[x0[1]], row1[x1[1]], 301 scale_x, all_y); 302 303 // 2 samples fully summed. 304 // ((16-y) * (Aa2 * (16-x1) + Aa3 * x1) + 305 // y * (Aa2' * (16-x1) + Aa3' * x1), 306 // ... 307 // (16-y) * (Ra0 * (16 - x0) + Ra1 * x0)) + 308 // y * (Ra0' * (16-x0) + Ra1' * x0)) 309 // Each component, again can be at most 256 * 255 = 65280, so no overflow. 310 sum0 = _mm_add_epi16(sum0, sum1); 311 312 return ScaleFourPixels<has_alpha, 8>(&sum0, alpha); 313 } 314 315 // Similar to ProcessTwoPixelPairs except the pixel indexes. 316 template<bool has_alpha> 317 inline __m128i ProcessTwoPixelPairsDXDY(const uint32_t* row00, 318 const uint32_t* row01, 319 const uint32_t* row10, 320 const uint32_t* row11, 321 const int* xy0, 322 const int* xy1, 323 const __m128i& scale_x, 324 const __m128i& all_y, 325 const __m128i& neg_y, 326 const __m128i& alpha) { 327 // first row 328 __m128i sum0 = ProcessPixelPair( 329 row00[xy0[0]], row00[xy1[0]], row10[xy0[1]], row10[xy1[1]], 330 scale_x, neg_y); 331 // second row 332 __m128i sum1 = ProcessPixelPair( 333 row01[xy0[0]], row01[xy1[0]], row11[xy0[1]], row11[xy1[1]], 334 scale_x, all_y); 335 336 // 2 samples fully summed. 337 // ((16-y1) * (Aa2 * (16-x1) + Aa3 * x1) + 338 // y0 * (Aa2' * (16-x1) + Aa3' * x1), 339 // ... 340 // (16-y0) * (Ra0 * (16 - x0) + Ra1 * x0)) + 341 // y0 * (Ra0' * (16-x0) + Ra1' * x0)) 342 // Each component, again can be at most 256 * 255 = 65280, so no overflow. 343 sum0 = _mm_add_epi16(sum0, sum1); 344 345 return ScaleFourPixels<has_alpha, 8>(&sum0, alpha); 346 } 347 348 349 // Same as ProcessPixelPair, except that performing the math one output pixel 350 // at a time. This means that only the bottom four 16 bit components are set. 351 inline __m128i ProcessOnePixel(uint32_t pixel0, uint32_t pixel1, 352 const __m128i& scale_x, const __m128i& y) { 353 __m128i a0 = _mm_cvtsi32_si128(pixel0); 354 __m128i a1 = _mm_cvtsi32_si128(pixel1); 355 356 // Interleave 357 // (0, 0, 0, 0, 0, 0, 0, 0, Aa1, Aa0, Ba1, Ba0, Ga1, Ga0, Ra1, Ra0) 358 a0 = _mm_unpacklo_epi8(a0, a1); 359 360 // (a0 * (16-x) + a1 * x) 361 a0 = _mm_maddubs_epi16(a0, scale_x); 362 363 // scale row by y 364 return _mm_mullo_epi16(a0, y); 365 } 366 367 // Notes about the various tricks that are used in this implementation: 368 // - specialization for sub_y == 0. 369 // Statistically, 1/16th of the samples will have sub_y == 0. When this 370 // happens, the math goes from: 371 // (16 - x)*(16 - y)*a00 + x*(16 - y)*a01 + (16 - x)*y*a10 + x*y*a11 372 // to: 373 // (16 - x)*a00 + 16*x*a01 374 // much simpler. The simplification makes for an easy boost in performance. 375 // - calculating 4 output pixels at a time. 376 // This allows loading the coefficients x0 and x1 and shuffling them to the 377 // optimum location only once per loop, instead of twice per loop. 378 // This also allows us to store the four pixels with a single store. 379 // - Use of 2 special SSSE3 instructions (comparatively to the SSE2 instruction 380 // version): 381 // _mm_shuffle_epi8 : this allows us to spread the coefficients x[0-3] loaded 382 // in 32 bit values to 8 bit values repeated four times. 383 // _mm_maddubs_epi16 : this allows us to perform multiplications and additions 384 // in one swoop of 8bit values storing the results in 16 bit values. This 385 // instruction is actually crucial for the speed of the implementation since 386 // as one can see in the SSE2 implementation, all inputs have to be used as 387 // 16 bits because the results are 16 bits. This basically allows us to process 388 // twice as many pixel components per iteration. 389 // 390 // As a result, this method behaves faster than the traditional SSE2. The actual 391 // boost varies greatly on the underlying architecture. 392 template<bool has_alpha> 393 void S32_generic_D32_filter_DX_SSSE3(const SkBitmapProcState& s, 394 const uint32_t* xy, 395 int count, uint32_t* colors) { 396 SkASSERT(count > 0 && colors != nullptr); 397 SkASSERT(s.fFilterLevel != kNone_SkFilterQuality); 398 SkASSERT(kN32_SkColorType == s.fPixmap.colorType()); 399 if (has_alpha) { 400 SkASSERT(s.fAlphaScale < 256); 401 } else { 402 SkASSERT(s.fAlphaScale == 256); 403 } 404 405 const uint8_t* src_addr = 406 static_cast<const uint8_t*>(s.fPixmap.addr()); 407 const size_t rb = s.fPixmap.rowBytes(); 408 const uint32_t XY = *xy++; 409 const unsigned y0 = XY >> 14; 410 const uint32_t* row0 = 411 reinterpret_cast<const uint32_t*>(src_addr + (y0 >> 4) * rb); 412 const uint32_t* row1 = 413 reinterpret_cast<const uint32_t*>(src_addr + (XY & 0x3FFF) * rb); 414 const unsigned sub_y = y0 & 0xF; 415 416 // vector constants 417 const __m128i mask_dist_select = _mm_set_epi8(12, 12, 12, 12, 418 8, 8, 8, 8, 419 4, 4, 4, 4, 420 0, 0, 0, 0); 421 const __m128i mask_3FFF = _mm_set1_epi32(0x3FFF); 422 const __m128i mask_000F = _mm_set1_epi32(0x000F); 423 const __m128i sixteen_8bit = _mm_set1_epi8(16); 424 // (0, 0, 0, 0, 0, 0, 0, 0) 425 const __m128i zero = _mm_setzero_si128(); 426 427 __m128i alpha = _mm_setzero_si128(); 428 if (has_alpha) { 429 // 8x(alpha) 430 alpha = _mm_set1_epi16(s.fAlphaScale); 431 } 432 433 if (sub_y == 0) { 434 // Unroll 4x, interleave bytes, use pmaddubsw (all_x is small) 435 while (count > 3) { 436 count -= 4; 437 438 int x0[4]; 439 int x1[4]; 440 __m128i all_x, sixteen_minus_x; 441 PrepareConstantsTwoPixelPairs(xy, mask_3FFF, mask_000F, 442 sixteen_8bit, mask_dist_select, 443 &all_x, &sixteen_minus_x, x0, x1); 444 xy += 4; 445 446 // First pair of pixel pairs. 447 // (4x(x1, 16-x1), 4x(x0, 16-x0)) 448 __m128i scale_x; 449 scale_x = _mm_unpacklo_epi8(sixteen_minus_x, all_x); 450 451 __m128i sum0 = ProcessPixelPairZeroSubY<has_alpha>( 452 row0[x0[0]], row0[x1[0]], row0[x0[1]], row0[x1[1]], 453 scale_x, alpha); 454 455 // second pair of pixel pairs 456 // (4x (x3, 16-x3), 4x (16-x2, x2)) 457 scale_x = _mm_unpackhi_epi8(sixteen_minus_x, all_x); 458 459 __m128i sum1 = ProcessPixelPairZeroSubY<has_alpha>( 460 row0[x0[2]], row0[x1[2]], row0[x0[3]], row0[x1[3]], 461 scale_x, alpha); 462 463 // Pack lower 4 16 bit values of sum into lower 4 bytes. 464 sum0 = _mm_packus_epi16(sum0, sum1); 465 466 // Extract low int and store. 467 _mm_storeu_si128(reinterpret_cast<__m128i *>(colors), sum0); 468 469 colors += 4; 470 } 471 472 // handle remainder 473 while (count-- > 0) { 474 uint32_t xx = *xy++; // x0:14 | 4 | x1:14 475 unsigned x0 = xx >> 18; 476 unsigned x1 = xx & 0x3FFF; 477 478 // 16x(x) 479 const __m128i all_x = _mm_set1_epi8((xx >> 14) & 0x0F); 480 481 // (16x(16-x)) 482 __m128i scale_x = _mm_sub_epi8(sixteen_8bit, all_x); 483 484 scale_x = _mm_unpacklo_epi8(scale_x, all_x); 485 486 __m128i sum = ProcessOnePixelZeroSubY<has_alpha>( 487 row0[x0], row0[x1], 488 scale_x, alpha); 489 490 // Pack lower 4 16 bit values of sum into lower 4 bytes. 491 sum = _mm_packus_epi16(sum, zero); 492 493 // Extract low int and store. 494 *colors++ = _mm_cvtsi128_si32(sum); 495 } 496 } else { // more general case, y != 0 497 // 8x(16) 498 const __m128i sixteen_16bit = _mm_set1_epi16(16); 499 500 // 8x (y) 501 const __m128i all_y = _mm_set1_epi16(sub_y); 502 503 // 8x (16-y) 504 const __m128i neg_y = _mm_sub_epi16(sixteen_16bit, all_y); 505 506 // Unroll 4x, interleave bytes, use pmaddubsw (all_x is small) 507 while (count > 3) { 508 count -= 4; 509 510 int x0[4]; 511 int x1[4]; 512 __m128i all_x, sixteen_minus_x; 513 PrepareConstantsTwoPixelPairs(xy, mask_3FFF, mask_000F, 514 sixteen_8bit, mask_dist_select, 515 &all_x, &sixteen_minus_x, x0, x1); 516 xy += 4; 517 518 // First pair of pixel pairs 519 // (4x(x1, 16-x1), 4x(x0, 16-x0)) 520 __m128i scale_x; 521 scale_x = _mm_unpacklo_epi8(sixteen_minus_x, all_x); 522 523 __m128i sum0 = ProcessTwoPixelPairs<has_alpha>( 524 row0, row1, x0, x1, 525 scale_x, all_y, neg_y, alpha); 526 527 // second pair of pixel pairs 528 // (4x (x3, 16-x3), 4x (16-x2, x2)) 529 scale_x = _mm_unpackhi_epi8(sixteen_minus_x, all_x); 530 531 __m128i sum1 = ProcessTwoPixelPairs<has_alpha>( 532 row0, row1, x0 + 2, x1 + 2, 533 scale_x, all_y, neg_y, alpha); 534 535 // Do the final packing of the two results 536 537 // Pack lower 4 16 bit values of sum into lower 4 bytes. 538 sum0 = _mm_packus_epi16(sum0, sum1); 539 540 // Extract low int and store. 541 _mm_storeu_si128(reinterpret_cast<__m128i *>(colors), sum0); 542 543 colors += 4; 544 } 545 546 // Left over. 547 while (count-- > 0) { 548 const uint32_t xx = *xy++; // x0:14 | 4 | x1:14 549 const unsigned x0 = xx >> 18; 550 const unsigned x1 = xx & 0x3FFF; 551 552 // 16x(x) 553 const __m128i all_x = _mm_set1_epi8((xx >> 14) & 0x0F); 554 555 // 16x (16-x) 556 __m128i scale_x = _mm_sub_epi8(sixteen_8bit, all_x); 557 558 // (8x (x, 16-x)) 559 scale_x = _mm_unpacklo_epi8(scale_x, all_x); 560 561 // first row. 562 __m128i sum0 = ProcessOnePixel(row0[x0], row0[x1], scale_x, neg_y); 563 // second row. 564 __m128i sum1 = ProcessOnePixel(row1[x0], row1[x1], scale_x, all_y); 565 566 // Add both rows for full sample 567 sum0 = _mm_add_epi16(sum0, sum1); 568 569 sum0 = ScaleFourPixels<has_alpha, 8>(&sum0, alpha); 570 571 // Pack lower 4 16 bit values of sum into lower 4 bytes. 572 sum0 = _mm_packus_epi16(sum0, zero); 573 574 // Extract low int and store. 575 *colors++ = _mm_cvtsi128_si32(sum0); 576 } 577 } 578 } 579 580 /* 581 * Similar to S32_generic_D32_filter_DX_SSSE3, we do not need to handle the 582 * special case suby == 0 as suby is changing in every loop. 583 */ 584 template<bool has_alpha> 585 void S32_generic_D32_filter_DXDY_SSSE3(const SkBitmapProcState& s, 586 const uint32_t* xy, 587 int count, uint32_t* colors) { 588 SkASSERT(count > 0 && colors != nullptr); 589 SkASSERT(s.fFilterLevel != kNone_SkFilterQuality); 590 SkASSERT(kN32_SkColorType == s.fPixmap.colorType()); 591 if (has_alpha) { 592 SkASSERT(s.fAlphaScale < 256); 593 } else { 594 SkASSERT(s.fAlphaScale == 256); 595 } 596 597 const uint8_t* src_addr = 598 static_cast<const uint8_t*>(s.fPixmap.addr()); 599 const size_t rb = s.fPixmap.rowBytes(); 600 601 // vector constants 602 const __m128i mask_dist_select = _mm_set_epi8(12, 12, 12, 12, 603 8, 8, 8, 8, 604 4, 4, 4, 4, 605 0, 0, 0, 0); 606 const __m128i mask_3FFF = _mm_set1_epi32(0x3FFF); 607 const __m128i mask_000F = _mm_set1_epi32(0x000F); 608 const __m128i sixteen_8bit = _mm_set1_epi8(16); 609 610 __m128i alpha; 611 if (has_alpha) { 612 // 8x(alpha) 613 alpha = _mm_set1_epi16(s.fAlphaScale); 614 } 615 616 // Unroll 2x, interleave bytes, use pmaddubsw (all_x is small) 617 while (count >= 2) { 618 int xy0[4]; 619 int xy1[4]; 620 __m128i all_xy, sixteen_minus_xy; 621 PrepareConstantsTwoPixelPairsDXDY(xy, mask_3FFF, mask_000F, 622 sixteen_8bit, mask_dist_select, 623 &all_xy, &sixteen_minus_xy, xy0, xy1); 624 625 // (4x(x1, 16-x1), 4x(x0, 16-x0)) 626 __m128i scale_x = _mm_unpacklo_epi8(sixteen_minus_xy, all_xy); 627 // (4x(0, y1), 4x(0, y0)) 628 __m128i all_y = _mm_unpackhi_epi8(all_xy, _mm_setzero_si128()); 629 __m128i neg_y = _mm_sub_epi16(_mm_set1_epi16(16), all_y); 630 631 const uint32_t* row00 = 632 reinterpret_cast<const uint32_t*>(src_addr + xy0[2] * rb); 633 const uint32_t* row01 = 634 reinterpret_cast<const uint32_t*>(src_addr + xy1[2] * rb); 635 const uint32_t* row10 = 636 reinterpret_cast<const uint32_t*>(src_addr + xy0[3] * rb); 637 const uint32_t* row11 = 638 reinterpret_cast<const uint32_t*>(src_addr + xy1[3] * rb); 639 640 __m128i sum0 = ProcessTwoPixelPairsDXDY<has_alpha>( 641 row00, row01, row10, row11, xy0, xy1, 642 scale_x, all_y, neg_y, alpha); 643 644 // Pack lower 4 16 bit values of sum into lower 4 bytes. 645 sum0 = _mm_packus_epi16(sum0, _mm_setzero_si128()); 646 647 // Extract low int and store. 648 _mm_storel_epi64(reinterpret_cast<__m128i *>(colors), sum0); 649 650 xy += 4; 651 colors += 2; 652 count -= 2; 653 } 654 655 // Handle the remainder 656 while (count-- > 0) { 657 uint32_t data = *xy++; 658 unsigned y0 = data >> 14; 659 unsigned y1 = data & 0x3FFF; 660 unsigned subY = y0 & 0xF; 661 y0 >>= 4; 662 663 data = *xy++; 664 unsigned x0 = data >> 14; 665 unsigned x1 = data & 0x3FFF; 666 unsigned subX = x0 & 0xF; 667 x0 >>= 4; 668 669 const uint32_t* row0 = 670 reinterpret_cast<const uint32_t*>(src_addr + y0 * rb); 671 const uint32_t* row1 = 672 reinterpret_cast<const uint32_t*>(src_addr + y1 * rb); 673 674 // 16x(x) 675 const __m128i all_x = _mm_set1_epi8(subX); 676 677 // 16x (16-x) 678 __m128i scale_x = _mm_sub_epi8(sixteen_8bit, all_x); 679 680 // (8x (x, 16-x)) 681 scale_x = _mm_unpacklo_epi8(scale_x, all_x); 682 683 // 8x(16) 684 const __m128i sixteen_16bit = _mm_set1_epi16(16); 685 686 // 8x (y) 687 const __m128i all_y = _mm_set1_epi16(subY); 688 689 // 8x (16-y) 690 const __m128i neg_y = _mm_sub_epi16(sixteen_16bit, all_y); 691 692 // first row. 693 __m128i sum0 = ProcessOnePixel(row0[x0], row0[x1], scale_x, neg_y); 694 // second row. 695 __m128i sum1 = ProcessOnePixel(row1[x0], row1[x1], scale_x, all_y); 696 697 // Add both rows for full sample 698 sum0 = _mm_add_epi16(sum0, sum1); 699 700 sum0 = ScaleFourPixels<has_alpha, 8>(&sum0, alpha); 701 702 // Pack lower 4 16 bit values of sum into lower 4 bytes. 703 sum0 = _mm_packus_epi16(sum0, _mm_setzero_si128()); 704 705 // Extract low int and store. 706 *colors++ = _mm_cvtsi128_si32(sum0); 707 } 708 } 709 } // namespace 710 711 void S32_opaque_D32_filter_DX_SSSE3(const SkBitmapProcState& s, 712 const uint32_t* xy, 713 int count, uint32_t* colors) { 714 S32_generic_D32_filter_DX_SSSE3<false>(s, xy, count, colors); 715 } 716 717 void S32_alpha_D32_filter_DX_SSSE3(const SkBitmapProcState& s, 718 const uint32_t* xy, 719 int count, uint32_t* colors) { 720 S32_generic_D32_filter_DX_SSSE3<true>(s, xy, count, colors); 721 } 722 723 void S32_opaque_D32_filter_DXDY_SSSE3(const SkBitmapProcState& s, 724 const uint32_t* xy, 725 int count, uint32_t* colors) { 726 S32_generic_D32_filter_DXDY_SSSE3<false>(s, xy, count, colors); 727 } 728 729 void S32_alpha_D32_filter_DXDY_SSSE3(const SkBitmapProcState& s, 730 const uint32_t* xy, 731 int count, uint32_t* colors) { 732 S32_generic_D32_filter_DXDY_SSSE3<true>(s, xy, count, colors); 733 } 734 735 #else // SK_CPU_SSE_LEVEL >= SK_CPU_SSE_LEVEL_SSSE3 736 737 void S32_opaque_D32_filter_DX_SSSE3(const SkBitmapProcState& s, 738 const uint32_t* xy, 739 int count, uint32_t* colors) { 740 sk_throw(); 741 } 742 743 void S32_alpha_D32_filter_DX_SSSE3(const SkBitmapProcState& s, 744 const uint32_t* xy, 745 int count, uint32_t* colors) { 746 sk_throw(); 747 } 748 749 void S32_opaque_D32_filter_DXDY_SSSE3(const SkBitmapProcState& s, 750 const uint32_t* xy, 751 int count, uint32_t* colors) { 752 sk_throw(); 753 } 754 755 void S32_alpha_D32_filter_DXDY_SSSE3(const SkBitmapProcState& s, 756 const uint32_t* xy, 757 int count, uint32_t* colors) { 758 sk_throw(); 759 } 760 761 #endif 762