1 /* 2 * Copyright 2012 The Android Open Source Project 3 * 4 * Use of this source code is governed by a BSD-style license that can be 5 * found in the LICENSE file. 6 */ 7 8 #include <tmmintrin.h> // SSSE3 9 #include "SkBitmapProcState_opts_SSSE3.h" 10 #include "SkPaint.h" 11 #include "SkUtils.h" 12 13 // adding anonymous namespace seemed to force gcc to inline directly the 14 // instantiation, instead of creating the functions 15 // S32_generic_D32_filter_DX_SSSE3<true> and 16 // S32_generic_D32_filter_DX_SSSE3<false> which were then called by the 17 // external functions. 18 namespace { 19 // In this file, variations for alpha and non alpha versions are implemented 20 // with a template, as it makes the code more compact and a bit easier to 21 // maintain, while making the compiler generate the same exact code as with 22 // two functions that only differ by a few lines. 23 24 25 // Prepare all necessary constants for a round of processing for two pixel 26 // pairs. 27 // @param xy is the location where the xy parameters for four pixels should be 28 // read from. It is identical in concept with argument two of 29 // S32_{opaque}_D32_filter_DX methods. 30 // @param mask_3FFF vector of 32 bit constants containing 3FFF, 31 // suitable to mask the bottom 14 bits of a XY value. 32 // @param mask_000F vector of 32 bit constants containing 000F, 33 // suitable to mask the bottom 4 bits of a XY value. 34 // @param sixteen_8bit vector of 8 bit components containing the value 16. 35 // @param mask_dist_select vector of 8 bit components containing the shuffling 36 // parameters to reorder x[0-3] parameters. 37 // @param all_x_result vector of 8 bit components that will contain the 38 // (4x(x3), 4x(x2), 4x(x1), 4x(x0)) upon return. 39 // @param sixteen_minus_x vector of 8 bit components, containing 40 // (4x(16 - x3), 4x(16 - x2), 4x(16 - x1), 4x(16 - x0)) 41 inline void PrepareConstantsTwoPixelPairs(const uint32_t* xy, 42 const __m128i& mask_3FFF, 43 const __m128i& mask_000F, 44 const __m128i& sixteen_8bit, 45 const __m128i& mask_dist_select, 46 __m128i* all_x_result, 47 __m128i* sixteen_minus_x, 48 int* x0, 49 int* x1) { 50 const __m128i xx = _mm_loadu_si128(reinterpret_cast<const __m128i *>(xy)); 51 52 // 4 delta X 53 // (x03, x02, x01, x00) 54 const __m128i x0_wide = _mm_srli_epi32(xx, 18); 55 // (x13, x12, x11, x10) 56 const __m128i x1_wide = _mm_and_si128(xx, mask_3FFF); 57 58 _mm_storeu_si128(reinterpret_cast<__m128i *>(x0), x0_wide); 59 _mm_storeu_si128(reinterpret_cast<__m128i *>(x1), x1_wide); 60 61 __m128i all_x = _mm_and_si128(_mm_srli_epi32(xx, 14), mask_000F); 62 63 // (4x(x3), 4x(x2), 4x(x1), 4x(x0)) 64 all_x = _mm_shuffle_epi8(all_x, mask_dist_select); 65 66 *all_x_result = all_x; 67 // (4x(16-x3), 4x(16-x2), 4x(16-x1), 4x(16-x0)) 68 *sixteen_minus_x = _mm_sub_epi8(sixteen_8bit, all_x); 69 } 70 71 // Prepare all necessary constants for a round of processing for two pixel 72 // pairs. 73 // @param xy is the location where the xy parameters for four pixels should be 74 // read from. It is identical in concept with argument two of 75 // S32_{opaque}_D32_filter_DXDY methods. 76 // @param mask_3FFF vector of 32 bit constants containing 3FFF, 77 // suitable to mask the bottom 14 bits of a XY value. 78 // @param mask_000F vector of 32 bit constants containing 000F, 79 // suitable to mask the bottom 4 bits of a XY value. 80 // @param sixteen_8bit vector of 8 bit components containing the value 16. 81 // @param mask_dist_select vector of 8 bit components containing the shuffling 82 // parameters to reorder x[0-3] parameters. 83 // @param all_xy_result vector of 8 bit components that will contain the 84 // (4x(y1), 4x(y0), 4x(x1), 4x(x0)) upon return. 85 // @param sixteen_minus_x vector of 8 bit components, containing 86 // (4x(16-y1), 4x(16-y0), 4x(16-x1), 4x(16-x0)). 87 inline void PrepareConstantsTwoPixelPairsDXDY(const uint32_t* xy, 88 const __m128i& mask_3FFF, 89 const __m128i& mask_000F, 90 const __m128i& sixteen_8bit, 91 const __m128i& mask_dist_select, 92 __m128i* all_xy_result, 93 __m128i* sixteen_minus_xy, 94 int* xy0, int* xy1) { 95 const __m128i xy_wide = 96 _mm_loadu_si128(reinterpret_cast<const __m128i *>(xy)); 97 98 // (x10, y10, x00, y00) 99 __m128i xy0_wide = _mm_srli_epi32(xy_wide, 18); 100 // (y10, y00, x10, x00) 101 xy0_wide = _mm_shuffle_epi32(xy0_wide, _MM_SHUFFLE(2, 0, 3, 1)); 102 // (x11, y11, x01, y01) 103 __m128i xy1_wide = _mm_and_si128(xy_wide, mask_3FFF); 104 // (y11, y01, x11, x01) 105 xy1_wide = _mm_shuffle_epi32(xy1_wide, _MM_SHUFFLE(2, 0, 3, 1)); 106 107 _mm_storeu_si128(reinterpret_cast<__m128i *>(xy0), xy0_wide); 108 _mm_storeu_si128(reinterpret_cast<__m128i *>(xy1), xy1_wide); 109 110 // (x1, y1, x0, y0) 111 __m128i all_xy = _mm_and_si128(_mm_srli_epi32(xy_wide, 14), mask_000F); 112 // (y1, y0, x1, x0) 113 all_xy = _mm_shuffle_epi32(all_xy, _MM_SHUFFLE(2, 0, 3, 1)); 114 // (4x(y1), 4x(y0), 4x(x1), 4x(x0)) 115 all_xy = _mm_shuffle_epi8(all_xy, mask_dist_select); 116 117 *all_xy_result = all_xy; 118 // (4x(16-y1), 4x(16-y0), 4x(16-x1), 4x(16-x0)) 119 *sixteen_minus_xy = _mm_sub_epi8(sixteen_8bit, all_xy); 120 } 121 122 // Helper function used when processing one pixel pair. 123 // @param pixel0..3 are the four input pixels 124 // @param scale_x vector of 8 bit components to multiply the pixel[0:3]. This 125 // will contain (4x(x1, 16-x1), 4x(x0, 16-x0)) 126 // or (4x(x3, 16-x3), 4x(x2, 16-x2)) 127 // @return a vector of 16 bit components containing: 128 // (Aa2 * (16 - x1) + Aa3 * x1, ... , Ra0 * (16 - x0) + Ra1 * x0) 129 inline __m128i ProcessPixelPairHelper(uint32_t pixel0, 130 uint32_t pixel1, 131 uint32_t pixel2, 132 uint32_t pixel3, 133 const __m128i& scale_x) { 134 __m128i a0, a1, a2, a3; 135 // Load 2 pairs of pixels 136 a0 = _mm_cvtsi32_si128(pixel0); 137 a1 = _mm_cvtsi32_si128(pixel1); 138 139 // Interleave pixels. 140 // (0, 0, 0, 0, 0, 0, 0, 0, Aa1, Aa0, Ba1, Ba0, Ga1, Ga0, Ra1, Ra0) 141 a0 = _mm_unpacklo_epi8(a0, a1); 142 143 a2 = _mm_cvtsi32_si128(pixel2); 144 a3 = _mm_cvtsi32_si128(pixel3); 145 // (0, 0, 0, 0, 0, 0, 0, 0, Aa3, Aa2, Ba3, Ba2, Ga3, Ga2, Ra3, Ra2) 146 a2 = _mm_unpacklo_epi8(a2, a3); 147 148 // two pairs of pixel pairs, interleaved. 149 // (Aa3, Aa2, Ba3, Ba2, Ga3, Ga2, Ra3, Ra2, 150 // Aa1, Aa0, Ba1, Ba0, Ga1, Ga0, Ra1, Ra0) 151 a0 = _mm_unpacklo_epi64(a0, a2); 152 153 // multiply and sum to 16 bit components. 154 // (Aa2 * (16 - x1) + Aa3 * x1, ... , Ra0 * (16 - x0) + Ra1 * x0) 155 // At that point, we use up a bit less than 12 bits for each 16 bit 156 // component: 157 // All components are less than 255. So, 158 // C0 * (16 - x) + C1 * x <= 255 * (16 - x) + 255 * x = 255 * 16. 159 return _mm_maddubs_epi16(a0, scale_x); 160 } 161 162 // Scale back the results after multiplications to the [0:255] range, and scale 163 // by alpha when has_alpha is true. 164 // Depending on whether one set or two sets of multiplications had been applied, 165 // the results have to be shifted by four places (dividing by 16), or shifted 166 // by eight places (dividing by 256), since each multiplication is by a quantity 167 // in the range [0:16]. 168 template<bool has_alpha, int scale> 169 inline __m128i ScaleFourPixels(__m128i* pixels, 170 const __m128i& alpha) { 171 // Divide each 16 bit component by 16 (or 256 depending on scale). 172 *pixels = _mm_srli_epi16(*pixels, scale); 173 174 if (has_alpha) { 175 // Multiply by alpha. 176 *pixels = _mm_mullo_epi16(*pixels, alpha); 177 178 // Divide each 16 bit component by 256. 179 *pixels = _mm_srli_epi16(*pixels, 8); 180 } 181 return *pixels; 182 } 183 184 // Wrapper to calculate two output pixels from four input pixels. The 185 // arguments are the same as ProcessPixelPairHelper. Technically, there are 186 // eight input pixels, but since sub_y == 0, the factors applied to half of the 187 // pixels is zero (sub_y), and are therefore omitted here to save on some 188 // processing. 189 // @param alpha when has_alpha is true, scale all resulting components by this 190 // value. 191 // @return a vector of 16 bit components containing: 192 // ((Aa2 * (16 - x1) + Aa3 * x1) * alpha, ..., 193 // (Ra0 * (16 - x0) + Ra1 * x0) * alpha) (when has_alpha is true) 194 // otherwise 195 // (Aa2 * (16 - x1) + Aa3 * x1, ... , Ra0 * (16 - x0) + Ra1 * x0) 196 // In both cases, the results are renormalized (divided by 16) to match the 197 // expected formats when storing back the results into memory. 198 template<bool has_alpha> 199 inline __m128i ProcessPixelPairZeroSubY(uint32_t pixel0, 200 uint32_t pixel1, 201 uint32_t pixel2, 202 uint32_t pixel3, 203 const __m128i& scale_x, 204 const __m128i& alpha) { 205 __m128i sum = ProcessPixelPairHelper(pixel0, pixel1, pixel2, pixel3, 206 scale_x); 207 return ScaleFourPixels<has_alpha, 4>(&sum, alpha); 208 } 209 210 // Same as ProcessPixelPairZeroSubY, expect processing one output pixel at a 211 // time instead of two. As in the above function, only two pixels are needed 212 // to generate a single pixel since sub_y == 0. 213 // @return same as ProcessPixelPairZeroSubY, except that only the bottom 4 214 // 16 bit components are set. 215 template<bool has_alpha> 216 inline __m128i ProcessOnePixelZeroSubY(uint32_t pixel0, 217 uint32_t pixel1, 218 __m128i scale_x, 219 __m128i alpha) { 220 __m128i a0 = _mm_cvtsi32_si128(pixel0); 221 __m128i a1 = _mm_cvtsi32_si128(pixel1); 222 223 // Interleave 224 a0 = _mm_unpacklo_epi8(a0, a1); 225 226 // (a0 * (16-x) + a1 * x) 227 __m128i sum = _mm_maddubs_epi16(a0, scale_x); 228 229 return ScaleFourPixels<has_alpha, 4>(&sum, alpha); 230 } 231 232 // Methods when sub_y != 0 233 234 235 // Same as ProcessPixelPairHelper, except that the values are scaled by y. 236 // @param y vector of 16 bit components containing 'y' values. There are two 237 // cases in practice, where y will contain the sub_y constant, or will 238 // contain the 16 - sub_y constant. 239 // @return vector of 16 bit components containing: 240 // (y * (Aa2 * (16 - x1) + Aa3 * x1), ... , y * (Ra0 * (16 - x0) + Ra1 * x0)) 241 inline __m128i ProcessPixelPair(uint32_t pixel0, 242 uint32_t pixel1, 243 uint32_t pixel2, 244 uint32_t pixel3, 245 const __m128i& scale_x, 246 const __m128i& y) { 247 __m128i sum = ProcessPixelPairHelper(pixel0, pixel1, pixel2, pixel3, 248 scale_x); 249 250 // first row times 16-y or y depending on whether 'y' represents one or 251 // the other. 252 // Values will be up to 255 * 16 * 16 = 65280. 253 // (y * (Aa2 * (16 - x1) + Aa3 * x1), ... , 254 // y * (Ra0 * (16 - x0) + Ra1 * x0)) 255 sum = _mm_mullo_epi16(sum, y); 256 257 return sum; 258 } 259 260 // Process two pixel pairs out of eight input pixels. 261 // In other methods, the distinct pixels are passed one by one, but in this 262 // case, the rows, and index offsets to the pixels into the row are passed 263 // to generate the 8 pixels. 264 // @param row0..1 top and bottom row where to find input pixels. 265 // @param x0..1 offsets into the row for all eight input pixels. 266 // @param all_y vector of 16 bit components containing the constant sub_y 267 // @param neg_y vector of 16 bit components containing the constant 16 - sub_y 268 // @param alpha vector of 16 bit components containing the alpha value to scale 269 // the results by, when has_alpha is true. 270 // @return 271 // (alpha * ((16-y) * (Aa2 * (16-x1) + Aa3 * x1) + 272 // y * (Aa2' * (16-x1) + Aa3' * x1)), 273 // ... 274 // alpha * ((16-y) * (Ra0 * (16-x0) + Ra1 * x0) + 275 // y * (Ra0' * (16-x0) + Ra1' * x0)) 276 // With the factor alpha removed when has_alpha is false. 277 // The values are scaled back to 16 bit components, but with only the bottom 278 // 8 bits being set. 279 template<bool has_alpha> 280 inline __m128i ProcessTwoPixelPairs(const uint32_t* row0, 281 const uint32_t* row1, 282 const int* x0, 283 const int* x1, 284 const __m128i& scale_x, 285 const __m128i& all_y, 286 const __m128i& neg_y, 287 const __m128i& alpha) { 288 __m128i sum0 = ProcessPixelPair( 289 row0[x0[0]], row0[x1[0]], row0[x0[1]], row0[x1[1]], 290 scale_x, neg_y); 291 __m128i sum1 = ProcessPixelPair( 292 row1[x0[0]], row1[x1[0]], row1[x0[1]], row1[x1[1]], 293 scale_x, all_y); 294 295 // 2 samples fully summed. 296 // ((16-y) * (Aa2 * (16-x1) + Aa3 * x1) + 297 // y * (Aa2' * (16-x1) + Aa3' * x1), 298 // ... 299 // (16-y) * (Ra0 * (16 - x0) + Ra1 * x0)) + 300 // y * (Ra0' * (16-x0) + Ra1' * x0)) 301 // Each component, again can be at most 256 * 255 = 65280, so no overflow. 302 sum0 = _mm_add_epi16(sum0, sum1); 303 304 return ScaleFourPixels<has_alpha, 8>(&sum0, alpha); 305 } 306 307 // Similar to ProcessTwoPixelPairs except the pixel indexes. 308 template<bool has_alpha> 309 inline __m128i ProcessTwoPixelPairsDXDY(const uint32_t* row00, 310 const uint32_t* row01, 311 const uint32_t* row10, 312 const uint32_t* row11, 313 const int* xy0, 314 const int* xy1, 315 const __m128i& scale_x, 316 const __m128i& all_y, 317 const __m128i& neg_y, 318 const __m128i& alpha) { 319 // first row 320 __m128i sum0 = ProcessPixelPair( 321 row00[xy0[0]], row00[xy1[0]], row10[xy0[1]], row10[xy1[1]], 322 scale_x, neg_y); 323 // second row 324 __m128i sum1 = ProcessPixelPair( 325 row01[xy0[0]], row01[xy1[0]], row11[xy0[1]], row11[xy1[1]], 326 scale_x, all_y); 327 328 // 2 samples fully summed. 329 // ((16-y1) * (Aa2 * (16-x1) + Aa3 * x1) + 330 // y0 * (Aa2' * (16-x1) + Aa3' * x1), 331 // ... 332 // (16-y0) * (Ra0 * (16 - x0) + Ra1 * x0)) + 333 // y0 * (Ra0' * (16-x0) + Ra1' * x0)) 334 // Each component, again can be at most 256 * 255 = 65280, so no overflow. 335 sum0 = _mm_add_epi16(sum0, sum1); 336 337 return ScaleFourPixels<has_alpha, 8>(&sum0, alpha); 338 } 339 340 341 // Same as ProcessPixelPair, except that performing the math one output pixel 342 // at a time. This means that only the bottom four 16 bit components are set. 343 inline __m128i ProcessOnePixel(uint32_t pixel0, uint32_t pixel1, 344 const __m128i& scale_x, const __m128i& y) { 345 __m128i a0 = _mm_cvtsi32_si128(pixel0); 346 __m128i a1 = _mm_cvtsi32_si128(pixel1); 347 348 // Interleave 349 // (0, 0, 0, 0, 0, 0, 0, 0, Aa1, Aa0, Ba1, Ba0, Ga1, Ga0, Ra1, Ra0) 350 a0 = _mm_unpacklo_epi8(a0, a1); 351 352 // (a0 * (16-x) + a1 * x) 353 a0 = _mm_maddubs_epi16(a0, scale_x); 354 355 // scale row by y 356 return _mm_mullo_epi16(a0, y); 357 } 358 359 // Notes about the various tricks that are used in this implementation: 360 // - specialization for sub_y == 0. 361 // Statistically, 1/16th of the samples will have sub_y == 0. When this 362 // happens, the math goes from: 363 // (16 - x)*(16 - y)*a00 + x*(16 - y)*a01 + (16 - x)*y*a10 + x*y*a11 364 // to: 365 // (16 - x)*a00 + 16*x*a01 366 // much simpler. The simplification makes for an easy boost in performance. 367 // - calculating 4 output pixels at a time. 368 // This allows loading the coefficients x0 and x1 and shuffling them to the 369 // optimum location only once per loop, instead of twice per loop. 370 // This also allows us to store the four pixels with a single store. 371 // - Use of 2 special SSSE3 instructions (comparatively to the SSE2 instruction 372 // version): 373 // _mm_shuffle_epi8 : this allows us to spread the coefficients x[0-3] loaded 374 // in 32 bit values to 8 bit values repeated four times. 375 // _mm_maddubs_epi16 : this allows us to perform multiplications and additions 376 // in one swoop of 8bit values storing the results in 16 bit values. This 377 // instruction is actually crucial for the speed of the implementation since 378 // as one can see in the SSE2 implementation, all inputs have to be used as 379 // 16 bits because the results are 16 bits. This basically allows us to process 380 // twice as many pixel components per iteration. 381 // 382 // As a result, this method behaves faster than the traditional SSE2. The actual 383 // boost varies greatly on the underlying architecture. 384 template<bool has_alpha> 385 void S32_generic_D32_filter_DX_SSSE3(const SkBitmapProcState& s, 386 const uint32_t* xy, 387 int count, uint32_t* colors) { 388 SkASSERT(count > 0 && colors != NULL); 389 SkASSERT(s.fFilterLevel != SkPaint::kNone_FilterLevel); 390 SkASSERT(s.fBitmap->config() == SkBitmap::kARGB_8888_Config); 391 if (has_alpha) { 392 SkASSERT(s.fAlphaScale < 256); 393 } else { 394 SkASSERT(s.fAlphaScale == 256); 395 } 396 397 const uint8_t* src_addr = 398 static_cast<const uint8_t*>(s.fBitmap->getPixels()); 399 const size_t rb = s.fBitmap->rowBytes(); 400 const uint32_t XY = *xy++; 401 const unsigned y0 = XY >> 14; 402 const uint32_t* row0 = 403 reinterpret_cast<const uint32_t*>(src_addr + (y0 >> 4) * rb); 404 const uint32_t* row1 = 405 reinterpret_cast<const uint32_t*>(src_addr + (XY & 0x3FFF) * rb); 406 const unsigned sub_y = y0 & 0xF; 407 408 // vector constants 409 const __m128i mask_dist_select = _mm_set_epi8(12, 12, 12, 12, 410 8, 8, 8, 8, 411 4, 4, 4, 4, 412 0, 0, 0, 0); 413 const __m128i mask_3FFF = _mm_set1_epi32(0x3FFF); 414 const __m128i mask_000F = _mm_set1_epi32(0x000F); 415 const __m128i sixteen_8bit = _mm_set1_epi8(16); 416 // (0, 0, 0, 0, 0, 0, 0, 0) 417 const __m128i zero = _mm_setzero_si128(); 418 419 __m128i alpha = _mm_setzero_si128(); 420 if (has_alpha) 421 // 8x(alpha) 422 alpha = _mm_set1_epi16(s.fAlphaScale); 423 424 if (sub_y == 0) { 425 // Unroll 4x, interleave bytes, use pmaddubsw (all_x is small) 426 while (count > 3) { 427 count -= 4; 428 429 int x0[4]; 430 int x1[4]; 431 __m128i all_x, sixteen_minus_x; 432 PrepareConstantsTwoPixelPairs(xy, mask_3FFF, mask_000F, 433 sixteen_8bit, mask_dist_select, 434 &all_x, &sixteen_minus_x, x0, x1); 435 xy += 4; 436 437 // First pair of pixel pairs. 438 // (4x(x1, 16-x1), 4x(x0, 16-x0)) 439 __m128i scale_x; 440 scale_x = _mm_unpacklo_epi8(sixteen_minus_x, all_x); 441 442 __m128i sum0 = ProcessPixelPairZeroSubY<has_alpha>( 443 row0[x0[0]], row0[x1[0]], row0[x0[1]], row0[x1[1]], 444 scale_x, alpha); 445 446 // second pair of pixel pairs 447 // (4x (x3, 16-x3), 4x (16-x2, x2)) 448 scale_x = _mm_unpackhi_epi8(sixteen_minus_x, all_x); 449 450 __m128i sum1 = ProcessPixelPairZeroSubY<has_alpha>( 451 row0[x0[2]], row0[x1[2]], row0[x0[3]], row0[x1[3]], 452 scale_x, alpha); 453 454 // Pack lower 4 16 bit values of sum into lower 4 bytes. 455 sum0 = _mm_packus_epi16(sum0, sum1); 456 457 // Extract low int and store. 458 _mm_storeu_si128(reinterpret_cast<__m128i *>(colors), sum0); 459 460 colors += 4; 461 } 462 463 // handle remainder 464 while (count-- > 0) { 465 uint32_t xx = *xy++; // x0:14 | 4 | x1:14 466 unsigned x0 = xx >> 18; 467 unsigned x1 = xx & 0x3FFF; 468 469 // 16x(x) 470 const __m128i all_x = _mm_set1_epi8((xx >> 14) & 0x0F); 471 472 // (16x(16-x)) 473 __m128i scale_x = _mm_sub_epi8(sixteen_8bit, all_x); 474 475 scale_x = _mm_unpacklo_epi8(scale_x, all_x); 476 477 __m128i sum = ProcessOnePixelZeroSubY<has_alpha>( 478 row0[x0], row0[x1], 479 scale_x, alpha); 480 481 // Pack lower 4 16 bit values of sum into lower 4 bytes. 482 sum = _mm_packus_epi16(sum, zero); 483 484 // Extract low int and store. 485 *colors++ = _mm_cvtsi128_si32(sum); 486 } 487 } else { // more general case, y != 0 488 // 8x(16) 489 const __m128i sixteen_16bit = _mm_set1_epi16(16); 490 491 // 8x (y) 492 const __m128i all_y = _mm_set1_epi16(sub_y); 493 494 // 8x (16-y) 495 const __m128i neg_y = _mm_sub_epi16(sixteen_16bit, all_y); 496 497 // Unroll 4x, interleave bytes, use pmaddubsw (all_x is small) 498 while (count > 3) { 499 count -= 4; 500 501 int x0[4]; 502 int x1[4]; 503 __m128i all_x, sixteen_minus_x; 504 PrepareConstantsTwoPixelPairs(xy, mask_3FFF, mask_000F, 505 sixteen_8bit, mask_dist_select, 506 &all_x, &sixteen_minus_x, x0, x1); 507 xy += 4; 508 509 // First pair of pixel pairs 510 // (4x(x1, 16-x1), 4x(x0, 16-x0)) 511 __m128i scale_x; 512 scale_x = _mm_unpacklo_epi8(sixteen_minus_x, all_x); 513 514 __m128i sum0 = ProcessTwoPixelPairs<has_alpha>( 515 row0, row1, x0, x1, 516 scale_x, all_y, neg_y, alpha); 517 518 // second pair of pixel pairs 519 // (4x (x3, 16-x3), 4x (16-x2, x2)) 520 scale_x = _mm_unpackhi_epi8(sixteen_minus_x, all_x); 521 522 __m128i sum1 = ProcessTwoPixelPairs<has_alpha>( 523 row0, row1, x0 + 2, x1 + 2, 524 scale_x, all_y, neg_y, alpha); 525 526 // Do the final packing of the two results 527 528 // Pack lower 4 16 bit values of sum into lower 4 bytes. 529 sum0 = _mm_packus_epi16(sum0, sum1); 530 531 // Extract low int and store. 532 _mm_storeu_si128(reinterpret_cast<__m128i *>(colors), sum0); 533 534 colors += 4; 535 } 536 537 // Left over. 538 while (count-- > 0) { 539 const uint32_t xx = *xy++; // x0:14 | 4 | x1:14 540 const unsigned x0 = xx >> 18; 541 const unsigned x1 = xx & 0x3FFF; 542 543 // 16x(x) 544 const __m128i all_x = _mm_set1_epi8((xx >> 14) & 0x0F); 545 546 // 16x (16-x) 547 __m128i scale_x = _mm_sub_epi8(sixteen_8bit, all_x); 548 549 // (8x (x, 16-x)) 550 scale_x = _mm_unpacklo_epi8(scale_x, all_x); 551 552 // first row. 553 __m128i sum0 = ProcessOnePixel(row0[x0], row0[x1], scale_x, neg_y); 554 // second row. 555 __m128i sum1 = ProcessOnePixel(row1[x0], row1[x1], scale_x, all_y); 556 557 // Add both rows for full sample 558 sum0 = _mm_add_epi16(sum0, sum1); 559 560 sum0 = ScaleFourPixels<has_alpha, 8>(&sum0, alpha); 561 562 // Pack lower 4 16 bit values of sum into lower 4 bytes. 563 sum0 = _mm_packus_epi16(sum0, zero); 564 565 // Extract low int and store. 566 *colors++ = _mm_cvtsi128_si32(sum0); 567 } 568 } 569 } 570 571 /* 572 * Similar to S32_generic_D32_filter_DX_SSSE3, we do not need to handle the 573 * special case suby == 0 as suby is changing in every loop. 574 */ 575 template<bool has_alpha> 576 void S32_generic_D32_filter_DXDY_SSSE3(const SkBitmapProcState& s, 577 const uint32_t* xy, 578 int count, uint32_t* colors) { 579 SkASSERT(count > 0 && colors != NULL); 580 SkASSERT(s.fFilterLevel != SkPaint::kNone_FilterLevel); 581 SkASSERT(s.fBitmap->config() == SkBitmap::kARGB_8888_Config); 582 if (has_alpha) { 583 SkASSERT(s.fAlphaScale < 256); 584 } else { 585 SkASSERT(s.fAlphaScale == 256); 586 } 587 588 const uint8_t* src_addr = 589 static_cast<const uint8_t*>(s.fBitmap->getPixels()); 590 const size_t rb = s.fBitmap->rowBytes(); 591 592 // vector constants 593 const __m128i mask_dist_select = _mm_set_epi8(12, 12, 12, 12, 594 8, 8, 8, 8, 595 4, 4, 4, 4, 596 0, 0, 0, 0); 597 const __m128i mask_3FFF = _mm_set1_epi32(0x3FFF); 598 const __m128i mask_000F = _mm_set1_epi32(0x000F); 599 const __m128i sixteen_8bit = _mm_set1_epi8(16); 600 601 __m128i alpha; 602 if (has_alpha) { 603 // 8x(alpha) 604 alpha = _mm_set1_epi16(s.fAlphaScale); 605 } 606 607 // Unroll 2x, interleave bytes, use pmaddubsw (all_x is small) 608 while (count >= 2) { 609 int xy0[4]; 610 int xy1[4]; 611 __m128i all_xy, sixteen_minus_xy; 612 PrepareConstantsTwoPixelPairsDXDY(xy, mask_3FFF, mask_000F, 613 sixteen_8bit, mask_dist_select, 614 &all_xy, &sixteen_minus_xy, xy0, xy1); 615 616 // (4x(x1, 16-x1), 4x(x0, 16-x0)) 617 __m128i scale_x = _mm_unpacklo_epi8(sixteen_minus_xy, all_xy); 618 // (4x(0, y1), 4x(0, y0)) 619 __m128i all_y = _mm_unpackhi_epi8(all_xy, _mm_setzero_si128()); 620 __m128i neg_y = _mm_sub_epi16(_mm_set1_epi16(16), all_y); 621 622 const uint32_t* row00 = 623 reinterpret_cast<const uint32_t*>(src_addr + xy0[2] * rb); 624 const uint32_t* row01 = 625 reinterpret_cast<const uint32_t*>(src_addr + xy1[2] * rb); 626 const uint32_t* row10 = 627 reinterpret_cast<const uint32_t*>(src_addr + xy0[3] * rb); 628 const uint32_t* row11 = 629 reinterpret_cast<const uint32_t*>(src_addr + xy1[3] * rb); 630 631 __m128i sum0 = ProcessTwoPixelPairsDXDY<has_alpha>( 632 row00, row01, row10, row11, xy0, xy1, 633 scale_x, all_y, neg_y, alpha); 634 635 // Pack lower 4 16 bit values of sum into lower 4 bytes. 636 sum0 = _mm_packus_epi16(sum0, _mm_setzero_si128()); 637 638 // Extract low int and store. 639 _mm_storel_epi64(reinterpret_cast<__m128i *>(colors), sum0); 640 641 xy += 4; 642 colors += 2; 643 count -= 2; 644 } 645 646 // Handle the remainder 647 while (count-- > 0) { 648 uint32_t data = *xy++; 649 unsigned y0 = data >> 14; 650 unsigned y1 = data & 0x3FFF; 651 unsigned subY = y0 & 0xF; 652 y0 >>= 4; 653 654 data = *xy++; 655 unsigned x0 = data >> 14; 656 unsigned x1 = data & 0x3FFF; 657 unsigned subX = x0 & 0xF; 658 x0 >>= 4; 659 660 const uint32_t* row0 = 661 reinterpret_cast<const uint32_t*>(src_addr + y0 * rb); 662 const uint32_t* row1 = 663 reinterpret_cast<const uint32_t*>(src_addr + y1 * rb); 664 665 // 16x(x) 666 const __m128i all_x = _mm_set1_epi8(subX); 667 668 // 16x (16-x) 669 __m128i scale_x = _mm_sub_epi8(sixteen_8bit, all_x); 670 671 // (8x (x, 16-x)) 672 scale_x = _mm_unpacklo_epi8(scale_x, all_x); 673 674 // 8x(16) 675 const __m128i sixteen_16bit = _mm_set1_epi16(16); 676 677 // 8x (y) 678 const __m128i all_y = _mm_set1_epi16(subY); 679 680 // 8x (16-y) 681 const __m128i neg_y = _mm_sub_epi16(sixteen_16bit, all_y); 682 683 // first row. 684 __m128i sum0 = ProcessOnePixel(row0[x0], row0[x1], scale_x, neg_y); 685 // second row. 686 __m128i sum1 = ProcessOnePixel(row1[x0], row1[x1], scale_x, all_y); 687 688 // Add both rows for full sample 689 sum0 = _mm_add_epi16(sum0, sum1); 690 691 sum0 = ScaleFourPixels<has_alpha, 8>(&sum0, alpha); 692 693 // Pack lower 4 16 bit values of sum into lower 4 bytes. 694 sum0 = _mm_packus_epi16(sum0, _mm_setzero_si128()); 695 696 // Extract low int and store. 697 *colors++ = _mm_cvtsi128_si32(sum0); 698 } 699 } 700 } // namepace 701 702 void S32_opaque_D32_filter_DX_SSSE3(const SkBitmapProcState& s, 703 const uint32_t* xy, 704 int count, uint32_t* colors) { 705 S32_generic_D32_filter_DX_SSSE3<false>(s, xy, count, colors); 706 } 707 708 void S32_alpha_D32_filter_DX_SSSE3(const SkBitmapProcState& s, 709 const uint32_t* xy, 710 int count, uint32_t* colors) { 711 S32_generic_D32_filter_DX_SSSE3<true>(s, xy, count, colors); 712 } 713 714 void S32_opaque_D32_filter_DXDY_SSSE3(const SkBitmapProcState& s, 715 const uint32_t* xy, 716 int count, uint32_t* colors) { 717 S32_generic_D32_filter_DXDY_SSSE3<false>(s, xy, count, colors); 718 } 719 720 void S32_alpha_D32_filter_DXDY_SSSE3(const SkBitmapProcState& s, 721 const uint32_t* xy, 722 int count, uint32_t* colors) { 723 S32_generic_D32_filter_DXDY_SSSE3<true>(s, xy, count, colors); 724 } 725