Home | History | Annotate | Download | only in opts
      1 /*
      2  * Copyright 2009 The Android Open Source Project
      3  *
      4  * Use of this source code is governed by a BSD-style license that can be
      5  * found in the LICENSE file.
      6  */
      7 
      8 #include <emmintrin.h>
      9 #include "SkBitmapProcState_opts_SSE2.h"
     10 #include "SkColorPriv.h"
     11 #include "SkPaint.h"
     12 #include "SkUtils.h"
     13 
     14 void S32_opaque_D32_filter_DX_SSE2(const SkBitmapProcState& s,
     15                                    const uint32_t* xy,
     16                                    int count, uint32_t* colors) {
     17     SkASSERT(count > 0 && colors != NULL);
     18     SkASSERT(s.fFilterLevel != kNone_SkFilterQuality);
     19     SkASSERT(kN32_SkColorType == s.fBitmap->colorType());
     20     SkASSERT(s.fAlphaScale == 256);
     21 
     22     const char* srcAddr = static_cast<const char*>(s.fBitmap->getPixels());
     23     size_t rb = s.fBitmap->rowBytes();
     24     uint32_t XY = *xy++;
     25     unsigned y0 = XY >> 14;
     26     const uint32_t* row0 = reinterpret_cast<const uint32_t*>(srcAddr + (y0 >> 4) * rb);
     27     const uint32_t* row1 = reinterpret_cast<const uint32_t*>(srcAddr + (XY & 0x3FFF) * rb);
     28     unsigned subY = y0 & 0xF;
     29 
     30     // ( 0,  0,  0,  0,  0,  0,  0, 16)
     31     __m128i sixteen = _mm_cvtsi32_si128(16);
     32 
     33     // ( 0,  0,  0,  0, 16, 16, 16, 16)
     34     sixteen = _mm_shufflelo_epi16(sixteen, 0);
     35 
     36     // ( 0,  0,  0,  0,  0,  0,  0,  y)
     37     __m128i allY = _mm_cvtsi32_si128(subY);
     38 
     39     // ( 0,  0,  0,  0,  y,  y,  y,  y)
     40     allY = _mm_shufflelo_epi16(allY, 0);
     41 
     42     // ( 0,  0,  0,  0, 16-y, 16-y, 16-y, 16-y)
     43     __m128i negY = _mm_sub_epi16(sixteen, allY);
     44 
     45     // (16-y, 16-y, 16-y, 16-y, y, y, y, y)
     46     allY = _mm_unpacklo_epi64(allY, negY);
     47 
     48     // (16, 16, 16, 16, 16, 16, 16, 16 )
     49     sixteen = _mm_shuffle_epi32(sixteen, 0);
     50 
     51     // ( 0,  0,  0,  0,  0,  0,  0,  0)
     52     __m128i zero = _mm_setzero_si128();
     53     do {
     54         uint32_t XX = *xy++;    // x0:14 | 4 | x1:14
     55         unsigned x0 = XX >> 18;
     56         unsigned x1 = XX & 0x3FFF;
     57 
     58         // (0, 0, 0, 0, 0, 0, 0, x)
     59         __m128i allX = _mm_cvtsi32_si128((XX >> 14) & 0x0F);
     60 
     61         // (0, 0, 0, 0, x, x, x, x)
     62         allX = _mm_shufflelo_epi16(allX, 0);
     63 
     64         // (x, x, x, x, x, x, x, x)
     65         allX = _mm_shuffle_epi32(allX, 0);
     66 
     67         // (16-x, 16-x, 16-x, 16-x, 16-x, 16-x, 16-x)
     68         __m128i negX = _mm_sub_epi16(sixteen, allX);
     69 
     70         // Load 4 samples (pixels).
     71         __m128i a00 = _mm_cvtsi32_si128(row0[x0]);
     72         __m128i a01 = _mm_cvtsi32_si128(row0[x1]);
     73         __m128i a10 = _mm_cvtsi32_si128(row1[x0]);
     74         __m128i a11 = _mm_cvtsi32_si128(row1[x1]);
     75 
     76         // (0, 0, a00, a10)
     77         __m128i a00a10 = _mm_unpacklo_epi32(a10, a00);
     78 
     79         // Expand to 16 bits per component.
     80         a00a10 = _mm_unpacklo_epi8(a00a10, zero);
     81 
     82         // ((a00 * (16-y)), (a10 * y)).
     83         a00a10 = _mm_mullo_epi16(a00a10, allY);
     84 
     85         // (a00 * (16-y) * (16-x), a10 * y * (16-x)).
     86         a00a10 = _mm_mullo_epi16(a00a10, negX);
     87 
     88         // (0, 0, a01, a10)
     89         __m128i a01a11 = _mm_unpacklo_epi32(a11, a01);
     90 
     91         // Expand to 16 bits per component.
     92         a01a11 = _mm_unpacklo_epi8(a01a11, zero);
     93 
     94         // (a01 * (16-y)), (a11 * y)
     95         a01a11 = _mm_mullo_epi16(a01a11, allY);
     96 
     97         // (a01 * (16-y) * x), (a11 * y * x)
     98         a01a11 = _mm_mullo_epi16(a01a11, allX);
     99 
    100         // (a00*w00 + a01*w01, a10*w10 + a11*w11)
    101         __m128i sum = _mm_add_epi16(a00a10, a01a11);
    102 
    103         // (DC, a00*w00 + a01*w01)
    104         __m128i shifted = _mm_shuffle_epi32(sum, 0xEE);
    105 
    106         // (DC, a00*w00 + a01*w01 + a10*w10 + a11*w11)
    107         sum = _mm_add_epi16(sum, shifted);
    108 
    109         // Divide each 16 bit component by 256.
    110         sum = _mm_srli_epi16(sum, 8);
    111 
    112         // Pack lower 4 16 bit values of sum into lower 4 bytes.
    113         sum = _mm_packus_epi16(sum, zero);
    114 
    115         // Extract low int and store.
    116         *colors++ = _mm_cvtsi128_si32(sum);
    117     } while (--count > 0);
    118 }
    119 
    120 void S32_alpha_D32_filter_DX_SSE2(const SkBitmapProcState& s,
    121                                   const uint32_t* xy,
    122                                   int count, uint32_t* colors) {
    123     SkASSERT(count > 0 && colors != NULL);
    124     SkASSERT(s.fFilterLevel != kNone_SkFilterQuality);
    125     SkASSERT(kN32_SkColorType == s.fBitmap->colorType());
    126     SkASSERT(s.fAlphaScale < 256);
    127 
    128     const char* srcAddr = static_cast<const char*>(s.fBitmap->getPixels());
    129     size_t rb = s.fBitmap->rowBytes();
    130     uint32_t XY = *xy++;
    131     unsigned y0 = XY >> 14;
    132     const uint32_t* row0 = reinterpret_cast<const uint32_t*>(srcAddr + (y0 >> 4) * rb);
    133     const uint32_t* row1 = reinterpret_cast<const uint32_t*>(srcAddr + (XY & 0x3FFF) * rb);
    134     unsigned subY = y0 & 0xF;
    135 
    136     // ( 0,  0,  0,  0,  0,  0,  0, 16)
    137     __m128i sixteen = _mm_cvtsi32_si128(16);
    138 
    139     // ( 0,  0,  0,  0, 16, 16, 16, 16)
    140     sixteen = _mm_shufflelo_epi16(sixteen, 0);
    141 
    142     // ( 0,  0,  0,  0,  0,  0,  0,  y)
    143     __m128i allY = _mm_cvtsi32_si128(subY);
    144 
    145     // ( 0,  0,  0,  0,  y,  y,  y,  y)
    146     allY = _mm_shufflelo_epi16(allY, 0);
    147 
    148     // ( 0,  0,  0,  0, 16-y, 16-y, 16-y, 16-y)
    149     __m128i negY = _mm_sub_epi16(sixteen, allY);
    150 
    151     // (16-y, 16-y, 16-y, 16-y, y, y, y, y)
    152     allY = _mm_unpacklo_epi64(allY, negY);
    153 
    154     // (16, 16, 16, 16, 16, 16, 16, 16 )
    155     sixteen = _mm_shuffle_epi32(sixteen, 0);
    156 
    157     // ( 0,  0,  0,  0,  0,  0,  0,  0)
    158     __m128i zero = _mm_setzero_si128();
    159 
    160     // ( alpha, alpha, alpha, alpha, alpha, alpha, alpha, alpha )
    161     __m128i alpha = _mm_set1_epi16(s.fAlphaScale);
    162 
    163     do {
    164         uint32_t XX = *xy++;    // x0:14 | 4 | x1:14
    165         unsigned x0 = XX >> 18;
    166         unsigned x1 = XX & 0x3FFF;
    167 
    168         // (0, 0, 0, 0, 0, 0, 0, x)
    169         __m128i allX = _mm_cvtsi32_si128((XX >> 14) & 0x0F);
    170 
    171         // (0, 0, 0, 0, x, x, x, x)
    172         allX = _mm_shufflelo_epi16(allX, 0);
    173 
    174         // (x, x, x, x, x, x, x, x)
    175         allX = _mm_shuffle_epi32(allX, 0);
    176 
    177         // (16-x, 16-x, 16-x, 16-x, 16-x, 16-x, 16-x)
    178         __m128i negX = _mm_sub_epi16(sixteen, allX);
    179 
    180         // Load 4 samples (pixels).
    181         __m128i a00 = _mm_cvtsi32_si128(row0[x0]);
    182         __m128i a01 = _mm_cvtsi32_si128(row0[x1]);
    183         __m128i a10 = _mm_cvtsi32_si128(row1[x0]);
    184         __m128i a11 = _mm_cvtsi32_si128(row1[x1]);
    185 
    186         // (0, 0, a00, a10)
    187         __m128i a00a10 = _mm_unpacklo_epi32(a10, a00);
    188 
    189         // Expand to 16 bits per component.
    190         a00a10 = _mm_unpacklo_epi8(a00a10, zero);
    191 
    192         // ((a00 * (16-y)), (a10 * y)).
    193         a00a10 = _mm_mullo_epi16(a00a10, allY);
    194 
    195         // (a00 * (16-y) * (16-x), a10 * y * (16-x)).
    196         a00a10 = _mm_mullo_epi16(a00a10, negX);
    197 
    198         // (0, 0, a01, a10)
    199         __m128i a01a11 = _mm_unpacklo_epi32(a11, a01);
    200 
    201         // Expand to 16 bits per component.
    202         a01a11 = _mm_unpacklo_epi8(a01a11, zero);
    203 
    204         // (a01 * (16-y)), (a11 * y)
    205         a01a11 = _mm_mullo_epi16(a01a11, allY);
    206 
    207         // (a01 * (16-y) * x), (a11 * y * x)
    208         a01a11 = _mm_mullo_epi16(a01a11, allX);
    209 
    210         // (a00*w00 + a01*w01, a10*w10 + a11*w11)
    211         __m128i sum = _mm_add_epi16(a00a10, a01a11);
    212 
    213         // (DC, a00*w00 + a01*w01)
    214         __m128i shifted = _mm_shuffle_epi32(sum, 0xEE);
    215 
    216         // (DC, a00*w00 + a01*w01 + a10*w10 + a11*w11)
    217         sum = _mm_add_epi16(sum, shifted);
    218 
    219         // Divide each 16 bit component by 256.
    220         sum = _mm_srli_epi16(sum, 8);
    221 
    222         // Multiply by alpha.
    223         sum = _mm_mullo_epi16(sum, alpha);
    224 
    225         // Divide each 16 bit component by 256.
    226         sum = _mm_srli_epi16(sum, 8);
    227 
    228         // Pack lower 4 16 bit values of sum into lower 4 bytes.
    229         sum = _mm_packus_epi16(sum, zero);
    230 
    231         // Extract low int and store.
    232         *colors++ = _mm_cvtsi128_si32(sum);
    233     } while (--count > 0);
    234 }
    235 
    236 static inline uint32_t ClampX_ClampY_pack_filter(SkFixed f, unsigned max,
    237                                                  SkFixed one) {
    238     unsigned i = SkClampMax(f >> 16, max);
    239     i = (i << 4) | ((f >> 12) & 0xF);
    240     return (i << 14) | SkClampMax((f + one) >> 16, max);
    241 }
    242 
    243 /*  SSE version of ClampX_ClampY_filter_scale()
    244  *  portable version is in core/SkBitmapProcState_matrix.h
    245  */
    246 void ClampX_ClampY_filter_scale_SSE2(const SkBitmapProcState& s, uint32_t xy[],
    247                                      int count, int x, int y) {
    248     SkASSERT((s.fInvType & ~(SkMatrix::kTranslate_Mask |
    249                              SkMatrix::kScale_Mask)) == 0);
    250     SkASSERT(s.fInvKy == 0);
    251 
    252     const unsigned maxX = s.fBitmap->width() - 1;
    253     const SkFixed one = s.fFilterOneX;
    254     const SkFixed dx = s.fInvSx;
    255     SkFixed fx;
    256 
    257     SkPoint pt;
    258     s.fInvProc(s.fInvMatrix, SkIntToScalar(x) + SK_ScalarHalf,
    259                              SkIntToScalar(y) + SK_ScalarHalf, &pt);
    260     const SkFixed fy = SkScalarToFixed(pt.fY) - (s.fFilterOneY >> 1);
    261     const unsigned maxY = s.fBitmap->height() - 1;
    262     // compute our two Y values up front
    263     *xy++ = ClampX_ClampY_pack_filter(fy, maxY, s.fFilterOneY);
    264     // now initialize fx
    265     fx = SkScalarToFixed(pt.fX) - (one >> 1);
    266 
    267     // test if we don't need to apply the tile proc
    268     if (dx > 0 && (unsigned)(fx >> 16) <= maxX &&
    269         (unsigned)((fx + dx * (count - 1)) >> 16) < maxX) {
    270         if (count >= 4) {
    271             // SSE version of decal_filter_scale
    272             while ((size_t(xy) & 0x0F) != 0) {
    273                 SkASSERT((fx >> (16 + 14)) == 0);
    274                 *xy++ = (fx >> 12 << 14) | ((fx >> 16) + 1);
    275                 fx += dx;
    276                 count--;
    277             }
    278 
    279             __m128i wide_1    = _mm_set1_epi32(1);
    280             __m128i wide_dx4  = _mm_set1_epi32(dx * 4);
    281             __m128i wide_fx   = _mm_set_epi32(fx + dx * 3, fx + dx * 2,
    282                                               fx + dx, fx);
    283 
    284             while (count >= 4) {
    285                 __m128i wide_out;
    286 
    287                 wide_out = _mm_slli_epi32(_mm_srai_epi32(wide_fx, 12), 14);
    288                 wide_out = _mm_or_si128(wide_out, _mm_add_epi32(
    289                                         _mm_srai_epi32(wide_fx, 16), wide_1));
    290 
    291                 _mm_store_si128(reinterpret_cast<__m128i*>(xy), wide_out);
    292 
    293                 xy += 4;
    294                 fx += dx * 4;
    295                 wide_fx  = _mm_add_epi32(wide_fx, wide_dx4);
    296                 count -= 4;
    297             } // while count >= 4
    298         } // if count >= 4
    299 
    300         while (count-- > 0) {
    301             SkASSERT((fx >> (16 + 14)) == 0);
    302             *xy++ = (fx >> 12 << 14) | ((fx >> 16) + 1);
    303             fx += dx;
    304         }
    305     } else {
    306         // SSE2 only support 16bit interger max & min, so only process the case
    307         // maxX less than the max 16bit interger. Actually maxX is the bitmap's
    308         // height, there should be rare bitmap whose height will be greater
    309         // than max 16bit interger in the real world.
    310         if ((count >= 4) && (maxX <= 0xFFFF)) {
    311             while (((size_t)xy & 0x0F) != 0) {
    312                 *xy++ = ClampX_ClampY_pack_filter(fx, maxX, one);
    313                 fx += dx;
    314                 count--;
    315             }
    316 
    317             __m128i wide_fx   = _mm_set_epi32(fx + dx * 3, fx + dx * 2,
    318                                               fx + dx, fx);
    319             __m128i wide_dx4  = _mm_set1_epi32(dx * 4);
    320             __m128i wide_one  = _mm_set1_epi32(one);
    321             __m128i wide_maxX = _mm_set1_epi32(maxX);
    322             __m128i wide_mask = _mm_set1_epi32(0xF);
    323 
    324              while (count >= 4) {
    325                 __m128i wide_i;
    326                 __m128i wide_lo;
    327                 __m128i wide_fx1;
    328 
    329                 // i = SkClampMax(f>>16,maxX)
    330                 wide_i = _mm_max_epi16(_mm_srli_epi32(wide_fx, 16),
    331                                        _mm_setzero_si128());
    332                 wide_i = _mm_min_epi16(wide_i, wide_maxX);
    333 
    334                 // i<<4 | TILEX_LOW_BITS(fx)
    335                 wide_lo = _mm_srli_epi32(wide_fx, 12);
    336                 wide_lo = _mm_and_si128(wide_lo, wide_mask);
    337                 wide_i  = _mm_slli_epi32(wide_i, 4);
    338                 wide_i  = _mm_or_si128(wide_i, wide_lo);
    339 
    340                 // i<<14
    341                 wide_i = _mm_slli_epi32(wide_i, 14);
    342 
    343                 // SkClampMax(((f+one))>>16,max)
    344                 wide_fx1 = _mm_add_epi32(wide_fx, wide_one);
    345                 wide_fx1 = _mm_max_epi16(_mm_srli_epi32(wide_fx1, 16),
    346                                                         _mm_setzero_si128());
    347                 wide_fx1 = _mm_min_epi16(wide_fx1, wide_maxX);
    348 
    349                 // final combination
    350                 wide_i = _mm_or_si128(wide_i, wide_fx1);
    351                 _mm_store_si128(reinterpret_cast<__m128i*>(xy), wide_i);
    352 
    353                 wide_fx = _mm_add_epi32(wide_fx, wide_dx4);
    354                 fx += dx * 4;
    355                 xy += 4;
    356                 count -= 4;
    357             } // while count >= 4
    358         } // if count >= 4
    359 
    360         while (count-- > 0) {
    361             *xy++ = ClampX_ClampY_pack_filter(fx, maxX, one);
    362             fx += dx;
    363         }
    364     }
    365 }
    366 
    367 /*  SSE version of ClampX_ClampY_nofilter_scale()
    368  *  portable version is in core/SkBitmapProcState_matrix.h
    369  */
    370 void ClampX_ClampY_nofilter_scale_SSE2(const SkBitmapProcState& s,
    371                                     uint32_t xy[], int count, int x, int y) {
    372     SkASSERT((s.fInvType & ~(SkMatrix::kTranslate_Mask |
    373                              SkMatrix::kScale_Mask)) == 0);
    374 
    375     // we store y, x, x, x, x, x
    376     const unsigned maxX = s.fBitmap->width() - 1;
    377     SkFixed fx;
    378     SkPoint pt;
    379     s.fInvProc(s.fInvMatrix, SkIntToScalar(x) + SK_ScalarHalf,
    380                              SkIntToScalar(y) + SK_ScalarHalf, &pt);
    381     fx = SkScalarToFixed(pt.fY);
    382     const unsigned maxY = s.fBitmap->height() - 1;
    383     *xy++ = SkClampMax(fx >> 16, maxY);
    384     fx = SkScalarToFixed(pt.fX);
    385 
    386     if (0 == maxX) {
    387         // all of the following X values must be 0
    388         memset(xy, 0, count * sizeof(uint16_t));
    389         return;
    390     }
    391 
    392     const SkFixed dx = s.fInvSx;
    393 
    394     // test if we don't need to apply the tile proc
    395     if ((unsigned)(fx >> 16) <= maxX &&
    396         (unsigned)((fx + dx * (count - 1)) >> 16) <= maxX) {
    397         // SSE version of decal_nofilter_scale
    398         if (count >= 8) {
    399             while (((size_t)xy & 0x0F) != 0) {
    400                 *xy++ = pack_two_shorts(fx >> 16, (fx + dx) >> 16);
    401                 fx += 2 * dx;
    402                 count -= 2;
    403             }
    404 
    405             __m128i wide_dx4 = _mm_set1_epi32(dx * 4);
    406             __m128i wide_dx8 = _mm_add_epi32(wide_dx4, wide_dx4);
    407 
    408             __m128i wide_low = _mm_set_epi32(fx + dx * 3, fx + dx * 2,
    409                                              fx + dx, fx);
    410             __m128i wide_high = _mm_add_epi32(wide_low, wide_dx4);
    411 
    412             while (count >= 8) {
    413                 __m128i wide_out_low = _mm_srli_epi32(wide_low, 16);
    414                 __m128i wide_out_high = _mm_srli_epi32(wide_high, 16);
    415 
    416                 __m128i wide_result = _mm_packs_epi32(wide_out_low,
    417                                                       wide_out_high);
    418                 _mm_store_si128(reinterpret_cast<__m128i*>(xy), wide_result);
    419 
    420                 wide_low = _mm_add_epi32(wide_low, wide_dx8);
    421                 wide_high = _mm_add_epi32(wide_high, wide_dx8);
    422 
    423                 xy += 4;
    424                 fx += dx * 8;
    425                 count -= 8;
    426             }
    427         } // if count >= 8
    428 
    429         uint16_t* xx = reinterpret_cast<uint16_t*>(xy);
    430         while (count-- > 0) {
    431             *xx++ = SkToU16(fx >> 16);
    432             fx += dx;
    433         }
    434     } else {
    435         // SSE2 only support 16bit interger max & min, so only process the case
    436         // maxX less than the max 16bit interger. Actually maxX is the bitmap's
    437         // height, there should be rare bitmap whose height will be greater
    438         // than max 16bit interger in the real world.
    439         if ((count >= 8) && (maxX <= 0xFFFF)) {
    440             while (((size_t)xy & 0x0F) != 0) {
    441                 *xy++ = pack_two_shorts(SkClampMax((fx + dx) >> 16, maxX),
    442                                         SkClampMax(fx >> 16, maxX));
    443                 fx += 2 * dx;
    444                 count -= 2;
    445             }
    446 
    447             __m128i wide_dx4 = _mm_set1_epi32(dx * 4);
    448             __m128i wide_dx8 = _mm_add_epi32(wide_dx4, wide_dx4);
    449 
    450             __m128i wide_low = _mm_set_epi32(fx + dx * 3, fx + dx * 2,
    451                                              fx + dx, fx);
    452             __m128i wide_high = _mm_add_epi32(wide_low, wide_dx4);
    453             __m128i wide_maxX = _mm_set1_epi32(maxX);
    454 
    455             while (count >= 8) {
    456                 __m128i wide_out_low = _mm_srli_epi32(wide_low, 16);
    457                 __m128i wide_out_high = _mm_srli_epi32(wide_high, 16);
    458 
    459                 wide_out_low  = _mm_max_epi16(wide_out_low,
    460                                               _mm_setzero_si128());
    461                 wide_out_low  = _mm_min_epi16(wide_out_low, wide_maxX);
    462                 wide_out_high = _mm_max_epi16(wide_out_high,
    463                                               _mm_setzero_si128());
    464                 wide_out_high = _mm_min_epi16(wide_out_high, wide_maxX);
    465 
    466                 __m128i wide_result = _mm_packs_epi32(wide_out_low,
    467                                                       wide_out_high);
    468                 _mm_store_si128(reinterpret_cast<__m128i*>(xy), wide_result);
    469 
    470                 wide_low  = _mm_add_epi32(wide_low, wide_dx8);
    471                 wide_high = _mm_add_epi32(wide_high, wide_dx8);
    472 
    473                 xy += 4;
    474                 fx += dx * 8;
    475                 count -= 8;
    476             }
    477         } // if count >= 8
    478 
    479         uint16_t* xx = reinterpret_cast<uint16_t*>(xy);
    480         while (count-- > 0) {
    481             *xx++ = SkClampMax(fx >> 16, maxX);
    482             fx += dx;
    483         }
    484     }
    485 }
    486 
    487 /*  SSE version of ClampX_ClampY_filter_affine()
    488  *  portable version is in core/SkBitmapProcState_matrix.h
    489  */
    490 void ClampX_ClampY_filter_affine_SSE2(const SkBitmapProcState& s,
    491                                       uint32_t xy[], int count, int x, int y) {
    492     SkPoint srcPt;
    493     s.fInvProc(s.fInvMatrix,
    494                SkIntToScalar(x) + SK_ScalarHalf,
    495                SkIntToScalar(y) + SK_ScalarHalf, &srcPt);
    496 
    497     SkFixed oneX = s.fFilterOneX;
    498     SkFixed oneY = s.fFilterOneY;
    499     SkFixed fx = SkScalarToFixed(srcPt.fX) - (oneX >> 1);
    500     SkFixed fy = SkScalarToFixed(srcPt.fY) - (oneY >> 1);
    501     SkFixed dx = s.fInvSx;
    502     SkFixed dy = s.fInvKy;
    503     unsigned maxX = s.fBitmap->width() - 1;
    504     unsigned maxY = s.fBitmap->height() - 1;
    505 
    506     if (count >= 2 && (maxX <= 0xFFFF)) {
    507         SkFixed dx2 = dx + dx;
    508         SkFixed dy2 = dy + dy;
    509 
    510         __m128i wide_f = _mm_set_epi32(fx + dx, fy + dy, fx, fy);
    511         __m128i wide_d2  = _mm_set_epi32(dx2, dy2, dx2, dy2);
    512         __m128i wide_one  = _mm_set_epi32(oneX, oneY, oneX, oneY);
    513         __m128i wide_max = _mm_set_epi32(maxX, maxY, maxX, maxY);
    514         __m128i wide_mask = _mm_set1_epi32(0xF);
    515 
    516         while (count >= 2) {
    517             // i = SkClampMax(f>>16,maxX)
    518             __m128i wide_i = _mm_max_epi16(_mm_srli_epi32(wide_f, 16),
    519                                            _mm_setzero_si128());
    520             wide_i = _mm_min_epi16(wide_i, wide_max);
    521 
    522             // i<<4 | TILEX_LOW_BITS(f)
    523             __m128i wide_lo = _mm_srli_epi32(wide_f, 12);
    524             wide_lo = _mm_and_si128(wide_lo, wide_mask);
    525             wide_i  = _mm_slli_epi32(wide_i, 4);
    526             wide_i  = _mm_or_si128(wide_i, wide_lo);
    527 
    528             // i<<14
    529             wide_i = _mm_slli_epi32(wide_i, 14);
    530 
    531             // SkClampMax(((f+one))>>16,max)
    532             __m128i wide_f1 = _mm_add_epi32(wide_f, wide_one);
    533             wide_f1 = _mm_max_epi16(_mm_srli_epi32(wide_f1, 16),
    534                                                    _mm_setzero_si128());
    535             wide_f1 = _mm_min_epi16(wide_f1, wide_max);
    536 
    537             // final combination
    538             wide_i = _mm_or_si128(wide_i, wide_f1);
    539             _mm_storeu_si128(reinterpret_cast<__m128i*>(xy), wide_i);
    540 
    541             wide_f = _mm_add_epi32(wide_f, wide_d2);
    542 
    543             fx += dx2;
    544             fy += dy2;
    545             xy += 4;
    546             count -= 2;
    547         } // while count >= 2
    548     } // if count >= 2
    549 
    550     while (count-- > 0) {
    551         *xy++ = ClampX_ClampY_pack_filter(fy, maxY, oneY);
    552         fy += dy;
    553         *xy++ = ClampX_ClampY_pack_filter(fx, maxX, oneX);
    554         fx += dx;
    555     }
    556 }
    557 
    558 /*  SSE version of ClampX_ClampY_nofilter_affine()
    559  *  portable version is in core/SkBitmapProcState_matrix.h
    560  */
    561 void ClampX_ClampY_nofilter_affine_SSE2(const SkBitmapProcState& s,
    562                                       uint32_t xy[], int count, int x, int y) {
    563     SkASSERT(s.fInvType & SkMatrix::kAffine_Mask);
    564     SkASSERT((s.fInvType & ~(SkMatrix::kTranslate_Mask |
    565                              SkMatrix::kScale_Mask |
    566                              SkMatrix::kAffine_Mask)) == 0);
    567 
    568     SkPoint srcPt;
    569     s.fInvProc(s.fInvMatrix,
    570                SkIntToScalar(x) + SK_ScalarHalf,
    571                SkIntToScalar(y) + SK_ScalarHalf, &srcPt);
    572 
    573     SkFixed fx = SkScalarToFixed(srcPt.fX);
    574     SkFixed fy = SkScalarToFixed(srcPt.fY);
    575     SkFixed dx = s.fInvSx;
    576     SkFixed dy = s.fInvKy;
    577     int maxX = s.fBitmap->width() - 1;
    578     int maxY = s.fBitmap->height() - 1;
    579 
    580     if (count >= 4 && (maxX <= 0xFFFF)) {
    581         while (((size_t)xy & 0x0F) != 0) {
    582             *xy++ = (SkClampMax(fy >> 16, maxY) << 16) |
    583                                   SkClampMax(fx >> 16, maxX);
    584             fx += dx;
    585             fy += dy;
    586             count--;
    587         }
    588 
    589         SkFixed dx4 = dx * 4;
    590         SkFixed dy4 = dy * 4;
    591 
    592         __m128i wide_fx   = _mm_set_epi32(fx + dx * 3, fx + dx * 2,
    593                                           fx + dx, fx);
    594         __m128i wide_fy   = _mm_set_epi32(fy + dy * 3, fy + dy * 2,
    595                                           fy + dy, fy);
    596         __m128i wide_dx4  = _mm_set1_epi32(dx4);
    597         __m128i wide_dy4  = _mm_set1_epi32(dy4);
    598 
    599         __m128i wide_maxX = _mm_set1_epi32(maxX);
    600         __m128i wide_maxY = _mm_set1_epi32(maxY);
    601 
    602         while (count >= 4) {
    603             // SkClampMax(fx>>16,maxX)
    604             __m128i wide_lo = _mm_max_epi16(_mm_srli_epi32(wide_fx, 16),
    605                                             _mm_setzero_si128());
    606             wide_lo = _mm_min_epi16(wide_lo, wide_maxX);
    607 
    608             // SkClampMax(fy>>16,maxY)
    609             __m128i wide_hi = _mm_max_epi16(_mm_srli_epi32(wide_fy, 16),
    610                                             _mm_setzero_si128());
    611             wide_hi = _mm_min_epi16(wide_hi, wide_maxY);
    612 
    613             // final combination
    614             __m128i wide_i = _mm_or_si128(_mm_slli_epi32(wide_hi, 16),
    615                                           wide_lo);
    616             _mm_store_si128(reinterpret_cast<__m128i*>(xy), wide_i);
    617 
    618             wide_fx = _mm_add_epi32(wide_fx, wide_dx4);
    619             wide_fy = _mm_add_epi32(wide_fy, wide_dy4);
    620 
    621             fx += dx4;
    622             fy += dy4;
    623             xy += 4;
    624             count -= 4;
    625         } // while count >= 4
    626     } // if count >= 4
    627 
    628     while (count-- > 0) {
    629         *xy++ = (SkClampMax(fy >> 16, maxY) << 16) |
    630                               SkClampMax(fx >> 16, maxX);
    631         fx += dx;
    632         fy += dy;
    633     }
    634 }
    635 
    636 /*  SSE version of S32_D16_filter_DX_SSE2
    637  *  Definition is in section of "D16 functions for SRC == 8888" in SkBitmapProcState.cpp
    638  *  It combines S32_opaque_D32_filter_DX_SSE2 and SkPixel32ToPixel16
    639  */
    640 void S32_D16_filter_DX_SSE2(const SkBitmapProcState& s,
    641                             const uint32_t* xy,
    642                             int count, uint16_t* colors) {
    643     SkASSERT(count > 0 && colors != NULL);
    644     SkASSERT(s.fFilterLevel != kNone_SkFilterQuality);
    645     SkASSERT(kN32_SkColorType == s.fBitmap->colorType());
    646     SkASSERT(s.fBitmap->isOpaque());
    647 
    648     SkPMColor dstColor;
    649     const char* srcAddr = static_cast<const char*>(s.fBitmap->getPixels());
    650     size_t rb = s.fBitmap->rowBytes();
    651     uint32_t XY = *xy++;
    652     unsigned y0 = XY >> 14;
    653     const uint32_t* row0 = reinterpret_cast<const uint32_t*>(srcAddr + (y0 >> 4) * rb);
    654     const uint32_t* row1 = reinterpret_cast<const uint32_t*>(srcAddr + (XY & 0x3FFF) * rb);
    655     unsigned subY = y0 & 0xF;
    656 
    657     // ( 0,  0,  0,  0,  0,  0,  0, 16)
    658     __m128i sixteen = _mm_cvtsi32_si128(16);
    659 
    660     // ( 0,  0,  0,  0, 16, 16, 16, 16)
    661     sixteen = _mm_shufflelo_epi16(sixteen, 0);
    662 
    663     // ( 0,  0,  0,  0,  0,  0,  0,  y)
    664     __m128i allY = _mm_cvtsi32_si128(subY);
    665 
    666     // ( 0,  0,  0,  0,  y,  y,  y,  y)
    667     allY = _mm_shufflelo_epi16(allY, 0);
    668 
    669     // ( 0,  0,  0,  0, 16-y, 16-y, 16-y, 16-y)
    670     __m128i negY = _mm_sub_epi16(sixteen, allY);
    671 
    672     // (16-y, 16-y, 16-y, 16-y, y, y, y, y)
    673     allY = _mm_unpacklo_epi64(allY, negY);
    674 
    675     // (16, 16, 16, 16, 16, 16, 16, 16 )
    676     sixteen = _mm_shuffle_epi32(sixteen, 0);
    677 
    678     // ( 0,  0,  0,  0,  0,  0,  0,  0)
    679     __m128i zero = _mm_setzero_si128();
    680 
    681     do {
    682         uint32_t XX = *xy++;    // x0:14 | 4 | x1:14
    683         unsigned x0 = XX >> 18;
    684         unsigned x1 = XX & 0x3FFF;
    685 
    686         // (0, 0, 0, 0, 0, 0, 0, x)
    687         __m128i allX = _mm_cvtsi32_si128((XX >> 14) & 0x0F);
    688 
    689         // (0, 0, 0, 0, x, x, x, x)
    690         allX = _mm_shufflelo_epi16(allX, 0);
    691 
    692         // (x, x, x, x, x, x, x, x)
    693         allX = _mm_shuffle_epi32(allX, 0);
    694 
    695         // (16-x, 16-x, 16-x, 16-x, 16-x, 16-x, 16-x)
    696         __m128i negX = _mm_sub_epi16(sixteen, allX);
    697 
    698         // Load 4 samples (pixels).
    699         __m128i a00 = _mm_cvtsi32_si128(row0[x0]);
    700         __m128i a01 = _mm_cvtsi32_si128(row0[x1]);
    701         __m128i a10 = _mm_cvtsi32_si128(row1[x0]);
    702         __m128i a11 = _mm_cvtsi32_si128(row1[x1]);
    703 
    704         // (0, 0, a00, a10)
    705         __m128i a00a10 = _mm_unpacklo_epi32(a10, a00);
    706 
    707         // Expand to 16 bits per component.
    708         a00a10 = _mm_unpacklo_epi8(a00a10, zero);
    709 
    710         // ((a00 * (16-y)), (a10 * y)).
    711         a00a10 = _mm_mullo_epi16(a00a10, allY);
    712 
    713         // (a00 * (16-y) * (16-x), a10 * y * (16-x)).
    714         a00a10 = _mm_mullo_epi16(a00a10, negX);
    715 
    716         // (0, 0, a01, a10)
    717         __m128i a01a11 = _mm_unpacklo_epi32(a11, a01);
    718 
    719         // Expand to 16 bits per component.
    720         a01a11 = _mm_unpacklo_epi8(a01a11, zero);
    721 
    722         // (a01 * (16-y)), (a11 * y)
    723         a01a11 = _mm_mullo_epi16(a01a11, allY);
    724 
    725         // (a01 * (16-y) * x), (a11 * y * x)
    726         a01a11 = _mm_mullo_epi16(a01a11, allX);
    727 
    728         // (a00*w00 + a01*w01, a10*w10 + a11*w11)
    729         __m128i sum = _mm_add_epi16(a00a10, a01a11);
    730 
    731         // (DC, a00*w00 + a01*w01)
    732         __m128i shifted = _mm_shuffle_epi32(sum, 0xEE);
    733 
    734         // (DC, a00*w00 + a01*w01 + a10*w10 + a11*w11)
    735         sum = _mm_add_epi16(sum, shifted);
    736 
    737         // Divide each 16 bit component by 256.
    738         sum = _mm_srli_epi16(sum, 8);
    739 
    740         // Pack lower 4 16 bit values of sum into lower 4 bytes.
    741         sum = _mm_packus_epi16(sum, zero);
    742 
    743         // Extract low int and store.
    744         dstColor = _mm_cvtsi128_si32(sum);
    745 
    746         *colors++ = SkPixel32ToPixel16(dstColor);
    747     } while (--count > 0);
    748 }
    749