1 /* 2 * Copyright 2015 Google Inc. 3 * 4 * Use of this source code is governed by a BSD-style license that can be 5 * found in the LICENSE file. 6 */ 7 8 #ifndef SkBlurImageFilter_opts_DEFINED 9 #define SkBlurImageFilter_opts_DEFINED 10 11 #include "SkColorPriv.h" 12 #include "SkRect.h" 13 14 #if SK_CPU_SSE_LEVEL >= SK_CPU_SSE_LEVEL_SSE2 15 #include <immintrin.h> 16 #endif 17 18 namespace SK_OPTS_NS { 19 20 enum class BlurDirection { kX, kY }; 21 22 #if SK_CPU_SSE_LEVEL >= SK_CPU_SSE_LEVEL_SSE2 23 #if SK_CPU_SSE_LEVEL >= SK_CPU_SSE_LEVEL_SSE41 24 // RGBA -> R000 G000 B000 A000 25 static inline __m128i expand(SkPMColor p) { 26 return _mm_cvtepu8_epi32(_mm_cvtsi32_si128(p)); 27 }; 28 // 000R 000G 000B 000A -> RGBA 29 static inline SkPMColor repack(__m128i p) { 30 const char _ = ~0; // Don't care what ends up in these bytes. This zeros them. 31 p = _mm_shuffle_epi8(p, _mm_setr_epi8(3,7,11,15, _,_,_,_, _,_,_,_, _,_,_,_)); 32 return _mm_cvtsi128_si32(p); 33 }; 34 #define mullo_epi32 _mm_mullo_epi32 35 36 #else 37 static inline __m128i expand(int p) { 38 auto result = _mm_cvtsi32_si128(p); 39 result = _mm_unpacklo_epi8 (result, _mm_setzero_si128()); 40 result = _mm_unpacklo_epi16(result, _mm_setzero_si128()); 41 return result; 42 }; 43 static inline SkPMColor repack(__m128i p) { 44 p = _mm_srli_epi32(p, 24); // R000 G000 B000 A000 45 p = _mm_packs_epi32(p, p); // R0G0 B0A0 xxxx xxxx 46 p = _mm_packus_epi16(p, p); // RGBA xxxx xxxx xxxx 47 return _mm_cvtsi128_si32(p); 48 }; 49 50 // _mm_mullo_epi32 is not available, so use the standard trick to emulate it. 51 static inline __m128i mullo_epi32(__m128i a, __m128i b) { 52 __m128i p02 = _mm_mul_epu32(a, b), 53 p13 = _mm_mul_epu32(_mm_srli_si128(a, 4), 54 _mm_srli_si128(b, 4)); 55 return _mm_unpacklo_epi32(_mm_shuffle_epi32(p02, _MM_SHUFFLE(0,0,2,0)), 56 _mm_shuffle_epi32(p13, _MM_SHUFFLE(0,0,2,0))); 57 }; 58 #endif 59 60 #define INIT_SCALE const __m128i scale = _mm_set1_epi32((1 << 24) / kernelSize); 61 #define INIT_HALF const __m128i half = _mm_set1_epi32(1 << 23); 62 #define INIT_SUMS __m128i sum = _mm_setzero_si128(); 63 #define INCREMENT_SUMS(c) sum = _mm_add_epi32(sum, expand(c)) 64 #define DECREMENT_SUMS(c) sum = _mm_sub_epi32(sum, expand(c)) 65 #define STORE_SUMS *dptr = repack(_mm_add_epi32(mullo_epi32(sum, scale), half)); 66 #define DOUBLE_ROW_OPTIMIZATION /*none*/ 67 68 #elif defined(SK_ARM_HAS_NEON) 69 70 // val = (sum * scale * 2 + 0x8000) >> 16 71 #define STORE_SUMS_DOUBLE \ 72 uint16x8_t resultPixels = vreinterpretq_u16_s16(vqrdmulhq_s16( \ 73 vreinterpretq_s16_u16(sum), vreinterpretq_s16_u16(scale))); \ 74 if (dstDirection == BlurDirection::kX) { \ 75 uint32x2_t px2 = vreinterpret_u32_u8(vmovn_u16(resultPixels)); \ 76 vst1_lane_u32(dptr + 0, px2, 0); \ 77 vst1_lane_u32(dptr + width, px2, 1); \ 78 } else { \ 79 vst1_u8((uint8_t*)dptr, vmovn_u16(resultPixels)); \ 80 } 81 82 #define INCREMENT_SUMS_DOUBLE(p) sum = vaddw_u8(sum, load_2_pixels(p)) 83 #define DECREMENT_SUMS_DOUBLE(p) sum = vsubw_u8(sum, load_2_pixels(p)) 84 85 // Fast path for kernel sizes between 2 and 127, working on two rows at a time. 86 template<BlurDirection srcDirection, BlurDirection dstDirection> 87 static int box_blur_double(const SkPMColor** src, int srcStride, const SkIRect& srcBounds, 88 SkPMColor** dst, int kernelSize, 89 int leftOffset, int rightOffset, int width, int height) { 90 // Load 2 pixels from adjacent rows. 91 auto load_2_pixels = [&](const SkPMColor* s) { 92 if (srcDirection == BlurDirection::kX) { 93 // 10% faster by adding these 2 prefetches 94 SK_PREFETCH(s + 16); 95 SK_PREFETCH(s + 16 + srcStride); 96 auto one = vld1_lane_u32(s + 0, vdup_n_u32(0), 0), 97 two = vld1_lane_u32(s + srcStride, one, 1); 98 return vreinterpret_u8_u32(two); 99 } else { 100 return vld1_u8((uint8_t*)s); 101 } 102 }; 103 int left = srcBounds.left(); 104 int right = srcBounds.right(); 105 int top = srcBounds.top(); 106 int bottom = srcBounds.bottom(); 107 int incrementStart = SkMax32(left - rightOffset - 1, left - right); 108 int incrementEnd = SkMax32(right - rightOffset - 1, 0); 109 int decrementStart = SkMin32(left + leftOffset, width); 110 int decrementEnd = SkMin32(right + leftOffset, width); 111 const int srcStrideX = srcDirection == BlurDirection::kX ? 1 : srcStride; 112 const int dstStrideX = dstDirection == BlurDirection::kX ? 1 : height; 113 const int srcStrideY = srcDirection == BlurDirection::kX ? srcStride : 1; 114 const int dstStrideY = dstDirection == BlurDirection::kX ? width : 1; 115 const uint16x8_t scale = vdupq_n_u16((1 << 15) / kernelSize); 116 117 for (; bottom - top >= 2; top += 2) { 118 uint16x8_t sum = vdupq_n_u16(0); 119 const SkPMColor* lptr = *src; 120 const SkPMColor* rptr = *src; 121 SkPMColor* dptr = *dst; 122 int x; 123 for (x = incrementStart; x < 0; ++x) { 124 INCREMENT_SUMS_DOUBLE(rptr); 125 rptr += srcStrideX; 126 } 127 // Clear to zero when sampling to the left our domain. "sum" is zero here because we 128 // initialized it above, and the preceeding loop has no effect in this case. 129 for (x = 0; x < incrementStart; ++x) { 130 STORE_SUMS_DOUBLE 131 dptr += dstStrideX; 132 } 133 for (; x < decrementStart && x < incrementEnd; ++x) { 134 STORE_SUMS_DOUBLE 135 dptr += dstStrideX; 136 INCREMENT_SUMS_DOUBLE(rptr); 137 rptr += srcStrideX; 138 } 139 for (x = decrementStart; x < incrementEnd; ++x) { 140 STORE_SUMS_DOUBLE 141 dptr += dstStrideX; 142 INCREMENT_SUMS_DOUBLE(rptr); 143 rptr += srcStrideX; 144 DECREMENT_SUMS_DOUBLE(lptr); 145 lptr += srcStrideX; 146 } 147 for (x = incrementEnd; x < decrementStart; ++x) { 148 STORE_SUMS_DOUBLE 149 dptr += dstStrideX; 150 } 151 for (; x < decrementEnd; ++x) { 152 STORE_SUMS_DOUBLE 153 dptr += dstStrideX; 154 DECREMENT_SUMS_DOUBLE(lptr); 155 lptr += srcStrideX; 156 } 157 // Clear to zero when sampling to the right of our domain. "sum" is 158 // zero here because we added on then subtracted off all of the pixels, leaving zero. 159 for (; x < width; ++x) { 160 STORE_SUMS_DOUBLE 161 dptr += dstStrideX; 162 } 163 *src += srcStrideY * 2; 164 *dst += dstStrideY * 2; 165 } 166 return top; 167 } 168 169 // RGBA -> R0G0 B0A0 170 static inline uint16x4_t expand(SkPMColor p) { 171 return vget_low_u16(vmovl_u8(vreinterpret_u8_u32(vdup_n_u32(p)))); 172 }; 173 174 #define INIT_SCALE const uint32x4_t scale = vdupq_n_u32((1 << 24) / kernelSize); 175 #define INIT_HALF const uint32x4_t half = vdupq_n_u32(1 << 23); 176 #define INIT_SUMS uint32x4_t sum = vdupq_n_u32(0); 177 #define INCREMENT_SUMS(c) sum = vaddw_u16(sum, expand(c)); 178 #define DECREMENT_SUMS(c) sum = vsubw_u16(sum, expand(c)); 179 180 #define STORE_SUMS \ 181 uint32x4_t result = vmlaq_u32(half, sum, scale); \ 182 uint16x4_t result16 = vqshrn_n_u32(result, 16); \ 183 uint8x8_t result8 = vqshrn_n_u16(vcombine_u16(result16, result16), 8); \ 184 vst1_lane_u32(dptr, vreinterpret_u32_u8(result8), 0); 185 186 #define DOUBLE_ROW_OPTIMIZATION \ 187 if (1 < kernelSize && kernelSize < 128) { \ 188 top = box_blur_double<srcDirection, dstDirection>(&src, srcStride, srcBounds, &dst, \ 189 kernelSize, \ 190 leftOffset, rightOffset, \ 191 width, height); \ 192 } 193 194 #else // Neither NEON nor >=SSE2. 195 196 #define INIT_SCALE uint32_t scale = (1 << 24) / kernelSize; 197 #define INIT_HALF uint32_t half = 1 << 23; 198 #define INIT_SUMS int sumA = 0, sumR = 0, sumG = 0, sumB = 0; 199 #define INCREMENT_SUMS(c) \ 200 sumA += SkGetPackedA32(c); \ 201 sumR += SkGetPackedR32(c); \ 202 sumG += SkGetPackedG32(c); \ 203 sumB += SkGetPackedB32(c) 204 #define DECREMENT_SUMS(c) \ 205 sumA -= SkGetPackedA32(c); \ 206 sumR -= SkGetPackedR32(c); \ 207 sumG -= SkGetPackedG32(c); \ 208 sumB -= SkGetPackedB32(c) 209 #define STORE_SUMS \ 210 *dptr = SkPackARGB32((sumA * scale + half) >> 24, \ 211 (sumR * scale + half) >> 24, \ 212 (sumG * scale + half) >> 24, \ 213 (sumB * scale + half) >> 24); 214 #define DOUBLE_ROW_OPTIMIZATION 215 216 #endif 217 218 template<BlurDirection srcDirection, BlurDirection dstDirection> 219 static void box_blur(const SkPMColor* src, int srcStride, const SkIRect& srcBounds, SkPMColor* dst, 220 int kernelSize, int leftOffset, int rightOffset, int width, int height) { 221 int left = srcBounds.left(); 222 int right = srcBounds.right(); 223 int top = srcBounds.top(); 224 int bottom = srcBounds.bottom(); 225 int incrementStart = SkMax32(left - rightOffset - 1, left - right); 226 int incrementEnd = SkMax32(right - rightOffset - 1, 0); 227 int decrementStart = SkMin32(left + leftOffset, width); 228 int decrementEnd = SkMin32(right + leftOffset, width); 229 int srcStrideX = srcDirection == BlurDirection::kX ? 1 : srcStride; 230 int dstStrideX = dstDirection == BlurDirection::kX ? 1 : height; 231 int srcStrideY = srcDirection == BlurDirection::kX ? srcStride : 1; 232 int dstStrideY = dstDirection == BlurDirection::kX ? width : 1; 233 INIT_SCALE 234 INIT_HALF 235 236 // Clear to zero when sampling above our domain. 237 for (int y = 0; y < top; y++) { 238 SkColor* dptr = dst; 239 for (int x = 0; x < width; ++x) { 240 *dptr = 0; 241 dptr += dstStrideX; 242 } 243 dst += dstStrideY; 244 } 245 246 DOUBLE_ROW_OPTIMIZATION 247 248 for (int y = top; y < bottom; ++y) { 249 INIT_SUMS 250 const SkPMColor* lptr = src; 251 const SkPMColor* rptr = src; 252 SkColor* dptr = dst; 253 int x; 254 for (x = incrementStart; x < 0; ++x) { 255 INCREMENT_SUMS(*rptr); 256 rptr += srcStrideX; 257 if (srcDirection == BlurDirection::kY) { 258 SK_PREFETCH(rptr); 259 } 260 } 261 // Clear to zero when sampling to the left of our domain. 262 for (x = 0; x < incrementStart; ++x) { 263 *dptr = 0; 264 dptr += dstStrideX; 265 } 266 for (; x < decrementStart && x < incrementEnd; ++x) { 267 STORE_SUMS 268 dptr += dstStrideX; 269 INCREMENT_SUMS(*rptr); 270 rptr += srcStrideX; 271 if (srcDirection == BlurDirection::kY) { 272 SK_PREFETCH(rptr); 273 } 274 } 275 for (x = decrementStart; x < incrementEnd; ++x) { 276 STORE_SUMS 277 dptr += dstStrideX; 278 INCREMENT_SUMS(*rptr); 279 rptr += srcStrideX; 280 if (srcDirection == BlurDirection::kY) { 281 SK_PREFETCH(rptr); 282 } 283 DECREMENT_SUMS(*lptr); 284 lptr += srcStrideX; 285 } 286 for (x = incrementEnd; x < decrementStart; ++x) { 287 STORE_SUMS 288 dptr += dstStrideX; 289 } 290 for (; x < decrementEnd; ++x) { 291 STORE_SUMS 292 dptr += dstStrideX; 293 DECREMENT_SUMS(*lptr); 294 lptr += srcStrideX; 295 } 296 // Clear to zero when sampling to the right of our domain. 297 for (; x < width; ++x) { 298 *dptr = 0; 299 dptr += dstStrideX; 300 } 301 src += srcStrideY; 302 dst += dstStrideY; 303 } 304 // Clear to zero when sampling below our domain. 305 for (int y = bottom; y < height; ++y) { 306 SkColor* dptr = dst; 307 for (int x = 0; x < width; ++x) { 308 *dptr = 0; 309 dptr += dstStrideX; 310 } 311 dst += dstStrideY; 312 } 313 } 314 315 static auto box_blur_xx = &box_blur<BlurDirection::kX, BlurDirection::kX>, 316 box_blur_xy = &box_blur<BlurDirection::kX, BlurDirection::kY>, 317 box_blur_yx = &box_blur<BlurDirection::kY, BlurDirection::kX>; 318 319 } // namespace SK_OPTS_NS 320 321 #endif 322
