1 /* NEON optimized code (C) COPYRIGHT 2009 Motorola 2 * 3 * Use of this source code is governed by a BSD-style license that can be 4 * found in the LICENSE file. 5 */ 6 7 /* 8 * Modifications done in-house at Motorola 9 * 10 * this is a clone of SkBitmapProcState_matrix.h 11 * and has been tuned to work with the NEON unit. 12 * 13 * Still going back and forth between whether this approach 14 * (clone the entire SkBitmapProcState_matrix.h file or 15 * if I should put just the modified routines in here and 16 * then use a construct like #define DONT_DO_THIS_FUNCTION or 17 * something like that... 18 * 19 * This is for the ClampX_ClampY instance 20 * 21 */ 22 23 24 #if !defined(__ARM_HAVE_NEON) 25 #error this file can be used only when the NEON unit is enabled 26 #endif 27 28 #include <arm_neon.h> 29 30 /* 31 * This has been modified on the knowledge that (at the time) 32 * we had the following macro definitions in the parent file 33 * 34 * #define MAKENAME(suffix) ClampX_ClampY ## suffix 35 * #define TILEX_PROCF(fx, max) SkClampMax((fx) >> 16, max) 36 * #define TILEY_PROCF(fy, max) SkClampMax((fy) >> 16, max) 37 * #define TILEX_LOW_BITS(fx, max) (((fx) >> 12) & 0xF) 38 * #define TILEY_LOW_BITS(fy, max) (((fy) >> 12) & 0xF) 39 * #define CHECK_FOR_DECAL 40 */ 41 42 /* SkClampMax(val,max) -- bound to 0..max */ 43 44 #define SCALE_NOFILTER_NAME MAKENAME(_nofilter_scale_neon) 45 #define SCALE_FILTER_NAME MAKENAME(_filter_scale_neon) 46 #define AFFINE_NOFILTER_NAME MAKENAME(_nofilter_affine_neon) 47 #define AFFINE_FILTER_NAME MAKENAME(_filter_affine_neon) 48 #define PERSP_NOFILTER_NAME MAKENAME(_nofilter_persp_neon) 49 #define PERSP_FILTER_NAME MAKENAME(_filter_persp_neon) 50 51 #define PACK_FILTER_X_NAME MAKENAME(_pack_filter_x) 52 #define PACK_FILTER_Y_NAME MAKENAME(_pack_filter_y) 53 54 #ifndef PREAMBLE 55 #define PREAMBLE(state) 56 #define PREAMBLE_PARAM_X 57 #define PREAMBLE_PARAM_Y 58 #define PREAMBLE_ARG_X 59 #define PREAMBLE_ARG_Y 60 #endif 61 62 static void SCALE_NOFILTER_NAME(const SkBitmapProcState& s, 63 uint32_t xy[], int count, int x, int y) { 64 SkASSERT((s.fInvType & ~(SkMatrix::kTranslate_Mask | 65 SkMatrix::kScale_Mask)) == 0); 66 67 PREAMBLE(s); 68 // we store y, x, x, x, x, x 69 70 const unsigned maxX = s.fBitmap->width() - 1; 71 SkFixed fx; 72 { 73 SkPoint pt; 74 s.fInvProc(*s.fInvMatrix, SkIntToScalar(x) + SK_ScalarHalf, 75 SkIntToScalar(y) + SK_ScalarHalf, &pt); 76 fx = SkScalarToFixed(pt.fY); 77 const unsigned maxY = s.fBitmap->height() - 1; 78 *xy++ = TILEY_PROCF(fx, maxY); 79 fx = SkScalarToFixed(pt.fX); 80 } 81 82 if (0 == maxX) { 83 // all of the following X values must be 0 84 memset(xy, 0, count * sizeof(uint16_t)); 85 return; 86 } 87 88 const SkFixed dx = s.fInvSx; 89 90 #ifdef CHECK_FOR_DECAL 91 // test if we don't need to apply the tile proc 92 if ((unsigned)(fx >> 16) <= maxX && 93 (unsigned)((fx + dx * (count - 1)) >> 16) <= maxX) { 94 decal_nofilter_scale(xy, fx, dx, count); 95 return; 96 } 97 #endif 98 99 int i; 100 101 /* very much like done in decal_nofilter, but with 102 * an extra clamping function applied. 103 * TILEX_PROCF(fx,max) SkClampMax((fx)>>16, max) 104 */ 105 if (count >= 8) { 106 /* SkFixed is 16.16 fixed point */ 107 SkFixed dx2 = dx+dx; 108 SkFixed dx4 = dx2+dx2; 109 SkFixed dx8 = dx4+dx4; 110 111 /* now build fx/fx+dx/fx+2dx/fx+3dx */ 112 SkFixed fx1, fx2, fx3; 113 int32x2_t lower, upper; 114 int32x4_t lbase, hbase; 115 int16_t *dst16 = (int16_t *)xy; 116 117 fx1 = fx+dx; 118 fx2 = fx1+dx; 119 fx3 = fx2+dx; 120 121 /* build my template(s) */ 122 /* avoid the 'lbase unitialized' warning */ 123 lbase = vdupq_n_s32(fx); 124 lbase = vsetq_lane_s32(fx1, lbase, 1); 125 lbase = vsetq_lane_s32(fx2, lbase, 2); 126 lbase = vsetq_lane_s32(fx3, lbase, 3); 127 128 hbase = vaddq_s32(lbase, vdupq_n_s32(dx4)); 129 130 /* store & bump */ 131 do { 132 int32x4_t lout; 133 int32x4_t hout; 134 int16x8_t hi16; 135 136 /* get the hi 16s of all those 32s */ 137 lout = lbase; 138 hout = hbase; 139 /* this sets up all lout's then all hout's in hout */ 140 asm ("vuzpq.16 %q0, %q1" : "+w" (lout), "+w" (hout)); 141 hi16 = vreinterpretq_s16_s32(hout); 142 143 /* clamp & output */ 144 hi16 = vmaxq_s16(hi16, vdupq_n_s16(0)); 145 hi16 = vminq_s16(hi16, vdupq_n_s16(maxX)); 146 vst1q_s16(dst16, hi16); 147 148 /* but preserving base & on to the next */ 149 lbase = vaddq_s32 (lbase, vdupq_n_s32(dx8)); 150 hbase = vaddq_s32 (hbase, vdupq_n_s32(dx8)); 151 dst16 += 8; 152 count -= 8; 153 fx += dx8; 154 } while (count >= 8); 155 xy = (uint32_t *) dst16; 156 } 157 158 uint16_t* xx = (uint16_t*)xy; 159 for (i = count; i > 0; --i) { 160 *xx++ = TILEX_PROCF(fx, maxX); fx += dx; 161 } 162 } 163 164 // note: we could special-case on a matrix which is skewed in X but not Y. 165 // this would require a more general setup thatn SCALE does, but could use 166 // SCALE's inner loop that only looks at dx 167 168 static void AFFINE_NOFILTER_NAME(const SkBitmapProcState& s, 169 uint32_t xy[], int count, int x, int y) { 170 SkASSERT(s.fInvType & SkMatrix::kAffine_Mask); 171 SkASSERT((s.fInvType & ~(SkMatrix::kTranslate_Mask | 172 SkMatrix::kScale_Mask | 173 SkMatrix::kAffine_Mask)) == 0); 174 175 PREAMBLE(s); 176 SkPoint srcPt; 177 s.fInvProc(*s.fInvMatrix, 178 SkIntToScalar(x) + SK_ScalarHalf, 179 SkIntToScalar(y) + SK_ScalarHalf, &srcPt); 180 181 SkFixed fx = SkScalarToFixed(srcPt.fX); 182 SkFixed fy = SkScalarToFixed(srcPt.fY); 183 SkFixed dx = s.fInvSx; 184 SkFixed dy = s.fInvKy; 185 int maxX = s.fBitmap->width() - 1; 186 int maxY = s.fBitmap->height() - 1; 187 188 /* NEON lets us do an 8x unrolling */ 189 if (count >= 8) { 190 /* SkFixed is 16.16 fixed point */ 191 SkFixed dx4 = dx * 4; 192 SkFixed dy4 = dy * 4; 193 SkFixed dx8 = dx * 8; 194 SkFixed dy8 = dy * 8; 195 196 int32x4_t xbase, ybase; 197 int32x4_t x2base, y2base; 198 int16_t *dst16 = (int16_t *) xy; 199 200 /* my sets of maxx/maxy for clamping */ 201 int32_t maxpair = (maxX&0xffff) | ((maxY&0xffff)<<16); 202 int16x8_t maxXY = vreinterpretq_s16_s32(vdupq_n_s32(maxpair)); 203 204 /* now build fx/fx+dx/fx+2dx/fx+3dx */ 205 /* avoid the 'xbase unitialized' warning...*/ 206 xbase = vdupq_n_s32(fx); 207 xbase = vsetq_lane_s32(fx+dx, xbase, 1); 208 xbase = vsetq_lane_s32(fx+dx+dx, xbase, 2); 209 xbase = vsetq_lane_s32(fx+dx+dx+dx, xbase, 3); 210 211 /* same for fy */ 212 /* avoid the 'ybase unitialized' warning...*/ 213 ybase = vdupq_n_s32(fy); 214 ybase = vsetq_lane_s32(fy+dy, ybase, 1); 215 ybase = vsetq_lane_s32(fy+dy+dy, ybase, 2); 216 ybase = vsetq_lane_s32(fy+dy+dy+dy, ybase, 3); 217 218 x2base = vaddq_s32(xbase, vdupq_n_s32(dx4)); 219 y2base = vaddq_s32(ybase, vdupq_n_s32(dy4)); 220 221 /* store & bump */ 222 do { 223 int32x4_t xout, yout; 224 int32x4_t x2out, y2out; 225 int16x8_t hi16, hi16_2; 226 227 xout = xbase; 228 yout = ybase; 229 230 /* overlay y's low16 with hi16 from x */ 231 /* so we properly shifted xyxyxyxy */ 232 yout = vsriq_n_s32(yout, xout, 16); 233 hi16 = vreinterpretq_s16_s32 (yout); 234 235 /* do the clamping; both guys get 0's */ 236 hi16 = vmaxq_s16 (hi16, vdupq_n_s16(0)); 237 hi16 = vminq_s16 (hi16, maxXY); 238 239 vst1q_s16 (dst16, hi16); 240 241 /* and for the other 4 pieces of this iteration */ 242 x2out = x2base; 243 y2out = y2base; 244 245 /* overlay y's low16 with hi16 from x */ 246 /* so we properly shifted xyxyxyxy */ 247 y2out = vsriq_n_s32(y2out, x2out, 16); 248 hi16_2 = vreinterpretq_s16_s32 (y2out); 249 250 /* do the clamping; both guys get 0's */ 251 hi16_2 = vmaxq_s16 (hi16_2, vdupq_n_s16(0)); 252 hi16_2 = vminq_s16 (hi16_2, maxXY); 253 254 /* RBE: gcc regenerates dst16+8 all the time instead 255 * of folding it into an addressing mode. *sigh* */ 256 vst1q_s16 (dst16+8, hi16_2); 257 258 /* moving base and on to the next */ 259 xbase = vaddq_s32 (xbase, vdupq_n_s32 (dx8)); 260 ybase = vaddq_s32 (ybase, vdupq_n_s32 (dy8)); 261 x2base = vaddq_s32 (x2base, vdupq_n_s32 (dx8)); 262 y2base = vaddq_s32 (y2base, vdupq_n_s32 (dy8)); 263 264 dst16 += 16; /* 8x32 aka 16x16 */ 265 count -= 8; 266 fx += dx8; 267 fy += dy8; 268 } while (count >= 8); 269 xy = (uint32_t *) dst16; 270 } 271 272 for (int i = count; i > 0; --i) { 273 *xy++ = (TILEY_PROCF(fy, maxY) << 16) | TILEX_PROCF(fx, maxX); 274 fx += dx; fy += dy; 275 } 276 } 277 278 #undef DEBUG_PERSP_NOFILTER 279 280 static void PERSP_NOFILTER_NAME(const SkBitmapProcState& s, 281 uint32_t* SK_RESTRICT xy, 282 int count, int x, int y) { 283 SkASSERT(s.fInvType & SkMatrix::kPerspective_Mask); 284 285 PREAMBLE(s); 286 /* max{X,Y} are int here, but later shown/assumed to fit in 16 bits */ 287 int maxX = s.fBitmap->width() - 1; 288 int maxY = s.fBitmap->height() - 1; 289 290 SkPerspIter iter(*s.fInvMatrix, 291 SkIntToScalar(x) + SK_ScalarHalf, 292 SkIntToScalar(y) + SK_ScalarHalf, count); 293 294 while ((count = iter.next()) != 0) { 295 const SkFixed* SK_RESTRICT srcXY = iter.getXY(); 296 297 #if defined(DEBUG_PERSP_NOFILTER) 298 /* debugging stuff */ 299 const SkFixed *end_srcXY = srcXY + (count*2); 300 uint32_t *end_xy = xy + (count); 301 const SkFixed *base_srcXY = srcXY; 302 uint32_t *base_xy = xy; 303 int base_count = count; 304 #endif 305 306 #if 1 307 // 2009/9/30: crashes in ApiDemos - Views - Animation - 3D Transition 308 // 2009/10/9: reworked to avoid illegal (but allowed by gas) insn 309 310 /* srcXY is a batch of 32 bit numbers X0,Y0,X1,Y1... 311 * but we immediately discard the low 16 bits... 312 * so what we're going to do is vld4, which will give us 313 * xlo,xhi,ylo,yhi distribution and we can ignore the 'lo' 314 * parts.... 315 */ 316 if (count >= 8) { 317 int16_t *mysrc = (int16_t *) srcXY; 318 int16_t *mydst = (int16_t *) xy; 319 int16x4_t maxX4 = vdup_n_s16((int16_t)maxX); 320 int16x4_t maxY4 = vdup_n_s16((int16_t)maxY); 321 int16x4_t zero4 = vdup_n_s16(0); 322 323 /* The constructs with local blocks for register assignments 324 * and asm() instructions is to make keep any hard register 325 * assignments to as small a scope as possible. and to avoid 326 * burning call-preserved hard registers on the vld/vst 327 * instructions. 328 */ 329 330 do { 331 int16x4_t xlo, xhi, ylo, yhi; 332 int16x4_t x2lo, x2hi, y2lo, y2hi; 333 334 /* vld4 does the de-interleaving for us */ 335 { 336 register int16x4_t t_xlo asm("d0"); 337 register int16x4_t t_xhi asm("d1"); 338 register int16x4_t t_ylo asm("d2"); 339 register int16x4_t t_yhi asm("d3"); 340 341 asm ("vld4.16 {d0-d3},[%4] /* xlo=%P0 xhi=%P1 ylo=%P2 yhi=%P3 */" 342 : "=w" (t_xlo), "=w" (t_xhi), "=w" (t_ylo), "=w" (t_yhi) 343 : "r" (mysrc) 344 ); 345 xlo = t_xlo; 346 xhi = t_xhi; 347 ylo = t_ylo; 348 yhi = t_yhi; 349 } 350 351 /* clamp X>>16 (aka xhi) to 0..maxX */ 352 xhi = vmax_s16(xhi, zero4); /* now 0.. */ 353 xhi = vmin_s16(xhi, maxX4); /* now 0..maxX */ 354 355 /* clamp Y>>16 (aka yhi) to 0..maxY */ 356 yhi = vmax_s16(yhi, zero4); /* now 0.. */ 357 yhi = vmin_s16(yhi, maxY4); /* now 0..maxY */ 358 359 /* deal with the second set of numbers */ 360 { 361 register int16x4_t t_xlo asm("d4"); 362 register int16x4_t t_xhi asm("d5"); 363 register int16x4_t t_ylo asm("d6"); 364 register int16x4_t t_yhi asm("d7"); 365 366 /* offset == 256 bits == 32 bytes == 8 longs == 16 shorts */ 367 asm ("vld4.16 {d4-d7},[%4] /* xlo=%P0 xhi=%P1 ylo=%P2 yhi=%P3 */" 368 : "=w" (t_xlo), "=w" (t_xhi), "=w" (t_ylo), "=w" (t_yhi) 369 : "r" (mysrc+16) 370 ); 371 x2lo = t_xlo; 372 x2hi = t_xhi; 373 y2lo = t_ylo; 374 y2hi = t_yhi; 375 } 376 377 /* clamp the second 4 here */ 378 379 if (0) { extern void rbe(void); rbe(); } 380 381 /* clamp X>>16 (aka xhi) to 0..maxX */ 382 x2hi = vmax_s16(x2hi, zero4); /* now 0.. */ 383 x2hi = vmin_s16(x2hi, maxX4); /* now 0..maxX */ 384 385 /* clamp Y>>16 (aka yhi) to 0..maxY */ 386 y2hi = vmax_s16(y2hi, zero4); /* now 0.. */ 387 y2hi = vmin_s16(y2hi, maxY4); /* now 0..maxY */ 388 389 /* we're storing as {x,y}s: x is [0], y is [1] */ 390 /* we'll use vst2 to make this happen */ 391 392 { 393 register int16x4_t out_x asm("d16") = xhi; 394 register int16x4_t out_y asm("d17") = yhi; 395 396 asm ("vst2.16 {d16-d17},[%2] /* xlo=%P0 xhi=%P1 */" 397 : 398 : "w" (out_x), "w" (out_y), "r" (mydst) 399 ); 400 } 401 { 402 register int16x4_t out_x asm("d18") = x2hi; 403 register int16x4_t out_y asm("d19") = y2hi; 404 405 asm ("vst2.16 {d18-d19},[%2] /* xlo=%P0 xhi=%P1 */" 406 : 407 : "w" (out_x), "w" (out_y), "r" (mydst+8) 408 ); 409 } 410 411 /* XXX: gcc isn't interleaving these with the NEON ops 412 * but i think that all the scoreboarding works out */ 413 count -= 8; /* 8 iterations */ 414 mysrc += 32; /* 16 longs, aka 32 shorts */ 415 mydst += 16; /* 16 shorts, aka 8 longs */ 416 } while (count >= 8); 417 /* get xy and srcXY fixed up */ 418 srcXY = (const SkFixed *) mysrc; 419 xy = (uint32_t *) mydst; 420 } 421 #endif 422 423 while (--count >= 0) { 424 *xy++ = (TILEY_PROCF(srcXY[1], maxY) << 16) | 425 TILEX_PROCF(srcXY[0], maxX); 426 srcXY += 2; 427 } 428 429 #if defined(DEBUG_PERSP_NOFILTER) 430 /* for checking our NEON-produced results against vanilla code */ 431 { 432 int bad = (-1); 433 for (int i = 0; i < base_count; i++) { 434 uint32_t val; 435 val = (TILEY_PROCF (base_srcXY[i * 2 + 1], maxY) << 16) | 436 TILEX_PROCF (base_srcXY[i * 2 + 0], maxX); 437 438 if (val != base_xy[i]) { 439 bad = i; 440 break; 441 } 442 } 443 if (bad >= 0) { 444 SkDebugf("clamp-nofilter-persp failed piece %d\n", bad); 445 SkDebugf(" maxX %08x maxY %08x\n", maxX, maxY); 446 bad -= (bad & 0x7); /* align */ 447 for (int i = bad; i < bad + 8; i++) { 448 uint32_t val; 449 val = (TILEY_PROCF (base_srcXY[i * 2 + 1], maxY) << 16) | 450 TILEX_PROCF (base_srcXY[i * 2 + 0], maxX); 451 452 SkDebugf("%d: got %08x want %08x srcXY[0] %08x srcXY[1] %08x\n", 453 i, base_xy[i], val, base_srcXY[i * 2 + 0], 454 base_srcXY[i * 2 + 1]); 455 } 456 SkDebugf ("---\n"); 457 } 458 459 if (end_xy != xy) { 460 SkDebugf("xy ended at %08x, should be %08x\n", xy, end_xy); 461 } 462 if (end_srcXY != srcXY) { 463 SkDebugf("srcXY ended at %08x, should be %08x\n", srcXY, 464 end_srcXY); 465 } 466 } 467 #endif 468 } 469 } 470 471 #undef DEBUG_PERSP_NOFILTER 472 473 ////////////////////////////////////////////////////////////////////////////// 474 475 static inline uint32_t PACK_FILTER_Y_NAME(SkFixed f, unsigned max, 476 SkFixed one PREAMBLE_PARAM_Y) { 477 unsigned i = TILEY_PROCF(f, max); 478 i = (i << 4) | TILEY_LOW_BITS(f, max); 479 return (i << 14) | (TILEY_PROCF((f + one), max)); 480 } 481 482 static inline uint32_t PACK_FILTER_X_NAME(SkFixed f, unsigned max, 483 SkFixed one PREAMBLE_PARAM_X) { 484 unsigned i = TILEX_PROCF(f, max); 485 i = (i << 4) | TILEX_LOW_BITS(f, max); 486 return (i << 14) | (TILEX_PROCF((f + one), max)); 487 } 488 489 static void SCALE_FILTER_NAME(const SkBitmapProcState& s, 490 uint32_t xy[], int count, int x, int y) { 491 SkASSERT((s.fInvType & ~(SkMatrix::kTranslate_Mask | 492 SkMatrix::kScale_Mask)) == 0); 493 SkASSERT(s.fInvKy == 0); 494 495 PREAMBLE(s); 496 497 const unsigned maxX = s.fBitmap->width() - 1; 498 const SkFixed one = s.fFilterOneX; 499 const SkFixed dx = s.fInvSx; 500 SkFixed fx; 501 502 { 503 SkPoint pt; 504 s.fInvProc(*s.fInvMatrix, SkIntToScalar(x) + SK_ScalarHalf, 505 SkIntToScalar(y) + SK_ScalarHalf, &pt); 506 const SkFixed fy = SkScalarToFixed(pt.fY) - (s.fFilterOneY >> 1); 507 const unsigned maxY = s.fBitmap->height() - 1; 508 // compute our two Y values up front 509 *xy++ = PACK_FILTER_Y_NAME(fy, maxY, s.fFilterOneY PREAMBLE_ARG_Y); 510 // now initialize fx 511 fx = SkScalarToFixed(pt.fX) - (one >> 1); 512 } 513 514 #ifdef CHECK_FOR_DECAL 515 // test if we don't need to apply the tile proc 516 if (dx > 0 && 517 (unsigned)(fx >> 16) <= maxX && 518 (unsigned)((fx + dx * (count - 1)) >> 16) < maxX) { 519 decal_filter_scale(xy, fx, dx, count); 520 } else 521 #endif 522 523 if (count >= 4) { 524 int32x4_t wide_dx, wide_one; 525 int32x4_t wide_fx, wide_fx1, wide_i, wide_lo; 526 #if 0 527 /* verification hooks -- see below */ 528 SkFixed debug_fx = fx; 529 int count_done = 0; 530 #endif 531 532 wide_fx = vdupq_n_s32(fx); 533 wide_fx = vsetq_lane_s32(fx+dx, wide_fx, 1); 534 wide_fx = vsetq_lane_s32(fx+dx+dx, wide_fx, 2); 535 wide_fx = vsetq_lane_s32(fx+dx+dx+dx, wide_fx, 3); 536 537 wide_dx = vdupq_n_s32(dx); 538 wide_one = vdupq_n_s32(one); 539 540 while (count >= 4) { 541 /* original expands to: 542 * unsigned i = SkClampMax((f) >> 16, max); 543 * i = (i << 4) | (((f) >> 12) & 0xF); 544 * return (i << 14) | (SkClampMax(((f + one)) >> 16, max)); 545 */ 546 547 /* i = SkClampMax(f>>16, maxX) */ 548 wide_i = vmaxq_s32(vshrq_n_s32(wide_fx,16), vdupq_n_s32(0)); 549 wide_i = vminq_s32(wide_i, vdupq_n_s32(maxX)); 550 551 /* i<<4 | TILEX_LOW_BITS(fx) */ 552 wide_lo = vshrq_n_s32(wide_fx, 12); 553 wide_i = vsliq_n_s32(wide_lo, wide_i, 4); 554 555 /* i<<14 */ 556 wide_i = vshlq_n_s32(wide_i, 14); 557 558 /* SkClampMax(((f + one)) >> 16, max) */ 559 wide_fx1 = vaddq_s32(wide_fx, wide_one); 560 wide_fx1 = vmaxq_s32(vshrq_n_s32(wide_fx1,16), vdupq_n_s32(0)); 561 wide_fx1 = vminq_s32(wide_fx1, vdupq_n_s32(maxX)); 562 563 /* final combination */ 564 wide_i = vorrq_s32(wide_i, wide_fx1); 565 566 vst1q_u32(xy, vreinterpretq_u32_s32(wide_i)); 567 568 #if 0 569 /* having a verification hook is a good idea */ 570 /* use debug_fx, debug_fx+dx, etc. */ 571 572 for (int i=0;i<4;i++) { 573 uint32_t want = PACK_FILTER_X_NAME(debug_fx, maxX, one PREAMBLE_ARG_X); 574 if (xy[i] != want) 575 { 576 /* print a nastygram */ 577 SkDebugf("clamp-filter-scale fails\n"); 578 SkDebugf("got %08x want %08x\n", xy[i], want); 579 SkDebugf("fx %08x debug_fx %08x dx %08x done %d\n", 580 fx, debug_fx, dx, count_done); 581 SkDebugf(" maxX %08x one %08x\n", maxX, one); 582 583 } 584 debug_fx += dx; 585 count_done++; 586 } 587 #endif 588 wide_fx += vdupq_n_s32(dx+dx+dx+dx); 589 fx += dx+dx+dx+dx; 590 xy += 4; 591 count -= 4; 592 } 593 } 594 595 while (--count >= 0) { 596 *xy++ = PACK_FILTER_X_NAME(fx, maxX, one PREAMBLE_ARG_X); 597 fx += dx; 598 } 599 } 600 601 static void AFFINE_FILTER_NAME(const SkBitmapProcState& s, 602 uint32_t xy[], int count, int x, int y) { 603 SkASSERT(s.fInvType & SkMatrix::kAffine_Mask); 604 SkASSERT((s.fInvType & ~(SkMatrix::kTranslate_Mask | 605 SkMatrix::kScale_Mask | 606 SkMatrix::kAffine_Mask)) == 0); 607 608 PREAMBLE(s); 609 SkPoint srcPt; 610 s.fInvProc(*s.fInvMatrix, 611 SkIntToScalar(x) + SK_ScalarHalf, 612 SkIntToScalar(y) + SK_ScalarHalf, &srcPt); 613 614 SkFixed oneX = s.fFilterOneX; 615 SkFixed oneY = s.fFilterOneY; 616 SkFixed fx = SkScalarToFixed(srcPt.fX) - (oneX >> 1); 617 SkFixed fy = SkScalarToFixed(srcPt.fY) - (oneY >> 1); 618 SkFixed dx = s.fInvSx; 619 SkFixed dy = s.fInvKy; 620 unsigned maxX = s.fBitmap->width() - 1; 621 unsigned maxY = s.fBitmap->height() - 1; 622 623 if (count >= 4) { 624 int32x4_t wide_one, wide_i, wide_lo; 625 int32x4_t wide_dx, wide_fx, wide_onex, wide_fx1; 626 int32x4_t wide_dy, wide_fy, wide_oney, wide_fy1; 627 628 #undef AFFINE_DEBUG 629 #if defined(AFFINE_DEBUG) 630 SkFixed fyp = fy; 631 SkFixed fxp = fx; 632 uint32_t *xyp = xy; 633 int count_done = 0; 634 #endif 635 636 wide_fx = vdupq_n_s32(fx); 637 wide_fx = vsetq_lane_s32(fx+dx, wide_fx, 1); 638 wide_fx = vsetq_lane_s32(fx+dx+dx, wide_fx, 2); 639 wide_fx = vsetq_lane_s32(fx+dx+dx+dx, wide_fx, 3); 640 wide_dx = vdupq_n_s32(dx); 641 642 wide_fy = vdupq_n_s32(fy); 643 wide_fy = vsetq_lane_s32(fy+dy, wide_fy, 1); 644 wide_fy = vsetq_lane_s32(fy+dy+dy, wide_fy, 2); 645 wide_fy = vsetq_lane_s32(fy+dy+dy+dy, wide_fy, 3); 646 wide_dy = vdupq_n_s32(dy); 647 648 wide_onex = vdupq_n_s32(oneX); 649 wide_oney = vdupq_n_s32(oneY); 650 651 while (count >= 4) { 652 int32x4_t wide_x; 653 int32x4_t wide_y; 654 655 /* do the X side, then the Y side, then interleave them */ 656 657 /* original expands to: 658 * unsigned i = SkClampMax((f) >> 16, max); 659 * i = (i << 4) | (((f) >> 12) & 0xF); 660 * return (i << 14) | (SkClampMax(((f + one)) >> 16, max)); 661 */ 662 663 /* i = SkClampMax(f>>16, maxX) */ 664 wide_i = vmaxq_s32(vshrq_n_s32(wide_fx,16), vdupq_n_s32(0)); 665 wide_i = vminq_s32(wide_i, vdupq_n_s32(maxX)); 666 667 /* i<<4 | TILEX_LOW_BITS(fx) */ 668 wide_lo = vshrq_n_s32(wide_fx, 12); 669 wide_i = vsliq_n_s32(wide_lo, wide_i, 4); 670 671 /* i<<14 */ 672 wide_i = vshlq_n_s32(wide_i, 14); 673 674 /* SkClampMax(((f + one)) >> 16, max) */ 675 wide_fx1 = vaddq_s32(wide_fx, wide_onex); 676 wide_fx1 = vmaxq_s32(vshrq_n_s32(wide_fx1,16), vdupq_n_s32(0)); 677 wide_fx1 = vminq_s32(wide_fx1, vdupq_n_s32(maxX)); 678 679 /* final combination */ 680 wide_x = vorrq_s32(wide_i, wide_fx1); 681 682 /* And now the Y side */ 683 684 /* i = SkClampMax(f>>16, maxX) */ 685 wide_i = vmaxq_s32(vshrq_n_s32(wide_fy,16), vdupq_n_s32(0)); 686 wide_i = vminq_s32(wide_i, vdupq_n_s32(maxY)); 687 688 /* i<<4 | TILEX_LOW_BITS(fx) */ 689 wide_lo = vshrq_n_s32(wide_fy, 12); 690 wide_i = vsliq_n_s32(wide_lo, wide_i, 4); 691 692 /* i<<14 */ 693 wide_i = vshlq_n_s32(wide_i, 14); 694 695 /* SkClampMax(((f + one)) >> 16, max) */ 696 wide_fy1 = vaddq_s32(wide_fy, wide_oney); 697 wide_fy1 = vmaxq_s32(vshrq_n_s32(wide_fy1,16), vdupq_n_s32(0)); 698 wide_fy1 = vminq_s32(wide_fy1, vdupq_n_s32(maxY)); 699 700 /* final combination */ 701 wide_y = vorrq_s32(wide_i, wide_fy1); 702 703 /* interleave as YXYXYXYX as part of the storing */ 704 { 705 /* vst2.32 needs side-by-side registers */ 706 register int32x4_t t_x asm("q1"); 707 register int32x4_t t_y asm("q0"); 708 709 t_x = wide_x; t_y = wide_y; 710 asm ("vst2.32 {q0-q1},[%2] /* y=%q0 x=%q1 */" 711 : 712 : "w" (t_y), "w" (t_x), "r" (xy) 713 ); 714 } 715 716 #if defined(AFFINE_DEBUG) 717 /* make sure we're good here -- check the 4 we just output */ 718 for (int i = 0; i<4;i++) { 719 uint32_t val; 720 val = PACK_FILTER_Y_NAME(fyp, maxY, oneY PREAMBLE_ARG_Y); 721 if (val != xy[i*2+0]) { 722 /* print a nastygram */ 723 SkDebugf("clamp-filter-affine fails\n"); 724 SkDebugf("[bad-y] got %08x want %08x\n", xy[i*2+0], val); 725 SkDebugf("fy %08x fxp %08x fyp %08x dx %08x dy %08x done %d\n", 726 fy, fxp, fyp, dx, dy, count_done); 727 SkDebugf(" maxY %08x oneY %08x\n", maxY, oneY); 728 } 729 val = PACK_FILTER_X_NAME(fxp, maxX, oneX PREAMBLE_ARG_X); 730 if (val != xy[i*2+1]) { 731 /* print a nastygram */ 732 SkDebugf("clamp-filter-affine fails\n"); 733 SkDebugf("[bad-x] got %08x want %08x\n", xy[i*2+1], val); 734 SkDebugf("fx %08x fxp %08x fyp %08x dx %08x dy %08x done %d\n", 735 fx, fxp, fyp, dx, dy, count_done); 736 SkDebugf(" maxX %08x one %08x\n", maxX, oneX); 737 } 738 fyp += dy; 739 fxp += dx; 740 count_done++; 741 } 742 #endif 743 744 wide_fx += vdupq_n_s32(dx+dx+dx+dx); 745 fx += dx+dx+dx+dx; 746 wide_fy += vdupq_n_s32(dy+dy+dy+dy); 747 fy += dy+dy+dy+dy; 748 xy += 8; /* 4 x's, 4 y's */ 749 count -= 4; 750 } 751 } 752 753 while (--count >= 0) { 754 /* NB: writing Y/X */ 755 *xy++ = PACK_FILTER_Y_NAME(fy, maxY, oneY PREAMBLE_ARG_Y); 756 fy += dy; 757 *xy++ = PACK_FILTER_X_NAME(fx, maxX, oneX PREAMBLE_ARG_X); 758 fx += dx; 759 } 760 } 761 762 static void PERSP_FILTER_NAME(const SkBitmapProcState& s, 763 uint32_t* SK_RESTRICT xy, int count, 764 int x, int y) { 765 SkASSERT(s.fInvType & SkMatrix::kPerspective_Mask); 766 767 PREAMBLE(s); 768 unsigned maxX = s.fBitmap->width() - 1; 769 unsigned maxY = s.fBitmap->height() - 1; 770 SkFixed oneX = s.fFilterOneX; 771 SkFixed oneY = s.fFilterOneY; 772 773 SkPerspIter iter(*s.fInvMatrix, 774 SkIntToScalar(x) + SK_ScalarHalf, 775 SkIntToScalar(y) + SK_ScalarHalf, count); 776 777 while ((count = iter.next()) != 0) { 778 const SkFixed* SK_RESTRICT srcXY = iter.getXY(); 779 780 if (count >= 4) { 781 int32x4_t wide_one, wide_i, wide_lo; 782 int32x4_t wide_fx1; 783 int32x4_t wide_fy1; 784 int32x4_t wide_x, wide_y; 785 786 while (count >= 4) { 787 /* RBE: it's good, but: 788 * -- we spill a constant that could be easily regnerated 789 * [perhaps tweak gcc's NEON constant costs?] 790 */ 791 792 /* load src: x-y-x-y-x-y-x-y */ 793 { 794 register int32x4_t q0 asm ("q0"); 795 register int32x4_t q1 asm ("q1"); 796 asm ("vld2.32 {q0-q1},[%2] /* x=%q0 y=%q1 */" 797 : "=w" (q0), "=w" (q1) 798 : "r" (srcXY)); 799 wide_x = q0; wide_y = q1; 800 } 801 802 /* do the X side, then the Y side, then interleave them */ 803 804 wide_x = vsubq_s32(wide_x, vdupq_n_s32 (oneX>>1)); 805 806 /* original expands to: 807 * unsigned i = SkClampMax((f) >> 16, max); 808 * i = (i << 4) | (((f) >> 12) & 0xF); 809 * return (i << 14) | (SkClampMax(((f + one)) >> 16, max)); 810 */ 811 812 /* i = SkClampMax(f>>16, maxX) */ 813 wide_i = vmaxq_s32 (vshrq_n_s32 (wide_x, 16), vdupq_n_s32 (0)); 814 wide_i = vminq_s32 (wide_i, vdupq_n_s32 (maxX)); 815 816 /* i<<4 | TILEX_LOW_BITS(fx) */ 817 wide_lo = vshrq_n_s32 (wide_x, 12); 818 wide_i = vsliq_n_s32 (wide_lo, wide_i, 4); 819 820 /* i<<14 */ 821 wide_i = vshlq_n_s32 (wide_i, 14); 822 823 /* SkClampMax(((f + one)) >> 16, max) */ 824 wide_fx1 = vaddq_s32 (wide_x, vdupq_n_s32(oneX)); 825 wide_fx1 = vmaxq_s32 (vshrq_n_s32 (wide_fx1, 16), vdupq_n_s32 (0)); 826 wide_fx1 = vminq_s32 (wide_fx1, vdupq_n_s32 (maxX)); 827 828 /* final combination */ 829 wide_x = vorrq_s32 (wide_i, wide_fx1); 830 831 832 /* And now the Y side */ 833 834 wide_y = vsubq_s32(wide_y, vdupq_n_s32 (oneY>>1)); 835 836 /* i = SkClampMax(f>>16, maxX) */ 837 wide_i = vmaxq_s32 (vshrq_n_s32 (wide_y, 16), vdupq_n_s32 (0)); 838 wide_i = vminq_s32 (wide_i, vdupq_n_s32 (maxY)); 839 840 /* i<<4 | TILEX_LOW_BITS(fx) */ 841 wide_lo = vshrq_n_s32 (wide_y, 12); 842 wide_i = vsliq_n_s32 (wide_lo, wide_i, 4); 843 844 /* i<<14 */ 845 wide_i = vshlq_n_s32 (wide_i, 14); 846 847 /* SkClampMax(((f + one)) >> 16, max) */ 848 849 /* wide_fy1_1 and wide_fy1_2 are just temporary variables to 850 * work-around an ICE in debug */ 851 int32x4_t wide_fy1_1 = vaddq_s32 (wide_y, vdupq_n_s32(oneY)); 852 int32x4_t wide_fy1_2 = vmaxq_s32 (vshrq_n_s32 (wide_fy1_1, 16), 853 vdupq_n_s32 (0)); 854 wide_fy1 = vminq_s32 (wide_fy1_2, vdupq_n_s32 (maxY)); 855 856 /* final combination */ 857 wide_y = vorrq_s32 (wide_i, wide_fy1); 858 859 /* switch them around; have to do it this way to get them 860 * in the proper registers to match our instruction */ 861 862 /* iteration bookkeeping, ahead of the asm() for scheduling */ 863 srcXY += 2*4; 864 count -= 4; 865 866 /* store interleaved as y-x-y-x-y-x-y-x (NB != read order) */ 867 { 868 register int32x4_t q0 asm ("q0") = wide_y; 869 register int32x4_t q1 asm ("q1") = wide_x; 870 871 asm ("vst2.32 {q0-q1},[%2] /* y=%q0 x=%q1 */" 872 : 873 : "w" (q0), "w" (q1), "r" (xy)); 874 } 875 876 /* on to the next iteration */ 877 /* count, srcXY are handled above */ 878 xy += 2*4; 879 } 880 } 881 882 /* was do-while; NEON code invalidates original count>0 assumption */ 883 while (--count >= 0) { 884 /* NB: we read x/y, we write y/x */ 885 *xy++ = PACK_FILTER_Y_NAME(srcXY[1] - (oneY >> 1), maxY, 886 oneY PREAMBLE_ARG_Y); 887 *xy++ = PACK_FILTER_X_NAME(srcXY[0] - (oneX >> 1), maxX, 888 oneX PREAMBLE_ARG_X); 889 srcXY += 2; 890 } 891 } 892 } 893 894 static SkBitmapProcState::MatrixProc MAKENAME(_Procs)[] = { 895 SCALE_NOFILTER_NAME, 896 SCALE_FILTER_NAME, 897 AFFINE_NOFILTER_NAME, 898 AFFINE_FILTER_NAME, 899 PERSP_NOFILTER_NAME, 900 PERSP_FILTER_NAME 901 }; 902 903 #undef MAKENAME 904 #undef TILEX_PROCF 905 #undef TILEY_PROCF 906 #ifdef CHECK_FOR_DECAL 907 #undef CHECK_FOR_DECAL 908 #endif 909 910 #undef SCALE_NOFILTER_NAME 911 #undef SCALE_FILTER_NAME 912 #undef AFFINE_NOFILTER_NAME 913 #undef AFFINE_FILTER_NAME 914 #undef PERSP_NOFILTER_NAME 915 #undef PERSP_FILTER_NAME 916 917 #undef PREAMBLE 918 #undef PREAMBLE_PARAM_X 919 #undef PREAMBLE_PARAM_Y 920 #undef PREAMBLE_ARG_X 921 #undef PREAMBLE_ARG_Y 922 923 #undef TILEX_LOW_BITS 924 #undef TILEY_LOW_BITS 925
