1 /* 2 * Copyright 2017 Google Inc. 3 * 4 * Use of this source code is governed by a BSD-style license that can be 5 * found in the LICENSE file. 6 */ 7 8 #include "SkJumper.h" 9 #include "SkJumper_misc.h" // SI, unaligned_load(), bit_cast() 10 #include "SkJumper_vectors.h" // F, I32, U32, U16, U8, cast(), expand() 11 12 // Our fundamental vector depth is our pixel stride. 13 static const size_t kStride = sizeof(F) / sizeof(float); 14 15 // A reminder: 16 // Code guarded by defined(JUMPER) can assume that it will be compiled by Clang 17 // and that F, I32, etc. are kStride-deep ext_vector_types of the appropriate type. 18 // Otherwise, F, I32, etc. just alias the basic scalar types (and so kStride == 1). 19 20 // You can use most constants in this file, but in a few rare exceptions we read from this struct. 21 using K = const SkJumper_constants; 22 23 // A little wrapper macro to name Stages differently depending on the instruction set. 24 // That lets us link together several options. 25 #if !defined(JUMPER) 26 #define WRAP(name) sk_##name 27 #elif defined(__aarch64__) 28 #define WRAP(name) sk_##name##_aarch64 29 #elif defined(__arm__) 30 #define WRAP(name) sk_##name##_vfp4 31 #elif defined(__AVX2__) 32 #define WRAP(name) sk_##name##_hsw 33 #elif defined(__AVX__) 34 #define WRAP(name) sk_##name##_avx 35 #elif defined(__SSE4_1__) 36 #define WRAP(name) sk_##name##_sse41 37 #elif defined(__SSE2__) 38 #define WRAP(name) sk_##name##_sse2 39 #endif 40 41 // We're finally going to get to what a Stage function looks like! 42 // tail == 0 ~~> work on a full kStride pixels 43 // tail != 0 ~~> work on only the first tail pixels 44 // tail is always < kStride. 45 // 46 // We keep program the second argument, so that it's passed in rsi for load_and_inc(). 47 using Stage = void(K* k, void** program, size_t x, size_t y, size_t tail, F,F,F,F, F,F,F,F); 48 49 #if defined(JUMPER) && defined(__AVX__) 50 // We really want to make sure all paths go through this function's (implicit) vzeroupper. 51 // If they don't, we'll experience severe slowdowns when we first use SSE instructions again. 52 __attribute__((disable_tail_calls)) 53 #endif 54 MAYBE_MSABI 55 extern "C" void WRAP(start_pipeline)(size_t x, size_t y, size_t limit, void** program, K* k) { 56 #if defined(JUMPER) 57 F v; 58 #else 59 F v{}; 60 #endif 61 auto start = (Stage*)load_and_inc(program); 62 while (x + kStride <= limit) { 63 start(k,program,x,y,0, v,v,v,v, v,v,v,v); 64 x += kStride; 65 } 66 if (size_t tail = limit - x) { 67 start(k,program,x,y,tail, v,v,v,v, v,v,v,v); 68 } 69 } 70 71 #if defined(JUMPER) && defined(__AVX__) 72 // We really want to make sure all paths go through this function's (implicit) vzeroupper. 73 // If they don't, we'll experience severe slowdowns when we first use SSE instructions again. 74 __attribute__((disable_tail_calls)) 75 #endif 76 #if defined(JUMPER) 77 __attribute__((flatten)) // Force-inline the call to start_pipeline(). 78 #endif 79 MAYBE_MSABI 80 extern "C" void WRAP(start_pipeline_2d)(size_t x, size_t y, size_t xlimit, size_t ylimit, 81 void** program, K* k) { 82 for (; y < ylimit; y++) { 83 WRAP(start_pipeline)(x,y,xlimit, program, k); 84 } 85 } 86 87 #define STAGE(name) \ 88 SI void name##_k(K* k, LazyCtx ctx, size_t x, size_t y, size_t tail, \ 89 F& r, F& g, F& b, F& a, F& dr, F& dg, F& db, F& da); \ 90 extern "C" void WRAP(name)(K* k, void** program, size_t x, size_t y, size_t tail, \ 91 F r, F g, F b, F a, F dr, F dg, F db, F da) { \ 92 LazyCtx ctx(program); \ 93 name##_k(k,ctx,x,y,tail, r,g,b,a, dr,dg,db,da); \ 94 auto next = (Stage*)load_and_inc(program); \ 95 next(k,program,x,y,tail, r,g,b,a, dr,dg,db,da); \ 96 } \ 97 SI void name##_k(K* k, LazyCtx ctx, size_t x, size_t y, size_t tail, \ 98 F& r, F& g, F& b, F& a, F& dr, F& dg, F& db, F& da) 99 100 101 // just_return() is a simple no-op stage that only exists to end the chain, 102 // returning back up to start_pipeline(), and from there to the caller. 103 extern "C" void WRAP(just_return)(K*, void**, size_t,size_t,size_t, F,F,F,F, F,F,F,F) {} 104 105 106 // We could start defining normal Stages now. But first, some helper functions. 107 108 // These load() and store() methods are tail-aware, 109 // but focus mainly on keeping the at-stride tail==0 case fast. 110 111 template <typename V, typename T> 112 SI V load(const T* src, size_t tail) { 113 #if defined(JUMPER) 114 __builtin_assume(tail < kStride); 115 if (__builtin_expect(tail, 0)) { 116 V v{}; // Any inactive lanes are zeroed. 117 switch (tail) { 118 case 7: v[6] = src[6]; 119 case 6: v[5] = src[5]; 120 case 5: v[4] = src[4]; 121 case 4: memcpy(&v, src, 4*sizeof(T)); break; 122 case 3: v[2] = src[2]; 123 case 2: memcpy(&v, src, 2*sizeof(T)); break; 124 case 1: memcpy(&v, src, 1*sizeof(T)); break; 125 } 126 return v; 127 } 128 #endif 129 return unaligned_load<V>(src); 130 } 131 132 template <typename V, typename T> 133 SI void store(T* dst, V v, size_t tail) { 134 #if defined(JUMPER) 135 __builtin_assume(tail < kStride); 136 if (__builtin_expect(tail, 0)) { 137 switch (tail) { 138 case 7: dst[6] = v[6]; 139 case 6: dst[5] = v[5]; 140 case 5: dst[4] = v[4]; 141 case 4: memcpy(dst, &v, 4*sizeof(T)); break; 142 case 3: dst[2] = v[2]; 143 case 2: memcpy(dst, &v, 2*sizeof(T)); break; 144 case 1: memcpy(dst, &v, 1*sizeof(T)); break; 145 } 146 return; 147 } 148 #endif 149 unaligned_store(dst, v); 150 } 151 152 // AVX adds some mask loads and stores that make for shorter, faster code. 153 #if defined(JUMPER) && defined(__AVX__) 154 SI U32 mask(size_t tail) { 155 // We go a little out of our way to avoid needing large constant values here. 156 157 // It's easiest to build the mask as 8 8-bit values, either 0x00 or 0xff. 158 // Start fully on, then shift away lanes from the top until we've got our mask. 159 uint64_t mask = 0xffffffffffffffff >> 8*(kStride-tail); 160 161 // Sign-extend each mask lane to its full width, 0x00000000 or 0xffffffff. 162 using S8 = int8_t __attribute__((ext_vector_type(8))); 163 using S32 = int32_t __attribute__((ext_vector_type(8))); 164 return (U32)__builtin_convertvector(unaligned_load<S8>(&mask), S32); 165 } 166 167 template <> 168 inline U32 load(const uint32_t* src, size_t tail) { 169 __builtin_assume(tail < kStride); 170 if (__builtin_expect(tail, 0)) { 171 return (U32)_mm256_maskload_ps((const float*)src, mask(tail)); 172 } 173 return unaligned_load<U32>(src); 174 } 175 176 template <> 177 inline void store(uint32_t* dst, U32 v, size_t tail) { 178 __builtin_assume(tail < kStride); 179 if (__builtin_expect(tail, 0)) { 180 return _mm256_maskstore_ps((float*)dst, mask(tail), (F)v); 181 } 182 unaligned_store(dst, v); 183 } 184 #endif 185 186 SI F from_byte(U8 b) { 187 return cast(expand(b)) * (1/255.0f); 188 } 189 SI void from_565(U16 _565, F* r, F* g, F* b) { 190 U32 wide = expand(_565); 191 *r = cast(wide & (31<<11)) * (1.0f / (31<<11)); 192 *g = cast(wide & (63<< 5)) * (1.0f / (63<< 5)); 193 *b = cast(wide & (31<< 0)) * (1.0f / (31<< 0)); 194 } 195 SI void from_4444(U16 _4444, F* r, F* g, F* b, F* a) { 196 U32 wide = expand(_4444); 197 *r = cast(wide & (15<<12)) * (1.0f / (15<<12)); 198 *g = cast(wide & (15<< 8)) * (1.0f / (15<< 8)); 199 *b = cast(wide & (15<< 4)) * (1.0f / (15<< 4)); 200 *a = cast(wide & (15<< 0)) * (1.0f / (15<< 0)); 201 } 202 SI void from_8888(U32 _8888, F* r, F* g, F* b, F* a) { 203 *r = cast((_8888 ) & 0xff) * (1/255.0f); 204 *g = cast((_8888 >> 8) & 0xff) * (1/255.0f); 205 *b = cast((_8888 >> 16) & 0xff) * (1/255.0f); 206 *a = cast((_8888 >> 24) ) * (1/255.0f); 207 } 208 209 template <typename T> 210 SI U32 ix_and_ptr(T** ptr, const SkJumper_MemoryCtx* ctx, F x, F y) { 211 *ptr = (const T*)ctx->pixels; 212 return trunc_(y)*ctx->stride + trunc_(x); 213 } 214 215 // Now finally, normal Stages! 216 217 STAGE(seed_shader) { 218 // It's important for speed to explicitly cast(x) and cast(y), 219 // which has the effect of splatting them to vectors before converting to floats. 220 // On Intel this breaks a data dependency on previous loop iterations' registers. 221 r = cast(x) + 0.5f + unaligned_load<F>(k->iota_F); 222 g = cast(y) + 0.5f; 223 b = 1.0f; 224 a = 0; 225 dr = dg = db = da = 0; 226 } 227 228 STAGE(dither) { 229 auto rate = *(const float*)ctx; 230 231 // Get [(x,y), (x+1,y), (x+2,y), ...] loaded up in integer vectors. 232 U32 X = x + unaligned_load<U32>(k->iota_U32), 233 Y = y; 234 235 // We're doing 8x8 ordered dithering, see https://en.wikipedia.org/wiki/Ordered_dithering. 236 // In this case n=8 and we're using the matrix that looks like 1/64 x [ 0 48 12 60 ... ]. 237 238 // We only need X and X^Y from here on, so it's easier to just think of that as "Y". 239 Y ^= X; 240 241 // We'll mix the bottom 3 bits of each of X and Y to make 6 bits, 242 // for 2^6 == 64 == 8x8 matrix values. If X=abc and Y=def, we make fcebda. 243 U32 M = (Y & 1) << 5 | (X & 1) << 4 244 | (Y & 2) << 2 | (X & 2) << 1 245 | (Y & 4) >> 1 | (X & 4) >> 2; 246 247 // Scale that dither to [0,1), then (-0.5,+0.5), here using 63/128 = 0.4921875 as 0.5-epsilon. 248 // We want to make sure our dither is less than 0.5 in either direction to keep exact values 249 // like 0 and 1 unchanged after rounding. 250 F dither = cast(M) * (2/128.0f) - (63/128.0f); 251 252 r += rate*dither; 253 g += rate*dither; 254 b += rate*dither; 255 256 r = max(0, min(r, a)); 257 g = max(0, min(g, a)); 258 b = max(0, min(b, a)); 259 } 260 261 // load 4 floats from memory, and splat them into r,g,b,a 262 STAGE(uniform_color) { 263 auto rgba = (const float*)ctx; 264 r = rgba[0]; 265 g = rgba[1]; 266 b = rgba[2]; 267 a = rgba[3]; 268 } 269 270 // splats opaque-black into r,g,b,a 271 STAGE(black_color) { 272 r = g = b = 0.0f; 273 a = 1.0f; 274 } 275 276 STAGE(white_color) { 277 r = g = b = a = 1.0f; 278 } 279 280 // load registers r,g,b,a from context (mirrors store_rgba) 281 STAGE(load_rgba) { 282 auto ptr = (const float*)ctx; 283 r = unaligned_load<F>(ptr + 0*kStride); 284 g = unaligned_load<F>(ptr + 1*kStride); 285 b = unaligned_load<F>(ptr + 2*kStride); 286 a = unaligned_load<F>(ptr + 3*kStride); 287 } 288 289 // store registers r,g,b,a into context (mirrors load_rgba) 290 STAGE(store_rgba) { 291 auto ptr = (float*)ctx; 292 unaligned_store(ptr + 0*kStride, r); 293 unaligned_store(ptr + 1*kStride, g); 294 unaligned_store(ptr + 2*kStride, b); 295 unaligned_store(ptr + 3*kStride, a); 296 } 297 298 // Most blend modes apply the same logic to each channel. 299 #define BLEND_MODE(name) \ 300 SI F name##_channel(F s, F d, F sa, F da); \ 301 STAGE(name) { \ 302 r = name##_channel(r,dr,a,da); \ 303 g = name##_channel(g,dg,a,da); \ 304 b = name##_channel(b,db,a,da); \ 305 a = name##_channel(a,da,a,da); \ 306 } \ 307 SI F name##_channel(F s, F d, F sa, F da) 308 309 SI F inv(F x) { return 1.0f - x; } 310 SI F two(F x) { return x + x; } 311 312 BLEND_MODE(clear) { return 0; } 313 BLEND_MODE(srcatop) { return s*da + d*inv(sa); } 314 BLEND_MODE(dstatop) { return d*sa + s*inv(da); } 315 BLEND_MODE(srcin) { return s * da; } 316 BLEND_MODE(dstin) { return d * sa; } 317 BLEND_MODE(srcout) { return s * inv(da); } 318 BLEND_MODE(dstout) { return d * inv(sa); } 319 BLEND_MODE(srcover) { return mad(d, inv(sa), s); } 320 BLEND_MODE(dstover) { return mad(s, inv(da), d); } 321 322 BLEND_MODE(modulate) { return s*d; } 323 BLEND_MODE(multiply) { return s*inv(da) + d*inv(sa) + s*d; } 324 BLEND_MODE(plus_) { return s + d; } 325 BLEND_MODE(screen) { return s + d - s*d; } 326 BLEND_MODE(xor_) { return s*inv(da) + d*inv(sa); } 327 #undef BLEND_MODE 328 329 // Most other blend modes apply the same logic to colors, and srcover to alpha. 330 #define BLEND_MODE(name) \ 331 SI F name##_channel(F s, F d, F sa, F da); \ 332 STAGE(name) { \ 333 r = name##_channel(r,dr,a,da); \ 334 g = name##_channel(g,dg,a,da); \ 335 b = name##_channel(b,db,a,da); \ 336 a = mad(da, inv(a), a); \ 337 } \ 338 SI F name##_channel(F s, F d, F sa, F da) 339 340 BLEND_MODE(darken) { return s + d - max(s*da, d*sa) ; } 341 BLEND_MODE(lighten) { return s + d - min(s*da, d*sa) ; } 342 BLEND_MODE(difference) { return s + d - two(min(s*da, d*sa)); } 343 BLEND_MODE(exclusion) { return s + d - two(s*d); } 344 345 BLEND_MODE(colorburn) { 346 return if_then_else(d == da, d + s*inv(da), 347 if_then_else(s == 0, s + d*inv(sa), 348 sa*(da - min(da, (da-d)*sa/s)) + s*inv(da) + d*inv(sa))); 349 } 350 BLEND_MODE(colordodge) { 351 return if_then_else(d == 0, d + s*inv(da), 352 if_then_else(s == sa, s + d*inv(sa), 353 sa*min(da, (d*sa)/(sa - s)) + s*inv(da) + d*inv(sa))); 354 } 355 BLEND_MODE(hardlight) { 356 return s*inv(da) + d*inv(sa) 357 + if_then_else(two(s) <= sa, two(s*d), sa*da - two((da-d)*(sa-s))); 358 } 359 BLEND_MODE(overlay) { 360 return s*inv(da) + d*inv(sa) 361 + if_then_else(two(d) <= da, two(s*d), sa*da - two((da-d)*(sa-s))); 362 } 363 364 BLEND_MODE(softlight) { 365 F m = if_then_else(da > 0, d / da, 0), 366 s2 = two(s), 367 m4 = two(two(m)); 368 369 // The logic forks three ways: 370 // 1. dark src? 371 // 2. light src, dark dst? 372 // 3. light src, light dst? 373 F darkSrc = d*(sa + (s2 - sa)*(1.0f - m)), // Used in case 1. 374 darkDst = (m4*m4 + m4)*(m - 1.0f) + 7.0f*m, // Used in case 2. 375 liteDst = rcp(rsqrt(m)) - m, // Used in case 3. 376 liteSrc = d*sa + da*(s2 - sa) * if_then_else(two(two(d)) <= da, darkDst, liteDst); // 2 or 3? 377 return s*inv(da) + d*inv(sa) + if_then_else(s2 <= sa, darkSrc, liteSrc); // 1 or (2 or 3)? 378 } 379 #undef BLEND_MODE 380 381 // We're basing our implemenation of non-separable blend modes on 382 // https://www.w3.org/TR/compositing-1/#blendingnonseparable. 383 // and 384 // https://www.khronos.org/registry/OpenGL/specs/es/3.2/es_spec_3.2.pdf 385 // They're equivalent, but ES' math has been better simplified. 386 // 387 // Anything extra we add beyond that is to make the math work with premul inputs. 388 389 SI F max(F r, F g, F b) { return max(r, max(g, b)); } 390 SI F min(F r, F g, F b) { return min(r, min(g, b)); } 391 392 SI F sat(F r, F g, F b) { return max(r,g,b) - min(r,g,b); } 393 SI F lum(F r, F g, F b) { return r*0.30f + g*0.59f + b*0.11f; } 394 395 SI void set_sat(F* r, F* g, F* b, F s) { 396 F mn = min(*r,*g,*b), 397 mx = max(*r,*g,*b), 398 sat = mx - mn; 399 400 // Map min channel to 0, max channel to s, and scale the middle proportionally. 401 auto scale = [=](F c) { 402 return if_then_else(sat == 0, 0, (c - mn) * s / sat); 403 }; 404 *r = scale(*r); 405 *g = scale(*g); 406 *b = scale(*b); 407 } 408 SI void set_lum(F* r, F* g, F* b, F l) { 409 F diff = l - lum(*r, *g, *b); 410 *r += diff; 411 *g += diff; 412 *b += diff; 413 } 414 SI void clip_color(F* r, F* g, F* b, F a) { 415 F mn = min(*r, *g, *b), 416 mx = max(*r, *g, *b), 417 l = lum(*r, *g, *b); 418 419 auto clip = [=](F c) { 420 c = if_then_else(mn >= 0, c, l + (c - l) * ( l) / (l - mn) ); 421 c = if_then_else(mx > a, l + (c - l) * (a - l) / (mx - l), c); 422 c = max(c, 0); // Sometimes without this we may dip just a little negative. 423 return c; 424 }; 425 *r = clip(*r); 426 *g = clip(*g); 427 *b = clip(*b); 428 } 429 430 STAGE(hue) { 431 F R = r*a, 432 G = g*a, 433 B = b*a; 434 435 set_sat(&R, &G, &B, sat(dr,dg,db)*a); 436 set_lum(&R, &G, &B, lum(dr,dg,db)*a); 437 clip_color(&R,&G,&B, a*da); 438 439 r = r*inv(da) + dr*inv(a) + R; 440 g = g*inv(da) + dg*inv(a) + G; 441 b = b*inv(da) + db*inv(a) + B; 442 a = a + da - a*da; 443 } 444 STAGE(saturation) { 445 F R = dr*a, 446 G = dg*a, 447 B = db*a; 448 449 set_sat(&R, &G, &B, sat( r, g, b)*da); 450 set_lum(&R, &G, &B, lum(dr,dg,db)* a); // (This is not redundant.) 451 clip_color(&R,&G,&B, a*da); 452 453 r = r*inv(da) + dr*inv(a) + R; 454 g = g*inv(da) + dg*inv(a) + G; 455 b = b*inv(da) + db*inv(a) + B; 456 a = a + da - a*da; 457 } 458 STAGE(color) { 459 F R = r*da, 460 G = g*da, 461 B = b*da; 462 463 set_lum(&R, &G, &B, lum(dr,dg,db)*a); 464 clip_color(&R,&G,&B, a*da); 465 466 r = r*inv(da) + dr*inv(a) + R; 467 g = g*inv(da) + dg*inv(a) + G; 468 b = b*inv(da) + db*inv(a) + B; 469 a = a + da - a*da; 470 } 471 STAGE(luminosity) { 472 F R = dr*a, 473 G = dg*a, 474 B = db*a; 475 476 set_lum(&R, &G, &B, lum(r,g,b)*da); 477 clip_color(&R,&G,&B, a*da); 478 479 r = r*inv(da) + dr*inv(a) + R; 480 g = g*inv(da) + dg*inv(a) + G; 481 b = b*inv(da) + db*inv(a) + B; 482 a = a + da - a*da; 483 } 484 485 STAGE(srcover_rgba_8888) { 486 auto ptr = *(uint32_t**)ctx + x; 487 488 U32 dst = load<U32>(ptr, tail); 489 dr = cast((dst ) & 0xff); 490 dg = cast((dst >> 8) & 0xff); 491 db = cast((dst >> 16) & 0xff); 492 da = cast((dst >> 24) ); 493 // {dr,dg,db,da} are in [0,255] 494 // { r, g, b, a} are in [0, 1] 495 496 r = mad(dr, inv(a), r*255.0f); 497 g = mad(dg, inv(a), g*255.0f); 498 b = mad(db, inv(a), b*255.0f); 499 a = mad(da, inv(a), a*255.0f); 500 // { r, g, b, a} are now in [0,255] 501 502 dst = round(r, 1.0f) 503 | round(g, 1.0f) << 8 504 | round(b, 1.0f) << 16 505 | round(a, 1.0f) << 24; 506 store(ptr, dst, tail); 507 } 508 509 STAGE(clamp_0) { 510 r = max(r, 0); 511 g = max(g, 0); 512 b = max(b, 0); 513 a = max(a, 0); 514 } 515 516 STAGE(clamp_1) { 517 r = min(r, 1.0f); 518 g = min(g, 1.0f); 519 b = min(b, 1.0f); 520 a = min(a, 1.0f); 521 } 522 523 STAGE(clamp_a) { 524 a = min(a, 1.0f); 525 r = min(r, a); 526 g = min(g, a); 527 b = min(b, a); 528 } 529 530 STAGE(clamp_a_dst) { 531 da = min(da, 1.0f); 532 dr = min(dr, da); 533 dg = min(dg, da); 534 db = min(db, da); 535 } 536 537 STAGE(set_rgb) { 538 auto rgb = (const float*)ctx; 539 r = rgb[0]; 540 g = rgb[1]; 541 b = rgb[2]; 542 } 543 STAGE(swap_rb) { 544 auto tmp = r; 545 r = b; 546 b = tmp; 547 } 548 549 STAGE(move_src_dst) { 550 dr = r; 551 dg = g; 552 db = b; 553 da = a; 554 } 555 STAGE(move_dst_src) { 556 r = dr; 557 g = dg; 558 b = db; 559 a = da; 560 } 561 562 STAGE(premul) { 563 r = r * a; 564 g = g * a; 565 b = b * a; 566 } 567 STAGE(premul_dst) { 568 dr = dr * da; 569 dg = dg * da; 570 db = db * da; 571 } 572 STAGE(unpremul) { 573 auto scale = if_then_else(a == 0, 0, 1.0f / a); 574 r *= scale; 575 g *= scale; 576 b *= scale; 577 } 578 579 SI F from_srgb(F s) { 580 auto lo = s * (1/12.92f); 581 auto hi = mad(s*s, mad(s, 0.3000f, 0.6975f), 0.0025f); 582 return if_then_else(s < 0.055f, lo, hi); 583 } 584 585 STAGE(from_srgb) { 586 r = from_srgb(r); 587 g = from_srgb(g); 588 b = from_srgb(b); 589 } 590 STAGE(from_srgb_dst) { 591 dr = from_srgb(dr); 592 dg = from_srgb(dg); 593 db = from_srgb(db); 594 } 595 STAGE(to_srgb) { 596 auto fn = [&](F l) { 597 // We tweak c and d for each instruction set to make sure fn(1) is exactly 1. 598 #if defined(JUMPER) && defined(__SSE2__) 599 const float c = 1.130048394203f, 600 d = 0.141357362270f; 601 #elif defined(JUMPER) && (defined(__aarch64__) || defined(__arm__)) 602 const float c = 1.129999995232f, 603 d = 0.141381442547f; 604 #else 605 const float c = 1.129999995232f, 606 d = 0.141377761960f; 607 #endif 608 F t = rsqrt(l); 609 auto lo = l * 12.92f; 610 auto hi = mad(t, mad(t, -0.0024542345f, 0.013832027f), c) 611 * rcp(d + t); 612 return if_then_else(l < 0.00465985f, lo, hi); 613 }; 614 r = fn(r); 615 g = fn(g); 616 b = fn(b); 617 } 618 619 STAGE(rgb_to_hsl) { 620 F mx = max(max(r,g), b), 621 mn = min(min(r,g), b), 622 d = mx - mn, 623 d_rcp = 1.0f / d; 624 625 F h = (1/6.0f) * 626 if_then_else(mx == mn, 0, 627 if_then_else(mx == r, (g-b)*d_rcp + if_then_else(g < b, 6.0f, 0), 628 if_then_else(mx == g, (b-r)*d_rcp + 2.0f, 629 (r-g)*d_rcp + 4.0f))); 630 631 F l = (mx + mn) * 0.5f; 632 F s = if_then_else(mx == mn, 0, 633 d / if_then_else(l > 0.5f, 2.0f-mx-mn, mx+mn)); 634 635 r = h; 636 g = s; 637 b = l; 638 } 639 STAGE(hsl_to_rgb) { 640 F h = r, 641 s = g, 642 l = b; 643 644 F q = l + if_then_else(l >= 0.5f, s - l*s, l*s), 645 p = 2.0f*l - q; 646 647 auto hue_to_rgb = [&](F t) { 648 t = fract(t); 649 650 F r = p; 651 r = if_then_else(t >= 4/6.0f, r, p + (q-p)*(4.0f - 6.0f*t)); 652 r = if_then_else(t >= 3/6.0f, r, q); 653 r = if_then_else(t >= 1/6.0f, r, p + (q-p)*( 6.0f*t)); 654 return r; 655 }; 656 657 r = if_then_else(s == 0, l, hue_to_rgb(h + (1/3.0f))); 658 g = if_then_else(s == 0, l, hue_to_rgb(h )); 659 b = if_then_else(s == 0, l, hue_to_rgb(h - (1/3.0f))); 660 } 661 662 STAGE(scale_1_float) { 663 auto c = *(const float*)ctx; 664 665 r = r * c; 666 g = g * c; 667 b = b * c; 668 a = a * c; 669 } 670 STAGE(scale_u8) { 671 auto ptr = *(const uint8_t**)ctx + x; 672 673 auto scales = load<U8>(ptr, tail); 674 auto c = from_byte(scales); 675 676 r = r * c; 677 g = g * c; 678 b = b * c; 679 a = a * c; 680 } 681 682 SI F lerp(F from, F to, F t) { 683 return mad(to-from, t, from); 684 } 685 686 STAGE(lerp_1_float) { 687 auto c = *(const float*)ctx; 688 689 r = lerp(dr, r, c); 690 g = lerp(dg, g, c); 691 b = lerp(db, b, c); 692 a = lerp(da, a, c); 693 } 694 STAGE(lerp_u8) { 695 auto ptr = *(const uint8_t**)ctx + x; 696 697 auto scales = load<U8>(ptr, tail); 698 auto c = from_byte(scales); 699 700 r = lerp(dr, r, c); 701 g = lerp(dg, g, c); 702 b = lerp(db, b, c); 703 a = lerp(da, a, c); 704 } 705 STAGE(lerp_565) { 706 auto ptr = *(const uint16_t**)ctx + x; 707 708 F cr,cg,cb; 709 from_565(load<U16>(ptr, tail), &cr, &cg, &cb); 710 711 r = lerp(dr, r, cr); 712 g = lerp(dg, g, cg); 713 b = lerp(db, b, cb); 714 a = max(lerp(da, a, cr), lerp(da, a, cg), lerp(da, a, cb)); 715 } 716 717 STAGE(load_tables) { 718 auto c = (const SkJumper_LoadTablesCtx*)ctx; 719 720 auto px = load<U32>((const uint32_t*)c->src + x, tail); 721 r = gather(c->r, (px ) & 0xff); 722 g = gather(c->g, (px >> 8) & 0xff); 723 b = gather(c->b, (px >> 16) & 0xff); 724 a = cast( (px >> 24)) * (1/255.0f); 725 } 726 STAGE(load_tables_u16_be) { 727 auto c = (const SkJumper_LoadTablesCtx*)ctx; 728 auto ptr = (const uint16_t*)c->src + 4*x; 729 730 U16 R,G,B,A; 731 load4(ptr, tail, &R,&G,&B,&A); 732 733 // c->src is big-endian, so & 0xff grabs the 8 most signficant bits. 734 r = gather(c->r, expand(R) & 0xff); 735 g = gather(c->g, expand(G) & 0xff); 736 b = gather(c->b, expand(B) & 0xff); 737 a = (1/65535.0f) * cast(expand(bswap(A))); 738 } 739 STAGE(load_tables_rgb_u16_be) { 740 auto c = (const SkJumper_LoadTablesCtx*)ctx; 741 auto ptr = (const uint16_t*)c->src + 3*x; 742 743 U16 R,G,B; 744 load3(ptr, tail, &R,&G,&B); 745 746 // c->src is big-endian, so & 0xff grabs the 8 most signficant bits. 747 r = gather(c->r, expand(R) & 0xff); 748 g = gather(c->g, expand(G) & 0xff); 749 b = gather(c->b, expand(B) & 0xff); 750 a = 1.0f; 751 } 752 753 STAGE(byte_tables) { 754 struct Tables { const uint8_t *r, *g, *b, *a; }; 755 auto tables = (const Tables*)ctx; 756 757 r = from_byte(gather(tables->r, round(r, 255.0f))); 758 g = from_byte(gather(tables->g, round(g, 255.0f))); 759 b = from_byte(gather(tables->b, round(b, 255.0f))); 760 a = from_byte(gather(tables->a, round(a, 255.0f))); 761 } 762 763 STAGE(byte_tables_rgb) { 764 struct Tables { const uint8_t *r, *g, *b; int n; }; 765 auto tables = (const Tables*)ctx; 766 767 F scale = tables->n - 1; 768 r = from_byte(gather(tables->r, round(r, scale))); 769 g = from_byte(gather(tables->g, round(g, scale))); 770 b = from_byte(gather(tables->b, round(b, scale))); 771 } 772 773 SI F table(F v, const SkJumper_TableCtx* ctx) { 774 return gather(ctx->table, round(v, ctx->size - 1)); 775 } 776 STAGE(table_r) { r = table(r, ctx); } 777 STAGE(table_g) { g = table(g, ctx); } 778 STAGE(table_b) { b = table(b, ctx); } 779 STAGE(table_a) { a = table(a, ctx); } 780 781 SI F parametric(F v, const SkJumper_ParametricTransferFunction* ctx) { 782 F r = if_then_else(v <= ctx->D, mad(ctx->C, v, ctx->F) 783 , approx_powf(mad(ctx->A, v, ctx->B), ctx->G) + ctx->E); 784 return min(max(r, 0), 1.0f); // Clamp to [0,1], with argument order mattering to handle NaN. 785 } 786 STAGE(parametric_r) { r = parametric(r, ctx); } 787 STAGE(parametric_g) { g = parametric(g, ctx); } 788 STAGE(parametric_b) { b = parametric(b, ctx); } 789 STAGE(parametric_a) { a = parametric(a, ctx); } 790 791 STAGE(lab_to_xyz) { 792 F L = r * 100.0f, 793 A = g * 255.0f - 128.0f, 794 B = b * 255.0f - 128.0f; 795 796 F Y = (L + 16.0f) * (1/116.0f), 797 X = Y + A*(1/500.0f), 798 Z = Y - B*(1/200.0f); 799 800 X = if_then_else(X*X*X > 0.008856f, X*X*X, (X - (16/116.0f)) * (1/7.787f)); 801 Y = if_then_else(Y*Y*Y > 0.008856f, Y*Y*Y, (Y - (16/116.0f)) * (1/7.787f)); 802 Z = if_then_else(Z*Z*Z > 0.008856f, Z*Z*Z, (Z - (16/116.0f)) * (1/7.787f)); 803 804 // Adjust to D50 illuminant. 805 r = X * 0.96422f; 806 g = Y ; 807 b = Z * 0.82521f; 808 } 809 810 STAGE(load_a8) { 811 auto ptr = *(const uint8_t**)ctx + x; 812 813 r = g = b = 0.0f; 814 a = from_byte(load<U8>(ptr, tail)); 815 } 816 STAGE(load_a8_dst) { 817 auto ptr = *(const uint8_t**)ctx + x; 818 819 dr = dg = db = 0.0f; 820 da = from_byte(load<U8>(ptr, tail)); 821 } 822 STAGE(gather_a8) { 823 const uint8_t* ptr; 824 U32 ix = ix_and_ptr(&ptr, ctx, r,g); 825 r = g = b = 0.0f; 826 a = from_byte(gather(ptr, ix)); 827 } 828 STAGE(store_a8) { 829 auto ptr = *(uint8_t**)ctx + x; 830 831 U8 packed = pack(pack(round(a, 255.0f))); 832 store(ptr, packed, tail); 833 } 834 835 STAGE(load_g8) { 836 auto ptr = *(const uint8_t**)ctx + x; 837 838 r = g = b = from_byte(load<U8>(ptr, tail)); 839 a = 1.0f; 840 } 841 STAGE(load_g8_dst) { 842 auto ptr = *(const uint8_t**)ctx + x; 843 844 dr = dg = db = from_byte(load<U8>(ptr, tail)); 845 da = 1.0f; 846 } 847 STAGE(gather_g8) { 848 const uint8_t* ptr; 849 U32 ix = ix_and_ptr(&ptr, ctx, r,g); 850 r = g = b = from_byte(gather(ptr, ix)); 851 a = 1.0f; 852 } 853 854 STAGE(load_565) { 855 auto ptr = *(const uint16_t**)ctx + x; 856 857 from_565(load<U16>(ptr, tail), &r,&g,&b); 858 a = 1.0f; 859 } 860 STAGE(load_565_dst) { 861 auto ptr = *(const uint16_t**)ctx + x; 862 863 from_565(load<U16>(ptr, tail), &dr,&dg,&db); 864 da = 1.0f; 865 } 866 STAGE(gather_565) { 867 const uint16_t* ptr; 868 U32 ix = ix_and_ptr(&ptr, ctx, r,g); 869 from_565(gather(ptr, ix), &r,&g,&b); 870 a = 1.0f; 871 } 872 STAGE(store_565) { 873 auto ptr = *(uint16_t**)ctx + x; 874 875 U16 px = pack( round(r, 31.0f) << 11 876 | round(g, 63.0f) << 5 877 | round(b, 31.0f) ); 878 store(ptr, px, tail); 879 } 880 881 STAGE(load_4444) { 882 auto ptr = *(const uint16_t**)ctx + x; 883 from_4444(load<U16>(ptr, tail), &r,&g,&b,&a); 884 } 885 STAGE(load_4444_dst) { 886 auto ptr = *(const uint16_t**)ctx + x; 887 from_4444(load<U16>(ptr, tail), &dr,&dg,&db,&da); 888 } 889 STAGE(gather_4444) { 890 const uint16_t* ptr; 891 U32 ix = ix_and_ptr(&ptr, ctx, r,g); 892 from_4444(gather(ptr, ix), &r,&g,&b,&a); 893 } 894 STAGE(store_4444) { 895 auto ptr = *(uint16_t**)ctx + x; 896 U16 px = pack( round(r, 15.0f) << 12 897 | round(g, 15.0f) << 8 898 | round(b, 15.0f) << 4 899 | round(a, 15.0f) ); 900 store(ptr, px, tail); 901 } 902 903 STAGE(load_8888) { 904 auto ptr = *(const uint32_t**)ctx + x; 905 from_8888(load<U32>(ptr, tail), &r,&g,&b,&a); 906 } 907 STAGE(load_8888_dst) { 908 auto ptr = *(const uint32_t**)ctx + x; 909 from_8888(load<U32>(ptr, tail), &dr,&dg,&db,&da); 910 } 911 STAGE(gather_8888) { 912 const uint32_t* ptr; 913 U32 ix = ix_and_ptr(&ptr, ctx, r,g); 914 from_8888(gather(ptr, ix), &r,&g,&b,&a); 915 } 916 STAGE(store_8888) { 917 auto ptr = *(uint32_t**)ctx + x; 918 919 U32 px = round(r, 255.0f) 920 | round(g, 255.0f) << 8 921 | round(b, 255.0f) << 16 922 | round(a, 255.0f) << 24; 923 store(ptr, px, tail); 924 } 925 926 STAGE(store_8888_2d) { 927 auto c = (const SkJumper_MemoryCtx*)ctx; 928 auto ptr = (uint32_t*)c->pixels + y*c->stride + x; 929 930 U32 px = round(r, 255.0f) 931 | round(g, 255.0f) << 8 932 | round(b, 255.0f) << 16 933 | round(a, 255.0f) << 24; 934 store(ptr, px, tail); 935 } 936 937 STAGE(load_bgra) { 938 auto ptr = *(const uint32_t**)ctx + x; 939 from_8888(load<U32>(ptr, tail), &b,&g,&r,&a); 940 } 941 STAGE(load_bgra_dst) { 942 auto ptr = *(const uint32_t**)ctx + x; 943 from_8888(load<U32>(ptr, tail), &db,&dg,&dr,&da); 944 } 945 STAGE(gather_bgra) { 946 const uint32_t* ptr; 947 U32 ix = ix_and_ptr(&ptr, ctx, r,g); 948 from_8888(gather(ptr, ix), &b,&g,&r,&a); 949 } 950 STAGE(store_bgra) { 951 auto ptr = *(uint32_t**)ctx + x; 952 953 U32 px = round(b, 255.0f) 954 | round(g, 255.0f) << 8 955 | round(r, 255.0f) << 16 956 | round(a, 255.0f) << 24; 957 store(ptr, px, tail); 958 } 959 960 STAGE(load_f16) { 961 auto ptr = *(const uint64_t**)ctx + x; 962 963 U16 R,G,B,A; 964 load4((const uint16_t*)ptr,tail, &R,&G,&B,&A); 965 r = from_half(R); 966 g = from_half(G); 967 b = from_half(B); 968 a = from_half(A); 969 } 970 STAGE(load_f16_dst) { 971 auto ptr = *(const uint64_t**)ctx + x; 972 973 U16 R,G,B,A; 974 load4((const uint16_t*)ptr,tail, &R,&G,&B,&A); 975 dr = from_half(R); 976 dg = from_half(G); 977 db = from_half(B); 978 da = from_half(A); 979 } 980 STAGE(gather_f16) { 981 const uint64_t* ptr; 982 U32 ix = ix_and_ptr(&ptr, ctx, r,g); 983 auto px = gather(ptr, ix); 984 985 U16 R,G,B,A; 986 load4((const uint16_t*)&px,0, &R,&G,&B,&A); 987 r = from_half(R); 988 g = from_half(G); 989 b = from_half(B); 990 a = from_half(A); 991 } 992 STAGE(store_f16) { 993 auto ptr = *(uint64_t**)ctx + x; 994 store4((uint16_t*)ptr,tail, to_half(r) 995 , to_half(g) 996 , to_half(b) 997 , to_half(a)); 998 } 999 1000 STAGE(load_u16_be) { 1001 auto ptr = *(const uint16_t**)ctx + 4*x; 1002 1003 U16 R,G,B,A; 1004 load4(ptr,tail, &R,&G,&B,&A); 1005 1006 r = (1/65535.0f) * cast(expand(bswap(R))); 1007 g = (1/65535.0f) * cast(expand(bswap(G))); 1008 b = (1/65535.0f) * cast(expand(bswap(B))); 1009 a = (1/65535.0f) * cast(expand(bswap(A))); 1010 } 1011 STAGE(load_rgb_u16_be) { 1012 auto ptr = *(const uint16_t**)ctx + 3*x; 1013 1014 U16 R,G,B; 1015 load3(ptr,tail, &R,&G,&B); 1016 1017 r = (1/65535.0f) * cast(expand(bswap(R))); 1018 g = (1/65535.0f) * cast(expand(bswap(G))); 1019 b = (1/65535.0f) * cast(expand(bswap(B))); 1020 a = 1.0f; 1021 } 1022 STAGE(store_u16_be) { 1023 auto ptr = *(uint16_t**)ctx + 4*x; 1024 1025 U16 R = bswap(pack(round(r, 65535.0f))), 1026 G = bswap(pack(round(g, 65535.0f))), 1027 B = bswap(pack(round(b, 65535.0f))), 1028 A = bswap(pack(round(a, 65535.0f))); 1029 1030 store4(ptr,tail, R,G,B,A); 1031 } 1032 1033 STAGE(load_f32) { 1034 auto ptr = *(const float**)ctx + 4*x; 1035 load4(ptr,tail, &r,&g,&b,&a); 1036 } 1037 STAGE(load_f32_dst) { 1038 auto ptr = *(const float**)ctx + 4*x; 1039 load4(ptr,tail, &dr,&dg,&db,&da); 1040 } 1041 STAGE(store_f32) { 1042 auto ptr = *(float**)ctx + 4*x; 1043 store4(ptr,tail, r,g,b,a); 1044 } 1045 1046 SI F ulp_before(F f) { 1047 U32 bits = -1 + unaligned_load<U32>(&f); 1048 return unaligned_load<F>(&bits); 1049 } 1050 1051 SI F exclusive_clamp(F v, const SkJumper_TileCtx* ctx) { 1052 v = max(0,v); 1053 return min(v, ulp_before(ctx->scale)); 1054 } 1055 SI F exclusive_repeat(F v, const SkJumper_TileCtx* ctx) { 1056 v = v - floor_(v*ctx->invScale)*ctx->scale; 1057 return min(v, ulp_before(ctx->scale)); 1058 } 1059 SI F exclusive_mirror(F v, const SkJumper_TileCtx* ctx) { 1060 auto limit = ctx->scale; 1061 auto invLimit = ctx->invScale; 1062 v = abs_( (v-limit) - (limit+limit)*floor_((v-limit)*(invLimit*0.5f)) - limit ); 1063 return min(v, ulp_before(limit)); 1064 } 1065 // Clamp x or y to [0,limit) == [0,limit - 1 ulp] (think, sampling from images). 1066 STAGE(clamp_x) { r = exclusive_clamp (r, (const SkJumper_TileCtx*)ctx); } 1067 STAGE(clamp_y) { g = exclusive_clamp (g, (const SkJumper_TileCtx*)ctx); } 1068 STAGE(repeat_x) { r = exclusive_repeat(r, (const SkJumper_TileCtx*)ctx); } 1069 STAGE(repeat_y) { g = exclusive_repeat(g, (const SkJumper_TileCtx*)ctx); } 1070 STAGE(mirror_x) { r = exclusive_mirror(r, (const SkJumper_TileCtx*)ctx); } 1071 STAGE(mirror_y) { g = exclusive_mirror(g, (const SkJumper_TileCtx*)ctx); } 1072 1073 // Clamp x to [0,1], both sides exclusive (think, gradients). 1074 STAGE( clamp_x_1) { r = min(max(0, r), 1.0f); } 1075 STAGE(repeat_x_1) { r = r - floor_(r); } 1076 STAGE(mirror_x_1) { r = abs_( (r-1.0f) - two(floor_((r-1.0f)*0.5f)) - 1.0f ); } 1077 1078 STAGE(luminance_to_alpha) { 1079 a = r*0.2126f + g*0.7152f + b*0.0722f; 1080 r = g = b = 0; 1081 } 1082 1083 STAGE(matrix_translate) { 1084 auto m = (const float*)ctx; 1085 1086 r += m[0]; 1087 g += m[1]; 1088 } 1089 STAGE(matrix_scale_translate) { 1090 auto m = (const float*)ctx; 1091 1092 r = mad(r,m[2], m[0]); 1093 g = mad(g,m[3], m[1]); 1094 } 1095 STAGE(matrix_2x3) { 1096 auto m = (const float*)ctx; 1097 1098 auto R = mad(r,m[0], mad(g,m[2], m[4])), 1099 G = mad(r,m[1], mad(g,m[3], m[5])); 1100 r = R; 1101 g = G; 1102 } 1103 STAGE(matrix_3x4) { 1104 auto m = (const float*)ctx; 1105 1106 auto R = mad(r,m[0], mad(g,m[3], mad(b,m[6], m[ 9]))), 1107 G = mad(r,m[1], mad(g,m[4], mad(b,m[7], m[10]))), 1108 B = mad(r,m[2], mad(g,m[5], mad(b,m[8], m[11]))); 1109 r = R; 1110 g = G; 1111 b = B; 1112 } 1113 STAGE(matrix_4x5) { 1114 auto m = (const float*)ctx; 1115 1116 auto R = mad(r,m[0], mad(g,m[4], mad(b,m[ 8], mad(a,m[12], m[16])))), 1117 G = mad(r,m[1], mad(g,m[5], mad(b,m[ 9], mad(a,m[13], m[17])))), 1118 B = mad(r,m[2], mad(g,m[6], mad(b,m[10], mad(a,m[14], m[18])))), 1119 A = mad(r,m[3], mad(g,m[7], mad(b,m[11], mad(a,m[15], m[19])))); 1120 r = R; 1121 g = G; 1122 b = B; 1123 a = A; 1124 } 1125 STAGE(matrix_4x3) { 1126 auto m = (const float*)ctx; 1127 auto X = r, 1128 Y = g; 1129 1130 r = mad(X, m[0], mad(Y, m[4], m[ 8])); 1131 g = mad(X, m[1], mad(Y, m[5], m[ 9])); 1132 b = mad(X, m[2], mad(Y, m[6], m[10])); 1133 a = mad(X, m[3], mad(Y, m[7], m[11])); 1134 } 1135 STAGE(matrix_perspective) { 1136 // N.B. Unlike the other matrix_ stages, this matrix is row-major. 1137 auto m = (const float*)ctx; 1138 1139 auto R = mad(r,m[0], mad(g,m[1], m[2])), 1140 G = mad(r,m[3], mad(g,m[4], m[5])), 1141 Z = mad(r,m[6], mad(g,m[7], m[8])); 1142 r = R * rcp(Z); 1143 g = G * rcp(Z); 1144 } 1145 1146 SI void gradient_lookup(const SkJumper_GradientCtx* c, U32 idx, F t, 1147 F* r, F* g, F* b, F* a) { 1148 F fr, br, fg, bg, fb, bb, fa, ba; 1149 #if defined(JUMPER) && defined(__AVX2__) 1150 if (c->stopCount <=8) { 1151 fr = _mm256_permutevar8x32_ps(_mm256_loadu_ps(c->fs[0]), idx); 1152 br = _mm256_permutevar8x32_ps(_mm256_loadu_ps(c->bs[0]), idx); 1153 fg = _mm256_permutevar8x32_ps(_mm256_loadu_ps(c->fs[1]), idx); 1154 bg = _mm256_permutevar8x32_ps(_mm256_loadu_ps(c->bs[1]), idx); 1155 fb = _mm256_permutevar8x32_ps(_mm256_loadu_ps(c->fs[2]), idx); 1156 bb = _mm256_permutevar8x32_ps(_mm256_loadu_ps(c->bs[2]), idx); 1157 fa = _mm256_permutevar8x32_ps(_mm256_loadu_ps(c->fs[3]), idx); 1158 ba = _mm256_permutevar8x32_ps(_mm256_loadu_ps(c->bs[3]), idx); 1159 } else 1160 #endif 1161 { 1162 fr = gather(c->fs[0], idx); 1163 br = gather(c->bs[0], idx); 1164 fg = gather(c->fs[1], idx); 1165 bg = gather(c->bs[1], idx); 1166 fb = gather(c->fs[2], idx); 1167 bb = gather(c->bs[2], idx); 1168 fa = gather(c->fs[3], idx); 1169 ba = gather(c->bs[3], idx); 1170 } 1171 1172 *r = mad(t, fr, br); 1173 *g = mad(t, fg, bg); 1174 *b = mad(t, fb, bb); 1175 *a = mad(t, fa, ba); 1176 } 1177 1178 STAGE(evenly_spaced_gradient) { 1179 auto c = (const SkJumper_GradientCtx*)ctx; 1180 auto t = r; 1181 auto idx = trunc_(t * (c->stopCount-1)); 1182 gradient_lookup(c, idx, t, &r, &g, &b, &a); 1183 } 1184 1185 STAGE(gauss_a_to_rgba) { 1186 // x = 1 - x; 1187 // exp(-x * x * 4) - 0.018f; 1188 // ... now approximate with quartic 1189 // 1190 const float c4 = -2.26661229133605957031f; 1191 const float c3 = 2.89795351028442382812f; 1192 const float c2 = 0.21345567703247070312f; 1193 const float c1 = 0.15489584207534790039f; 1194 const float c0 = 0.00030726194381713867f; 1195 a = mad(a, mad(a, mad(a, mad(a, c4, c3), c2), c1), c0); 1196 r = a; 1197 g = a; 1198 b = a; 1199 } 1200 1201 STAGE(gradient) { 1202 auto c = (const SkJumper_GradientCtx*)ctx; 1203 auto t = r; 1204 U32 idx = 0; 1205 1206 // N.B. The loop starts at 1 because idx 0 is the color to use before the first stop. 1207 for (size_t i = 1; i < c->stopCount; i++) { 1208 idx += if_then_else(t >= c->ts[i], U32(1), U32(0)); 1209 } 1210 1211 gradient_lookup(c, idx, t, &r, &g, &b, &a); 1212 } 1213 1214 STAGE(evenly_spaced_2_stop_gradient) { 1215 struct Ctx { float f[4], b[4]; }; 1216 auto c = (const Ctx*)ctx; 1217 1218 auto t = r; 1219 r = mad(t, c->f[0], c->b[0]); 1220 g = mad(t, c->f[1], c->b[1]); 1221 b = mad(t, c->f[2], c->b[2]); 1222 a = mad(t, c->f[3], c->b[3]); 1223 } 1224 1225 STAGE(xy_to_unit_angle) { 1226 F X = r, 1227 Y = g; 1228 F xabs = abs_(X), 1229 yabs = abs_(Y); 1230 1231 F slope = min(xabs, yabs)/max(xabs, yabs); 1232 F s = slope * slope; 1233 1234 // Use a 7th degree polynomial to approximate atan. 1235 // This was generated using sollya.gforge.inria.fr. 1236 // A float optimized polynomial was generated using the following command. 1237 // P1 = fpminimax((1/(2*Pi))*atan(x),[|1,3,5,7|],[|24...|],[2^(-40),1],relative); 1238 F phi = slope 1239 * (0.15912117063999176025390625f + s 1240 * (-5.185396969318389892578125e-2f + s 1241 * (2.476101927459239959716796875e-2f + s 1242 * (-7.0547382347285747528076171875e-3f)))); 1243 1244 phi = if_then_else(xabs < yabs, 1.0f/4.0f - phi, phi); 1245 phi = if_then_else(X < 0.0f , 1.0f/2.0f - phi, phi); 1246 phi = if_then_else(Y < 0.0f , 1.0f - phi , phi); 1247 phi = if_then_else(phi != phi , 0 , phi); // Check for NaN. 1248 r = phi; 1249 } 1250 1251 STAGE(xy_to_radius) { 1252 F X2 = r * r, 1253 Y2 = g * g; 1254 r = sqrt_(X2 + Y2); 1255 } 1256 1257 SI F solve_2pt_conical_quadratic(const SkJumper_2PtConicalCtx* c, F x, F y, F (*select)(F, F)) { 1258 // At this point, (x, y) is mapped into a synthetic gradient space with 1259 // the start circle centerd on (0, 0), and the end circle centered on (1, 0) 1260 // (see the stage setup). 1261 // 1262 // We're searching along X-axis for x', such that 1263 // 1264 // 1) r(x') is a linear interpolation between r0 and r1 1265 // 2) (x, y) is on the circle C(x', 0, r(x')) 1266 // 1267 // Solving this system boils down to a quadratic equation with coefficients 1268 // 1269 // a = 1 - (r1 - r0)^2 <- constant, precomputed in ctx->fCoeffA) 1270 // 1271 // b = -2 * (x + (r1 - r0) * r0) 1272 // 1273 // c = x^2 + y^2 - r0^2 1274 // 1275 // Since the start/end circle centers are the extremes of the [0, 1] interval 1276 // on the X axis, the solution (x') is exactly the t we are looking for. 1277 1278 const F coeffA = c->fCoeffA, 1279 coeffB = -2 * (x + c->fDR*c->fR0), 1280 coeffC = x*x + y*y - c->fR0*c->fR0; 1281 1282 const F disc = mad(coeffB, coeffB, -4 * coeffA * coeffC); 1283 const F sqrt_disc = sqrt_(disc); 1284 1285 const F invCoeffA = c->fInvCoeffA; 1286 return select((-coeffB + sqrt_disc) * (invCoeffA * 0.5f), 1287 (-coeffB - sqrt_disc) * (invCoeffA * 0.5f)); 1288 } 1289 1290 STAGE(xy_to_2pt_conical_quadratic_max) { 1291 r = solve_2pt_conical_quadratic(ctx, r, g, max); 1292 } 1293 1294 STAGE(xy_to_2pt_conical_quadratic_min) { 1295 r = solve_2pt_conical_quadratic(ctx, r, g, min); 1296 } 1297 1298 STAGE(xy_to_2pt_conical_linear) { 1299 auto* c = (const SkJumper_2PtConicalCtx*)ctx; 1300 1301 const F coeffB = -2 * (r + c->fDR*c->fR0), 1302 coeffC = r*r + g*g - c->fR0*c->fR0; 1303 1304 r = -coeffC / coeffB; 1305 } 1306 1307 STAGE(mask_2pt_conical_degenerates) { 1308 auto* c = (SkJumper_2PtConicalCtx*)ctx; 1309 1310 // Compute and save a mask for degenerate values. 1311 U32 mask = 0xffffffff; 1312 1313 // TODO: mtklein kindly volunteered to revisit this at some point. 1314 #if defined(JUMPER) 1315 // Vector comparisons set all bits, so we can use something like this. 1316 mask = mask & (mad(r, c->fDR, c->fR0) >= 0); // R(t) >= 0 1317 mask = mask & (r == r); // t != NaN 1318 #else 1319 // The portable version is more involved, 'cause we only get one bit back. 1320 mask = mask & if_then_else(mad(r, c->fDR, c->fR0) >= 0, U32(0xffffffff), U32(0)); // R(t) >= 0 1321 mask = mask & if_then_else(r == r, U32(0xffffffff), U32(0)); // t != NaN 1322 #endif 1323 1324 unaligned_store(&c->fMask, mask); 1325 } 1326 1327 STAGE(apply_vector_mask) { 1328 const U32 mask = unaligned_load<U32>((const uint32_t*)ctx); 1329 r = bit_cast<F>(bit_cast<U32>(r) & mask); 1330 g = bit_cast<F>(bit_cast<U32>(g) & mask); 1331 b = bit_cast<F>(bit_cast<U32>(b) & mask); 1332 a = bit_cast<F>(bit_cast<U32>(a) & mask); 1333 } 1334 1335 STAGE(save_xy) { 1336 auto c = (SkJumper_SamplerCtx*)ctx; 1337 1338 // Whether bilinear or bicubic, all sample points are at the same fractional offset (fx,fy). 1339 // They're either the 4 corners of a logical 1x1 pixel or the 16 corners of a 3x3 grid 1340 // surrounding (x,y) at (0.5,0.5) off-center. 1341 F fx = fract(r + 0.5f), 1342 fy = fract(g + 0.5f); 1343 1344 // Samplers will need to load x and fx, or y and fy. 1345 unaligned_store(c->x, r); 1346 unaligned_store(c->y, g); 1347 unaligned_store(c->fx, fx); 1348 unaligned_store(c->fy, fy); 1349 } 1350 1351 STAGE(accumulate) { 1352 auto c = (const SkJumper_SamplerCtx*)ctx; 1353 1354 // Bilinear and bicubic filters are both separable, so we produce independent contributions 1355 // from x and y, multiplying them together here to get each pixel's total scale factor. 1356 auto scale = unaligned_load<F>(c->scalex) 1357 * unaligned_load<F>(c->scaley); 1358 dr = mad(scale, r, dr); 1359 dg = mad(scale, g, dg); 1360 db = mad(scale, b, db); 1361 da = mad(scale, a, da); 1362 } 1363 1364 // In bilinear interpolation, the 4 pixels at +/- 0.5 offsets from the sample pixel center 1365 // are combined in direct proportion to their area overlapping that logical query pixel. 1366 // At positive offsets, the x-axis contribution to that rectangle is fx, or (1-fx) at negative x. 1367 // The y-axis is symmetric. 1368 1369 template <int kScale> 1370 SI void bilinear_x(SkJumper_SamplerCtx* ctx, F* x) { 1371 *x = unaligned_load<F>(ctx->x) + (kScale * 0.5f); 1372 F fx = unaligned_load<F>(ctx->fx); 1373 1374 F scalex; 1375 if (kScale == -1) { scalex = 1.0f - fx; } 1376 if (kScale == +1) { scalex = fx; } 1377 unaligned_store(ctx->scalex, scalex); 1378 } 1379 template <int kScale> 1380 SI void bilinear_y(SkJumper_SamplerCtx* ctx, F* y) { 1381 *y = unaligned_load<F>(ctx->y) + (kScale * 0.5f); 1382 F fy = unaligned_load<F>(ctx->fy); 1383 1384 F scaley; 1385 if (kScale == -1) { scaley = 1.0f - fy; } 1386 if (kScale == +1) { scaley = fy; } 1387 unaligned_store(ctx->scaley, scaley); 1388 } 1389 1390 STAGE(bilinear_nx) { bilinear_x<-1>(ctx, &r); } 1391 STAGE(bilinear_px) { bilinear_x<+1>(ctx, &r); } 1392 STAGE(bilinear_ny) { bilinear_y<-1>(ctx, &g); } 1393 STAGE(bilinear_py) { bilinear_y<+1>(ctx, &g); } 1394 1395 1396 // In bicubic interpolation, the 16 pixels and +/- 0.5 and +/- 1.5 offsets from the sample 1397 // pixel center are combined with a non-uniform cubic filter, with higher values near the center. 1398 // 1399 // We break this function into two parts, one for near 0.5 offsets and one for far 1.5 offsets. 1400 // See GrCubicEffect for details of this particular filter. 1401 1402 SI F bicubic_near(F t) { 1403 // 1/18 + 9/18t + 27/18t^2 - 21/18t^3 == t ( t ( -21/18t + 27/18) + 9/18) + 1/18 1404 return mad(t, mad(t, mad((-21/18.0f), t, (27/18.0f)), (9/18.0f)), (1/18.0f)); 1405 } 1406 SI F bicubic_far(F t) { 1407 // 0/18 + 0/18*t - 6/18t^2 + 7/18t^3 == t^2 (7/18t - 6/18) 1408 return (t*t)*mad((7/18.0f), t, (-6/18.0f)); 1409 } 1410 1411 template <int kScale> 1412 SI void bicubic_x(SkJumper_SamplerCtx* ctx, F* x) { 1413 *x = unaligned_load<F>(ctx->x) + (kScale * 0.5f); 1414 F fx = unaligned_load<F>(ctx->fx); 1415 1416 F scalex; 1417 if (kScale == -3) { scalex = bicubic_far (1.0f - fx); } 1418 if (kScale == -1) { scalex = bicubic_near(1.0f - fx); } 1419 if (kScale == +1) { scalex = bicubic_near( fx); } 1420 if (kScale == +3) { scalex = bicubic_far ( fx); } 1421 unaligned_store(ctx->scalex, scalex); 1422 } 1423 template <int kScale> 1424 SI void bicubic_y(SkJumper_SamplerCtx* ctx, F* y) { 1425 *y = unaligned_load<F>(ctx->y) + (kScale * 0.5f); 1426 F fy = unaligned_load<F>(ctx->fy); 1427 1428 F scaley; 1429 if (kScale == -3) { scaley = bicubic_far (1.0f - fy); } 1430 if (kScale == -1) { scaley = bicubic_near(1.0f - fy); } 1431 if (kScale == +1) { scaley = bicubic_near( fy); } 1432 if (kScale == +3) { scaley = bicubic_far ( fy); } 1433 unaligned_store(ctx->scaley, scaley); 1434 } 1435 1436 STAGE(bicubic_n3x) { bicubic_x<-3>(ctx, &r); } 1437 STAGE(bicubic_n1x) { bicubic_x<-1>(ctx, &r); } 1438 STAGE(bicubic_p1x) { bicubic_x<+1>(ctx, &r); } 1439 STAGE(bicubic_p3x) { bicubic_x<+3>(ctx, &r); } 1440 1441 STAGE(bicubic_n3y) { bicubic_y<-3>(ctx, &g); } 1442 STAGE(bicubic_n1y) { bicubic_y<-1>(ctx, &g); } 1443 STAGE(bicubic_p1y) { bicubic_y<+1>(ctx, &g); } 1444 STAGE(bicubic_p3y) { bicubic_y<+3>(ctx, &g); } 1445 1446 STAGE(callback) { 1447 auto c = (SkJumper_CallbackCtx*)ctx; 1448 store4(c->rgba,0, r,g,b,a); 1449 c->fn(c, tail ? tail : kStride); 1450 load4(c->read_from,0, &r,&g,&b,&a); 1451 } 1452