1 2 /* filter_msa_intrinsics.c - MSA optimised filter functions 3 * 4 * Copyright (c) 2016 Glenn Randers-Pehrson 5 * Written by Mandar Sahastrabuddhe, August 2016. 6 * Last changed in libpng 1.6.25 [September 1, 2016] 7 * 8 * This code is released under the libpng license. 9 * For conditions of distribution and use, see the disclaimer 10 * and license in png.h 11 */ 12 #include <stdio.h> 13 #include <stdint.h> 14 #include "../pngpriv.h" 15 16 #ifdef PNG_READ_SUPPORTED 17 18 /* This code requires -mfpu=msa on the command line: */ 19 #if PNG_MIPS_MSA_IMPLEMENTATION == 1 /* intrinsics code from pngpriv.h */ 20 21 #include <msa.h> 22 23 /* libpng row pointers are not necessarily aligned to any particular boundary, 24 * however this code will only work with appropriate alignment. mips/mips_init.c 25 * checks for this (and will not compile unless it is done). This code uses 26 * variants of png_aligncast to avoid compiler warnings. 27 */ 28 #define png_ptr(type,pointer) png_aligncast(type *,pointer) 29 #define png_ptrc(type,pointer) png_aligncastconst(const type *,pointer) 30 31 /* The following relies on a variable 'temp_pointer' being declared with type 32 * 'type'. This is written this way just to hide the GCC strict aliasing 33 * warning; note that the code is safe because there never is an alias between 34 * the input and output pointers. 35 */ 36 #define png_ldr(type,pointer)\ 37 (temp_pointer = png_ptr(type,pointer), *temp_pointer) 38 39 #if PNG_MIPS_MSA_OPT > 0 40 41 #ifdef CLANG_BUILD 42 #define MSA_SRLI_B(a, b) __msa_srli_b((v16i8) a, b) 43 44 #define LW(psrc) \ 45 ( { \ 46 uint8_t *psrc_lw_m = (uint8_t *) (psrc); \ 47 uint32_t val_m; \ 48 \ 49 asm volatile ( \ 50 "lw %[val_m], %[psrc_lw_m] \n\t" \ 51 \ 52 : [val_m] "=r" (val_m) \ 53 : [psrc_lw_m] "m" (*psrc_lw_m) \ 54 ); \ 55 \ 56 val_m; \ 57 } ) 58 59 #define SH(val, pdst) \ 60 { \ 61 uint8_t *pdst_sh_m = (uint8_t *) (pdst); \ 62 uint16_t val_m = (val); \ 63 \ 64 asm volatile ( \ 65 "sh %[val_m], %[pdst_sh_m] \n\t" \ 66 \ 67 : [pdst_sh_m] "=m" (*pdst_sh_m) \ 68 : [val_m] "r" (val_m) \ 69 ); \ 70 } 71 72 #define SW(val, pdst) \ 73 { \ 74 uint8_t *pdst_sw_m = (uint8_t *) (pdst); \ 75 uint32_t val_m = (val); \ 76 \ 77 asm volatile ( \ 78 "sw %[val_m], %[pdst_sw_m] \n\t" \ 79 \ 80 : [pdst_sw_m] "=m" (*pdst_sw_m) \ 81 : [val_m] "r" (val_m) \ 82 ); \ 83 } 84 85 #if (__mips == 64) 86 #define SD(val, pdst) \ 87 { \ 88 uint8_t *pdst_sd_m = (uint8_t *) (pdst); \ 89 uint64_t val_m = (val); \ 90 \ 91 asm volatile ( \ 92 "sd %[val_m], %[pdst_sd_m] \n\t" \ 93 \ 94 : [pdst_sd_m] "=m" (*pdst_sd_m) \ 95 : [val_m] "r" (val_m) \ 96 ); \ 97 } 98 #else 99 #define SD(val, pdst) \ 100 { \ 101 uint8_t *pdst_sd_m = (uint8_t *) (pdst); \ 102 uint32_t val0_m, val1_m; \ 103 \ 104 val0_m = (uint32_t) ((val) & 0x00000000FFFFFFFF); \ 105 val1_m = (uint32_t) (((val) >> 32) & 0x00000000FFFFFFFF); \ 106 \ 107 SW(val0_m, pdst_sd_m); \ 108 SW(val1_m, pdst_sd_m + 4); \ 109 } 110 #endif 111 #else 112 #define MSA_SRLI_B(a, b) (a >> b) 113 114 #if (__mips_isa_rev >= 6) 115 #define LW(psrc) \ 116 ( { \ 117 uint8_t *psrc_lw_m = (uint8_t *) (psrc); \ 118 uint32_t val_m; \ 119 \ 120 asm volatile ( \ 121 "lw %[val_m], %[psrc_lw_m] \n\t" \ 122 \ 123 : [val_m] "=r" (val_m) \ 124 : [psrc_lw_m] "m" (*psrc_lw_m) \ 125 ); \ 126 \ 127 val_m; \ 128 } ) 129 130 #define SH(val, pdst) \ 131 { \ 132 uint8_t *pdst_sh_m = (uint8_t *) (pdst); \ 133 uint16_t val_m = (val); \ 134 \ 135 asm volatile ( \ 136 "sh %[val_m], %[pdst_sh_m] \n\t" \ 137 \ 138 : [pdst_sh_m] "=m" (*pdst_sh_m) \ 139 : [val_m] "r" (val_m) \ 140 ); \ 141 } 142 143 #define SW(val, pdst) \ 144 { \ 145 uint8_t *pdst_sw_m = (uint8_t *) (pdst); \ 146 uint32_t val_m = (val); \ 147 \ 148 asm volatile ( \ 149 "sw %[val_m], %[pdst_sw_m] \n\t" \ 150 \ 151 : [pdst_sw_m] "=m" (*pdst_sw_m) \ 152 : [val_m] "r" (val_m) \ 153 ); \ 154 } 155 156 #if (__mips == 64) 157 #define SD(val, pdst) \ 158 { \ 159 uint8_t *pdst_sd_m = (uint8_t *) (pdst); \ 160 uint64_t val_m = (val); \ 161 \ 162 asm volatile ( \ 163 "sd %[val_m], %[pdst_sd_m] \n\t" \ 164 \ 165 : [pdst_sd_m] "=m" (*pdst_sd_m) \ 166 : [val_m] "r" (val_m) \ 167 ); \ 168 } 169 #else 170 #define SD(val, pdst) \ 171 { \ 172 uint8_t *pdst_sd_m = (uint8_t *) (pdst); \ 173 uint32_t val0_m, val1_m; \ 174 \ 175 val0_m = (uint32_t) ((val) & 0x00000000FFFFFFFF); \ 176 val1_m = (uint32_t) (((val) >> 32) & 0x00000000FFFFFFFF); \ 177 \ 178 SW(val0_m, pdst_sd_m); \ 179 SW(val1_m, pdst_sd_m + 4); \ 180 } 181 #endif 182 #else // !(__mips_isa_rev >= 6) 183 #define LW(psrc) \ 184 ( { \ 185 uint8_t *psrc_lw_m = (uint8_t *) (psrc); \ 186 uint32_t val_m; \ 187 \ 188 asm volatile ( \ 189 "ulw %[val_m], %[psrc_lw_m] \n\t" \ 190 \ 191 : [val_m] "=r" (val_m) \ 192 : [psrc_lw_m] "m" (*psrc_lw_m) \ 193 ); \ 194 \ 195 val_m; \ 196 } ) 197 198 #define SH(val, pdst) \ 199 { \ 200 uint8_t *pdst_sh_m = (uint8_t *) (pdst); \ 201 uint16_t val_m = (val); \ 202 \ 203 asm volatile ( \ 204 "ush %[val_m], %[pdst_sh_m] \n\t" \ 205 \ 206 : [pdst_sh_m] "=m" (*pdst_sh_m) \ 207 : [val_m] "r" (val_m) \ 208 ); \ 209 } 210 211 #define SW(val, pdst) \ 212 { \ 213 uint8_t *pdst_sw_m = (uint8_t *) (pdst); \ 214 uint32_t val_m = (val); \ 215 \ 216 asm volatile ( \ 217 "usw %[val_m], %[pdst_sw_m] \n\t" \ 218 \ 219 : [pdst_sw_m] "=m" (*pdst_sw_m) \ 220 : [val_m] "r" (val_m) \ 221 ); \ 222 } 223 224 #define SD(val, pdst) \ 225 { \ 226 uint8_t *pdst_sd_m = (uint8_t *) (pdst); \ 227 uint32_t val0_m, val1_m; \ 228 \ 229 val0_m = (uint32_t) ((val) & 0x00000000FFFFFFFF); \ 230 val1_m = (uint32_t) (((val) >> 32) & 0x00000000FFFFFFFF); \ 231 \ 232 SW(val0_m, pdst_sd_m); \ 233 SW(val1_m, pdst_sd_m + 4); \ 234 } 235 236 #define SW_ZERO(pdst) \ 237 { \ 238 uint8_t *pdst_m = (uint8_t *) (pdst); \ 239 \ 240 asm volatile ( \ 241 "usw $0, %[pdst_m] \n\t" \ 242 \ 243 : [pdst_m] "=m" (*pdst_m) \ 244 : \ 245 ); \ 246 } 247 #endif // (__mips_isa_rev >= 6) 248 #endif 249 250 #define LD_B(RTYPE, psrc) *((RTYPE *) (psrc)) 251 #define LD_UB(...) LD_B(v16u8, __VA_ARGS__) 252 #define LD_B2(RTYPE, psrc, stride, out0, out1) \ 253 { \ 254 out0 = LD_B(RTYPE, (psrc)); \ 255 out1 = LD_B(RTYPE, (psrc) + stride); \ 256 } 257 #define LD_UB2(...) LD_B2(v16u8, __VA_ARGS__) 258 #define LD_B4(RTYPE, psrc, stride, out0, out1, out2, out3) \ 259 { \ 260 LD_B2(RTYPE, (psrc), stride, out0, out1); \ 261 LD_B2(RTYPE, (psrc) + 2 * stride , stride, out2, out3); \ 262 } 263 #define LD_UB4(...) LD_B4(v16u8, __VA_ARGS__) 264 265 #define ST_B(RTYPE, in, pdst) *((RTYPE *) (pdst)) = (in) 266 #define ST_UB(...) ST_B(v16u8, __VA_ARGS__) 267 #define ST_B2(RTYPE, in0, in1, pdst, stride) \ 268 { \ 269 ST_B(RTYPE, in0, (pdst)); \ 270 ST_B(RTYPE, in1, (pdst) + stride); \ 271 } 272 #define ST_UB2(...) ST_B2(v16u8, __VA_ARGS__) 273 #define ST_B4(RTYPE, in0, in1, in2, in3, pdst, stride) \ 274 { \ 275 ST_B2(RTYPE, in0, in1, (pdst), stride); \ 276 ST_B2(RTYPE, in2, in3, (pdst) + 2 * stride, stride); \ 277 } 278 #define ST_UB4(...) ST_B4(v16u8, __VA_ARGS__) 279 280 #define ADD2(in0, in1, in2, in3, out0, out1) \ 281 { \ 282 out0 = in0 + in1; \ 283 out1 = in2 + in3; \ 284 } 285 #define ADD3(in0, in1, in2, in3, in4, in5, \ 286 out0, out1, out2) \ 287 { \ 288 ADD2(in0, in1, in2, in3, out0, out1); \ 289 out2 = in4 + in5; \ 290 } 291 #define ADD4(in0, in1, in2, in3, in4, in5, in6, in7, \ 292 out0, out1, out2, out3) \ 293 { \ 294 ADD2(in0, in1, in2, in3, out0, out1); \ 295 ADD2(in4, in5, in6, in7, out2, out3); \ 296 } 297 298 #define ILVR_B2(RTYPE, in0, in1, in2, in3, out0, out1) \ 299 { \ 300 out0 = (RTYPE) __msa_ilvr_b((v16i8) in0, (v16i8) in1); \ 301 out1 = (RTYPE) __msa_ilvr_b((v16i8) in2, (v16i8) in3); \ 302 } 303 #define ILVR_B2_SH(...) ILVR_B2(v8i16, __VA_ARGS__) 304 305 #define HSUB_UB2(RTYPE, in0, in1, out0, out1) \ 306 { \ 307 out0 = (RTYPE) __msa_hsub_u_h((v16u8) in0, (v16u8) in0); \ 308 out1 = (RTYPE) __msa_hsub_u_h((v16u8) in1, (v16u8) in1); \ 309 } 310 #define HSUB_UB2_SH(...) HSUB_UB2(v8i16, __VA_ARGS__) 311 312 #define SLDI_B2_0(RTYPE, in0, in1, out0, out1, slide_val) \ 313 { \ 314 v16i8 zero_m = { 0 }; \ 315 out0 = (RTYPE) __msa_sldi_b((v16i8) zero_m, (v16i8) in0, slide_val); \ 316 out1 = (RTYPE) __msa_sldi_b((v16i8) zero_m, (v16i8) in1, slide_val); \ 317 } 318 #define SLDI_B2_0_UB(...) SLDI_B2_0(v16u8, __VA_ARGS__) 319 320 #define SLDI_B3_0(RTYPE, in0, in1, in2, out0, out1, out2, slide_val) \ 321 { \ 322 v16i8 zero_m = { 0 }; \ 323 SLDI_B2_0(RTYPE, in0, in1, out0, out1, slide_val); \ 324 out2 = (RTYPE) __msa_sldi_b((v16i8) zero_m, (v16i8) in2, slide_val); \ 325 } 326 #define SLDI_B3_0_UB(...) SLDI_B3_0(v16u8, __VA_ARGS__) 327 328 #define ILVEV_W2(RTYPE, in0, in1, in2, in3, out0, out1) \ 329 { \ 330 out0 = (RTYPE) __msa_ilvev_w((v4i32) in1, (v4i32) in0); \ 331 out1 = (RTYPE) __msa_ilvev_w((v4i32) in3, (v4i32) in2); \ 332 } 333 #define ILVEV_W2_UB(...) ILVEV_W2(v16u8, __VA_ARGS__) 334 335 #define ADD_ABS_H3(RTYPE, in0, in1, in2, out0, out1, out2) \ 336 { \ 337 RTYPE zero = {0}; \ 338 \ 339 out0 = __msa_add_a_h((v8i16) zero, in0); \ 340 out1 = __msa_add_a_h((v8i16) zero, in1); \ 341 out2 = __msa_add_a_h((v8i16) zero, in2); \ 342 } 343 #define ADD_ABS_H3_SH(...) ADD_ABS_H3(v8i16, __VA_ARGS__) 344 345 #define VSHF_B2(RTYPE, in0, in1, in2, in3, mask0, mask1, out0, out1) \ 346 { \ 347 out0 = (RTYPE) __msa_vshf_b((v16i8) mask0, (v16i8) in1, (v16i8) in0); \ 348 out1 = (RTYPE) __msa_vshf_b((v16i8) mask1, (v16i8) in3, (v16i8) in2); \ 349 } 350 #define VSHF_B2_UB(...) VSHF_B2(v16u8, __VA_ARGS__) 351 352 #define CMP_AND_SELECT(inp0, inp1, inp2, inp3, inp4, inp5, out0) \ 353 { \ 354 v8i16 _sel_h0, _sel_h1; \ 355 v16u8 _sel_b0, _sel_b1; \ 356 _sel_h0 = (v8i16) __msa_clt_u_h((v8u16) inp1, (v8u16) inp0); \ 357 _sel_b0 = (v16u8) __msa_pckev_b((v16i8) _sel_h0, (v16i8) _sel_h0); \ 358 inp0 = (v8i16) __msa_bmnz_v((v16u8) inp0, (v16u8) inp1, (v16u8) _sel_h0); \ 359 inp4 = (v16u8) __msa_bmnz_v(inp3, inp4, _sel_b0); \ 360 _sel_h1 = (v8i16) __msa_clt_u_h((v8u16) inp2, (v8u16) inp0); \ 361 _sel_b1 = (v16u8) __msa_pckev_b((v16i8) _sel_h1, (v16i8) _sel_h1); \ 362 inp4 = (v16u8) __msa_bmnz_v(inp4, inp5, _sel_b1); \ 363 out0 += inp4; \ 364 } 365 366 void png_read_filter_row_up_msa(png_row_infop row_info, png_bytep row, 367 png_const_bytep prev_row) 368 { 369 png_size_t i, cnt, cnt16, cnt32; 370 png_size_t istop = row_info->rowbytes; 371 png_bytep rp = row; 372 png_const_bytep pp = prev_row; 373 v16u8 src0, src1, src2, src3, src4, src5, src6, src7; 374 375 for (i = 0; i < (istop >> 6); i++) 376 { 377 LD_UB4(rp, 16, src0, src1, src2, src3); 378 LD_UB4(pp, 16, src4, src5, src6, src7); 379 pp += 64; 380 381 ADD4(src0, src4, src1, src5, src2, src6, src3, src7, 382 src0, src1, src2, src3); 383 384 ST_UB4(src0, src1, src2, src3, rp, 16); 385 rp += 64; 386 } 387 388 if (istop & 0x3F) 389 { 390 cnt32 = istop & 0x20; 391 cnt16 = istop & 0x10; 392 cnt = istop & 0xF; 393 394 if(cnt32) 395 { 396 if (cnt16 && cnt) 397 { 398 LD_UB4(rp, 16, src0, src1, src2, src3); 399 LD_UB4(pp, 16, src4, src5, src6, src7); 400 401 ADD4(src0, src4, src1, src5, src2, src6, src3, src7, 402 src0, src1, src2, src3); 403 404 ST_UB4(src0, src1, src2, src3, rp, 16); 405 rp += 64; 406 } 407 else if (cnt16 || cnt) 408 { 409 LD_UB2(rp, 16, src0, src1); 410 LD_UB2(pp, 16, src4, src5); 411 pp += 32; 412 src2 = LD_UB(rp + 32); 413 src6 = LD_UB(pp); 414 415 ADD3(src0, src4, src1, src5, src2, src6, src0, src1, src2); 416 417 ST_UB2(src0, src1, rp, 16); 418 rp += 32; 419 ST_UB(src2, rp); 420 rp += 16; 421 } 422 else 423 { 424 LD_UB2(rp, 16, src0, src1); 425 LD_UB2(pp, 16, src4, src5); 426 427 ADD2(src0, src4, src1, src5, src0, src1); 428 429 ST_UB2(src0, src1, rp, 16); 430 rp += 32; 431 } 432 } 433 else if (cnt16 && cnt) 434 { 435 LD_UB2(rp, 16, src0, src1); 436 LD_UB2(pp, 16, src4, src5); 437 438 ADD2(src0, src4, src1, src5, src0, src1); 439 440 ST_UB2(src0, src1, rp, 16); 441 rp += 32; 442 } 443 else if (cnt16 || cnt) 444 { 445 src0 = LD_UB(rp); 446 src4 = LD_UB(pp); 447 pp += 16; 448 449 src0 += src4; 450 451 ST_UB(src0, rp); 452 rp += 16; 453 } 454 } 455 } 456 457 void png_read_filter_row_sub4_msa(png_row_infop row_info, png_bytep row, 458 png_const_bytep prev_row) 459 { 460 png_size_t count; 461 png_size_t istop = row_info->rowbytes; 462 png_bytep src = row; 463 png_bytep nxt = row + 4; 464 int32_t inp0; 465 v16u8 src0, src1, src2, src3, src4; 466 v16u8 dst0, dst1; 467 v16u8 zero = { 0 }; 468 469 istop -= 4; 470 471 inp0 = LW(src); 472 src += 4; 473 src0 = (v16u8) __msa_insert_w((v4i32) zero, 0, inp0); 474 475 for (count = 0; count < istop; count += 16) 476 { 477 src1 = LD_UB(src); 478 src += 16; 479 480 src2 = (v16u8) __msa_sldi_b((v16i8) zero, (v16i8) src1, 4); 481 src3 = (v16u8) __msa_sldi_b((v16i8) zero, (v16i8) src1, 8); 482 src4 = (v16u8) __msa_sldi_b((v16i8) zero, (v16i8) src1, 12); 483 src1 += src0; 484 src2 += src1; 485 src3 += src2; 486 src4 += src3; 487 src0 = src4; 488 ILVEV_W2_UB(src1, src2, src3, src4, dst0, dst1); 489 dst0 = (v16u8) __msa_pckev_d((v2i64) dst1, (v2i64) dst0); 490 491 ST_UB(dst0, nxt); 492 nxt += 16; 493 } 494 } 495 496 void png_read_filter_row_sub3_msa(png_row_infop row_info, png_bytep row, 497 png_const_bytep prev_row) 498 { 499 png_size_t count; 500 png_size_t istop = row_info->rowbytes; 501 png_bytep src = row; 502 png_bytep nxt = row + 3; 503 int64_t out0; 504 int32_t inp0, out1; 505 v16u8 src0, src1, src2, src3, src4, dst0, dst1; 506 v16u8 zero = { 0 }; 507 v16i8 mask0 = { 0, 1, 2, 16, 17, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 }; 508 v16i8 mask1 = { 0, 1, 2, 3, 4, 5, 16, 17, 18, 19, 20, 21, 0, 0, 0, 0 }; 509 510 istop -= 3; 511 512 inp0 = LW(src); 513 src += 3; 514 src0 = (v16u8) __msa_insert_w((v4i32) zero, 0, inp0); 515 516 for (count = 0; count < istop; count += 12) 517 { 518 src1 = LD_UB(src); 519 src += 12; 520 521 src2 = (v16u8) __msa_sldi_b((v16i8) zero, (v16i8) src1, 3); 522 src3 = (v16u8) __msa_sldi_b((v16i8) zero, (v16i8) src1, 6); 523 src4 = (v16u8) __msa_sldi_b((v16i8) zero, (v16i8) src1, 9); 524 src1 += src0; 525 src2 += src1; 526 src3 += src2; 527 src4 += src3; 528 src0 = src4; 529 VSHF_B2_UB(src1, src2, src3, src4, mask0, mask0, dst0, dst1); 530 dst0 = (v16u8) __msa_vshf_b(mask1, (v16i8) dst1, (v16i8) dst0); 531 out0 = __msa_copy_s_d((v2i64) dst0, 0); 532 out1 = __msa_copy_s_w((v4i32) dst0, 2); 533 534 SD(out0, nxt); 535 nxt += 8; 536 SW(out1, nxt); 537 nxt += 4; 538 } 539 } 540 541 void png_read_filter_row_avg4_msa(png_row_infop row_info, png_bytep row, 542 png_const_bytep prev_row) 543 { 544 png_size_t i; 545 png_bytep src = row; 546 png_bytep nxt = row; 547 png_const_bytep pp = prev_row; 548 png_size_t istop = row_info->rowbytes - 4; 549 int32_t inp0, inp1, out0; 550 v16u8 src0, src1, src2, src3, src4, src5, src6, src7, src8, src9, dst0, dst1; 551 v16u8 zero = { 0 }; 552 553 inp0 = LW(pp); 554 pp += 4; 555 inp1 = LW(src); 556 src += 4; 557 src0 = (v16u8) __msa_insert_w((v4i32) zero, 0, inp0); 558 src1 = (v16u8) __msa_insert_w((v4i32) zero, 0, inp1); 559 src0 = (v16u8) MSA_SRLI_B(src0, 1); 560 src1 += src0; 561 out0 = __msa_copy_s_w((v4i32) src1, 0); 562 SW(out0, nxt); 563 nxt += 4; 564 565 for (i = 0; i < istop; i += 16) 566 { 567 src2 = LD_UB(pp); 568 pp += 16; 569 src6 = LD_UB(src); 570 src += 16; 571 572 SLDI_B2_0_UB(src2, src6, src3, src7, 4); 573 SLDI_B2_0_UB(src2, src6, src4, src8, 8); 574 SLDI_B2_0_UB(src2, src6, src5, src9, 12); 575 src2 = __msa_ave_u_b(src2, src1); 576 src6 += src2; 577 src3 = __msa_ave_u_b(src3, src6); 578 src7 += src3; 579 src4 = __msa_ave_u_b(src4, src7); 580 src8 += src4; 581 src5 = __msa_ave_u_b(src5, src8); 582 src9 += src5; 583 src1 = src9; 584 ILVEV_W2_UB(src6, src7, src8, src9, dst0, dst1); 585 dst0 = (v16u8) __msa_pckev_d((v2i64) dst1, (v2i64) dst0); 586 587 ST_UB(dst0, nxt); 588 nxt += 16; 589 } 590 } 591 592 void png_read_filter_row_avg3_msa(png_row_infop row_info, png_bytep row, 593 png_const_bytep prev_row) 594 { 595 png_size_t i; 596 png_bytep src = row; 597 png_bytep nxt = row; 598 png_const_bytep pp = prev_row; 599 png_size_t istop = row_info->rowbytes - 3; 600 int64_t out0; 601 int32_t inp0, inp1, out1; 602 int16_t out2; 603 v16u8 src0, src1, src2, src3, src4, src5, src6, src7, src8, src9, dst0, dst1; 604 v16u8 zero = { 0 }; 605 v16i8 mask0 = { 0, 1, 2, 16, 17, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 }; 606 v16i8 mask1 = { 0, 1, 2, 3, 4, 5, 16, 17, 18, 19, 20, 21, 0, 0, 0, 0 }; 607 608 inp0 = LW(pp); 609 pp += 3; 610 inp1 = LW(src); 611 src += 3; 612 src0 = (v16u8) __msa_insert_w((v4i32) zero, 0, inp0); 613 src1 = (v16u8) __msa_insert_w((v4i32) zero, 0, inp1); 614 src0 = (v16u8) MSA_SRLI_B(src0, 1); 615 src1 += src0; 616 out2 = __msa_copy_s_h((v8i16) src1, 0); 617 SH(out2, nxt); 618 nxt += 2; 619 nxt[0] = src1[2]; 620 nxt++; 621 622 for (i = 0; i < istop; i += 12) 623 { 624 src2 = LD_UB(pp); 625 pp += 12; 626 src6 = LD_UB(src); 627 src += 12; 628 629 SLDI_B2_0_UB(src2, src6, src3, src7, 3); 630 SLDI_B2_0_UB(src2, src6, src4, src8, 6); 631 SLDI_B2_0_UB(src2, src6, src5, src9, 9); 632 src2 = __msa_ave_u_b(src2, src1); 633 src6 += src2; 634 src3 = __msa_ave_u_b(src3, src6); 635 src7 += src3; 636 src4 = __msa_ave_u_b(src4, src7); 637 src8 += src4; 638 src5 = __msa_ave_u_b(src5, src8); 639 src9 += src5; 640 src1 = src9; 641 VSHF_B2_UB(src6, src7, src8, src9, mask0, mask0, dst0, dst1); 642 dst0 = (v16u8) __msa_vshf_b(mask1, (v16i8) dst1, (v16i8) dst0); 643 out0 = __msa_copy_s_d((v2i64) dst0, 0); 644 out1 = __msa_copy_s_w((v4i32) dst0, 2); 645 646 SD(out0, nxt); 647 nxt += 8; 648 SW(out1, nxt); 649 nxt += 4; 650 } 651 } 652 653 void png_read_filter_row_paeth4_msa(png_row_infop row_info, 654 png_bytep row, 655 png_const_bytep prev_row) 656 { 657 int32_t count, rp_end; 658 png_bytep nxt; 659 png_const_bytep prev_nxt; 660 int32_t inp0, inp1, res0; 661 v16u8 src0, src1, src2, src3, src4, src5, src6, src7, src8, src9; 662 v16u8 src10, src11, src12, src13, dst0, dst1; 663 v8i16 vec0, vec1, vec2; 664 v16u8 zero = { 0 }; 665 666 nxt = row; 667 prev_nxt = prev_row; 668 669 inp0 = LW(nxt); 670 inp1 = LW(prev_nxt); 671 prev_nxt += 4; 672 src0 = (v16u8) __msa_insert_w((v4i32) zero, 0, inp0); 673 src1 = (v16u8) __msa_insert_w((v4i32) zero, 0, inp1); 674 675 src1 += src0; 676 res0 = __msa_copy_s_w((v4i32) src1, 0); 677 678 SW(res0, nxt); 679 nxt += 4; 680 681 /* Remainder */ 682 rp_end = row_info->rowbytes - 4; 683 684 for (count = 0; count < rp_end; count += 16) 685 { 686 src2 = LD_UB(prev_nxt); 687 prev_nxt += 16; 688 src6 = LD_UB(prev_row); 689 prev_row += 16; 690 src10 = LD_UB(nxt); 691 692 SLDI_B3_0_UB(src2, src6, src10, src3, src7, src11, 4); 693 SLDI_B3_0_UB(src2, src6, src10, src4, src8, src12, 8); 694 SLDI_B3_0_UB(src2, src6, src10, src5, src9, src13, 12); 695 ILVR_B2_SH(src2, src6, src1, src6, vec0, vec1); 696 HSUB_UB2_SH(vec0, vec1, vec0, vec1); 697 vec2 = vec0 + vec1; 698 ADD_ABS_H3_SH(vec0, vec1, vec2, vec0, vec1, vec2); 699 CMP_AND_SELECT(vec0, vec1, vec2, src1, src2, src6, src10); 700 ILVR_B2_SH(src3, src7, src10, src7, vec0, vec1); 701 HSUB_UB2_SH(vec0, vec1, vec0, vec1); 702 vec2 = vec0 + vec1; 703 ADD_ABS_H3_SH(vec0, vec1, vec2, vec0, vec1, vec2); 704 CMP_AND_SELECT(vec0, vec1, vec2, src10, src3, src7, src11); 705 ILVR_B2_SH(src4, src8, src11, src8, vec0, vec1); 706 HSUB_UB2_SH(vec0, vec1, vec0, vec1); 707 vec2 = vec0 + vec1; 708 ADD_ABS_H3_SH(vec0, vec1, vec2, vec0, vec1, vec2); 709 CMP_AND_SELECT(vec0, vec1, vec2, src11, src4, src8, src12); 710 ILVR_B2_SH(src5, src9, src12, src9, vec0, vec1); 711 HSUB_UB2_SH(vec0, vec1, vec0, vec1); 712 vec2 = vec0 + vec1; 713 ADD_ABS_H3_SH(vec0, vec1, vec2, vec0, vec1, vec2); 714 CMP_AND_SELECT(vec0, vec1, vec2, src12, src5, src9, src13); 715 src1 = src13; 716 ILVEV_W2_UB(src10, src11, src12, src1, dst0, dst1); 717 dst0 = (v16u8) __msa_pckev_d((v2i64) dst1, (v2i64) dst0); 718 719 ST_UB(dst0, nxt); 720 nxt += 16; 721 } 722 } 723 724 void png_read_filter_row_paeth3_msa(png_row_infop row_info, 725 png_bytep row, 726 png_const_bytep prev_row) 727 { 728 int32_t count, rp_end; 729 png_bytep nxt; 730 png_const_bytep prev_nxt; 731 int64_t out0; 732 int32_t inp0, inp1, out1; 733 int16_t out2; 734 v16u8 src0, src1, src2, src3, src4, src5, src6, src7, src8, src9, dst0, dst1; 735 v16u8 src10, src11, src12, src13; 736 v8i16 vec0, vec1, vec2; 737 v16u8 zero = { 0 }; 738 v16i8 mask0 = { 0, 1, 2, 16, 17, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 }; 739 v16i8 mask1 = { 0, 1, 2, 3, 4, 5, 16, 17, 18, 19, 20, 21, 0, 0, 0, 0 }; 740 741 nxt = row; 742 prev_nxt = prev_row; 743 744 inp0 = LW(nxt); 745 inp1 = LW(prev_nxt); 746 prev_nxt += 3; 747 src0 = (v16u8) __msa_insert_w((v4i32) zero, 0, inp0); 748 src1 = (v16u8) __msa_insert_w((v4i32) zero, 0, inp1); 749 750 src1 += src0; 751 out2 = __msa_copy_s_h((v8i16) src1, 0); 752 753 SH(out2, nxt); 754 nxt += 2; 755 nxt[0] = src1[2]; 756 nxt++; 757 758 /* Remainder */ 759 rp_end = row_info->rowbytes - 3; 760 761 for (count = 0; count < rp_end; count += 12) 762 { 763 src2 = LD_UB(prev_nxt); 764 prev_nxt += 12; 765 src6 = LD_UB(prev_row); 766 prev_row += 12; 767 src10 = LD_UB(nxt); 768 769 SLDI_B3_0_UB(src2, src6, src10, src3, src7, src11, 3); 770 SLDI_B3_0_UB(src2, src6, src10, src4, src8, src12, 6); 771 SLDI_B3_0_UB(src2, src6, src10, src5, src9, src13, 9); 772 ILVR_B2_SH(src2, src6, src1, src6, vec0, vec1); 773 HSUB_UB2_SH(vec0, vec1, vec0, vec1); 774 vec2 = vec0 + vec1; 775 ADD_ABS_H3_SH(vec0, vec1, vec2, vec0, vec1, vec2); 776 CMP_AND_SELECT(vec0, vec1, vec2, src1, src2, src6, src10); 777 ILVR_B2_SH(src3, src7, src10, src7, vec0, vec1); 778 HSUB_UB2_SH(vec0, vec1, vec0, vec1); 779 vec2 = vec0 + vec1; 780 ADD_ABS_H3_SH(vec0, vec1, vec2, vec0, vec1, vec2); 781 CMP_AND_SELECT(vec0, vec1, vec2, src10, src3, src7, src11); 782 ILVR_B2_SH(src4, src8, src11, src8, vec0, vec1); 783 HSUB_UB2_SH(vec0, vec1, vec0, vec1); 784 vec2 = vec0 + vec1; 785 ADD_ABS_H3_SH(vec0, vec1, vec2, vec0, vec1, vec2); 786 CMP_AND_SELECT(vec0, vec1, vec2, src11, src4, src8, src12); 787 ILVR_B2_SH(src5, src9, src12, src9, vec0, vec1); 788 HSUB_UB2_SH(vec0, vec1, vec0, vec1); 789 vec2 = vec0 + vec1; 790 ADD_ABS_H3_SH(vec0, vec1, vec2, vec0, vec1, vec2); 791 CMP_AND_SELECT(vec0, vec1, vec2, src12, src5, src9, src13); 792 src1 = src13; 793 VSHF_B2_UB(src10, src11, src12, src13, mask0, mask0, dst0, dst1); 794 dst0 = (v16u8) __msa_vshf_b(mask1, (v16i8) dst1, (v16i8) dst0); 795 out0 = __msa_copy_s_d((v2i64) dst0, 0); 796 out1 = __msa_copy_s_w((v4i32) dst0, 2); 797 798 SD(out0, nxt); 799 nxt += 8; 800 SW(out1, nxt); 801 nxt += 4; 802 } 803 } 804 805 #endif /* PNG_MIPS_MSA_OPT > 0 */ 806 #endif /* PNG_MIPS_MSA_IMPLEMENTATION == 1 (intrinsics) */ 807 #endif /* READ */ 808
