1 // Copyright 2016 Google Inc. All Rights Reserved. 2 // 3 // Use of this source code is governed by a BSD-style license 4 // that can be found in the COPYING file in the root of the source 5 // tree. An additional intellectual property rights grant can be found 6 // in the file PATENTS. All contributing project authors may 7 // be found in the AUTHORS file in the root of the source tree. 8 // ----------------------------------------------------------------------------- 9 // 10 // MSA common macros 11 // 12 // Author(s): Prashant Patil (prashant.patil (at) imgtec.com) 13 14 #ifndef WEBP_DSP_MSA_MACRO_H_ 15 #define WEBP_DSP_MSA_MACRO_H_ 16 17 #include <stdint.h> 18 #include <msa.h> 19 20 #if defined(__clang__) 21 #define CLANG_BUILD 22 #endif 23 24 #ifdef CLANG_BUILD 25 #define ALPHAVAL (-1) 26 #define ADDVI_H(a, b) __msa_addvi_h((v8i16)a, b) 27 #define ADDVI_W(a, b) __msa_addvi_w((v4i32)a, b) 28 #define SRAI_B(a, b) __msa_srai_b((v16i8)a, b) 29 #define SRAI_H(a, b) __msa_srai_h((v8i16)a, b) 30 #define SRAI_W(a, b) __msa_srai_w((v4i32)a, b) 31 #define SRLI_H(a, b) __msa_srli_h((v8i16)a, b) 32 #define SLLI_B(a, b) __msa_slli_b((v4i32)a, b) 33 #define ANDI_B(a, b) __msa_andi_b((v16u8)a, b) 34 #define ORI_B(a, b) __msa_ori_b((v16u8)a, b) 35 #else 36 #define ALPHAVAL (0xff) 37 #define ADDVI_H(a, b) (a + b) 38 #define ADDVI_W(a, b) (a + b) 39 #define SRAI_B(a, b) (a >> b) 40 #define SRAI_H(a, b) (a >> b) 41 #define SRAI_W(a, b) (a >> b) 42 #define SRLI_H(a, b) (a << b) 43 #define SLLI_B(a, b) (a << b) 44 #define ANDI_B(a, b) (a & b) 45 #define ORI_B(a, b) (a | b) 46 #endif 47 48 #define LD_B(RTYPE, psrc) *((RTYPE*)(psrc)) 49 #define LD_UB(...) LD_B(v16u8, __VA_ARGS__) 50 #define LD_SB(...) LD_B(v16i8, __VA_ARGS__) 51 52 #define LD_H(RTYPE, psrc) *((RTYPE*)(psrc)) 53 #define LD_UH(...) LD_H(v8u16, __VA_ARGS__) 54 #define LD_SH(...) LD_H(v8i16, __VA_ARGS__) 55 56 #define LD_W(RTYPE, psrc) *((RTYPE*)(psrc)) 57 #define LD_UW(...) LD_W(v4u32, __VA_ARGS__) 58 #define LD_SW(...) LD_W(v4i32, __VA_ARGS__) 59 60 #define ST_B(RTYPE, in, pdst) *((RTYPE*)(pdst)) = in 61 #define ST_UB(...) ST_B(v16u8, __VA_ARGS__) 62 #define ST_SB(...) ST_B(v16i8, __VA_ARGS__) 63 64 #define ST_H(RTYPE, in, pdst) *((RTYPE*)(pdst)) = in 65 #define ST_UH(...) ST_H(v8u16, __VA_ARGS__) 66 #define ST_SH(...) ST_H(v8i16, __VA_ARGS__) 67 68 #define ST_W(RTYPE, in, pdst) *((RTYPE*)(pdst)) = in 69 #define ST_UW(...) ST_W(v4u32, __VA_ARGS__) 70 #define ST_SW(...) ST_W(v4i32, __VA_ARGS__) 71 72 #define MSA_LOAD_FUNC(TYPE, INSTR, FUNC_NAME) \ 73 static inline TYPE FUNC_NAME(const void* const psrc) { \ 74 const uint8_t* const psrc_m = (const uint8_t*)psrc; \ 75 TYPE val_m; \ 76 asm volatile ( \ 77 "" #INSTR " %[val_m], %[psrc_m] \n\t" \ 78 : [val_m] "=r" (val_m) \ 79 : [psrc_m] "m" (*psrc_m)); \ 80 return val_m; \ 81 } 82 83 #define MSA_LOAD(psrc, FUNC_NAME) FUNC_NAME(psrc) 84 85 #define MSA_STORE_FUNC(TYPE, INSTR, FUNC_NAME) \ 86 static inline void FUNC_NAME(TYPE val, void* const pdst) { \ 87 uint8_t* const pdst_m = (uint8_t*)pdst; \ 88 TYPE val_m = val; \ 89 asm volatile ( \ 90 " " #INSTR " %[val_m], %[pdst_m] \n\t" \ 91 : [pdst_m] "=m" (*pdst_m) \ 92 : [val_m] "r" (val_m)); \ 93 } 94 95 #define MSA_STORE(val, pdst, FUNC_NAME) FUNC_NAME(val, pdst) 96 97 #if (__mips_isa_rev >= 6) 98 MSA_LOAD_FUNC(uint16_t, lh, msa_lh); 99 #define LH(psrc) MSA_LOAD(psrc, msa_lh) 100 MSA_LOAD_FUNC(uint32_t, lw, msa_lw); 101 #define LW(psrc) MSA_LOAD(psrc, msa_lw) 102 #if (__mips == 64) 103 MSA_LOAD_FUNC(uint64_t, ld, msa_ld); 104 #define LD(psrc) MSA_LOAD(psrc, msa_ld) 105 #else // !(__mips == 64) 106 #define LD(psrc) ((((uint64_t)MSA_LOAD(psrc + 4, msa_lw)) << 32) | \ 107 MSA_LOAD(psrc, msa_lw)) 108 #endif // (__mips == 64) 109 110 MSA_STORE_FUNC(uint16_t, sh, msa_sh); 111 #define SH(val, pdst) MSA_STORE(val, pdst, msa_sh) 112 MSA_STORE_FUNC(uint32_t, sw, msa_sw); 113 #define SW(val, pdst) MSA_STORE(val, pdst, msa_sw) 114 MSA_STORE_FUNC(uint64_t, sd, msa_sd); 115 #define SD(val, pdst) MSA_STORE(val, pdst, msa_sd) 116 #else // !(__mips_isa_rev >= 6) 117 MSA_LOAD_FUNC(uint16_t, ulh, msa_ulh); 118 #define LH(psrc) MSA_LOAD(psrc, msa_ulh) 119 MSA_LOAD_FUNC(uint32_t, ulw, msa_ulw); 120 #define LW(psrc) MSA_LOAD(psrc, msa_ulw) 121 #if (__mips == 64) 122 MSA_LOAD_FUNC(uint64_t, uld, msa_uld); 123 #define LD(psrc) MSA_LOAD(psrc, msa_uld) 124 #else // !(__mips == 64) 125 #define LD(psrc) ((((uint64_t)MSA_LOAD(psrc + 4, msa_ulw)) << 32) | \ 126 MSA_LOAD(psrc, msa_ulw)) 127 #endif // (__mips == 64) 128 129 MSA_STORE_FUNC(uint16_t, ush, msa_ush); 130 #define SH(val, pdst) MSA_STORE(val, pdst, msa_ush) 131 MSA_STORE_FUNC(uint32_t, usw, msa_usw); 132 #define SW(val, pdst) MSA_STORE(val, pdst, msa_usw) 133 #define SD(val, pdst) do { \ 134 uint8_t* const pdst_sd_m = (uint8_t*)(pdst); \ 135 const uint32_t val0_m = (uint32_t)(val & 0x00000000FFFFFFFF); \ 136 const uint32_t val1_m = (uint32_t)((val >> 32) & 0x00000000FFFFFFFF); \ 137 SW(val0_m, pdst_sd_m); \ 138 SW(val1_m, pdst_sd_m + 4); \ 139 } while (0) 140 #endif // (__mips_isa_rev >= 6) 141 142 /* Description : Load 4 words with stride 143 * Arguments : Inputs - psrc, stride 144 * Outputs - out0, out1, out2, out3 145 * Details : Load word in 'out0' from (psrc) 146 * Load word in 'out1' from (psrc + stride) 147 * Load word in 'out2' from (psrc + 2 * stride) 148 * Load word in 'out3' from (psrc + 3 * stride) 149 */ 150 #define LW4(psrc, stride, out0, out1, out2, out3) do { \ 151 const uint8_t* ptmp = (const uint8_t*)psrc; \ 152 out0 = LW(ptmp); \ 153 ptmp += stride; \ 154 out1 = LW(ptmp); \ 155 ptmp += stride; \ 156 out2 = LW(ptmp); \ 157 ptmp += stride; \ 158 out3 = LW(ptmp); \ 159 } while (0) 160 161 /* Description : Store words with stride 162 * Arguments : Inputs - in0, in1, in2, in3, pdst, stride 163 * Details : Store word from 'in0' to (pdst) 164 * Store word from 'in1' to (pdst + stride) 165 * Store word from 'in2' to (pdst + 2 * stride) 166 * Store word from 'in3' to (pdst + 3 * stride) 167 */ 168 #define SW4(in0, in1, in2, in3, pdst, stride) do { \ 169 uint8_t* ptmp = (uint8_t*)pdst; \ 170 SW(in0, ptmp); \ 171 ptmp += stride; \ 172 SW(in1, ptmp); \ 173 ptmp += stride; \ 174 SW(in2, ptmp); \ 175 ptmp += stride; \ 176 SW(in3, ptmp); \ 177 } while (0) 178 179 #define SW3(in0, in1, in2, pdst, stride) do { \ 180 uint8_t* ptmp = (uint8_t*)pdst; \ 181 SW(in0, ptmp); \ 182 ptmp += stride; \ 183 SW(in1, ptmp); \ 184 ptmp += stride; \ 185 SW(in2, ptmp); \ 186 } while (0) 187 188 #define SW2(in0, in1, pdst, stride) do { \ 189 uint8_t* ptmp = (uint8_t*)pdst; \ 190 SW(in0, ptmp); \ 191 ptmp += stride; \ 192 SW(in1, ptmp); \ 193 } while (0) 194 195 /* Description : Store 4 double words with stride 196 * Arguments : Inputs - in0, in1, in2, in3, pdst, stride 197 * Details : Store double word from 'in0' to (pdst) 198 * Store double word from 'in1' to (pdst + stride) 199 * Store double word from 'in2' to (pdst + 2 * stride) 200 * Store double word from 'in3' to (pdst + 3 * stride) 201 */ 202 #define SD4(in0, in1, in2, in3, pdst, stride) do { \ 203 uint8_t* ptmp = (uint8_t*)pdst; \ 204 SD(in0, ptmp); \ 205 ptmp += stride; \ 206 SD(in1, ptmp); \ 207 ptmp += stride; \ 208 SD(in2, ptmp); \ 209 ptmp += stride; \ 210 SD(in3, ptmp); \ 211 } while (0) 212 213 /* Description : Load vectors with 16 byte elements with stride 214 * Arguments : Inputs - psrc, stride 215 * Outputs - out0, out1 216 * Return Type - as per RTYPE 217 * Details : Load 16 byte elements in 'out0' from (psrc) 218 * Load 16 byte elements in 'out1' from (psrc + stride) 219 */ 220 #define LD_B2(RTYPE, psrc, stride, out0, out1) do { \ 221 out0 = LD_B(RTYPE, psrc); \ 222 out1 = LD_B(RTYPE, psrc + stride); \ 223 } while (0) 224 #define LD_UB2(...) LD_B2(v16u8, __VA_ARGS__) 225 #define LD_SB2(...) LD_B2(v16i8, __VA_ARGS__) 226 227 #define LD_B3(RTYPE, psrc, stride, out0, out1, out2) do { \ 228 LD_B2(RTYPE, psrc, stride, out0, out1); \ 229 out2 = LD_B(RTYPE, psrc + 2 * stride); \ 230 } while (0) 231 #define LD_UB3(...) LD_B3(v16u8, __VA_ARGS__) 232 #define LD_SB3(...) LD_B3(v16i8, __VA_ARGS__) 233 234 #define LD_B4(RTYPE, psrc, stride, out0, out1, out2, out3) do { \ 235 LD_B2(RTYPE, psrc, stride, out0, out1); \ 236 LD_B2(RTYPE, psrc + 2 * stride , stride, out2, out3); \ 237 } while (0) 238 #define LD_UB4(...) LD_B4(v16u8, __VA_ARGS__) 239 #define LD_SB4(...) LD_B4(v16i8, __VA_ARGS__) 240 241 #define LD_B8(RTYPE, psrc, stride, \ 242 out0, out1, out2, out3, out4, out5, out6, out7) do { \ 243 LD_B4(RTYPE, psrc, stride, out0, out1, out2, out3); \ 244 LD_B4(RTYPE, psrc + 4 * stride, stride, out4, out5, out6, out7); \ 245 } while (0) 246 #define LD_UB8(...) LD_B8(v16u8, __VA_ARGS__) 247 #define LD_SB8(...) LD_B8(v16i8, __VA_ARGS__) 248 249 /* Description : Load vectors with 8 halfword elements with stride 250 * Arguments : Inputs - psrc, stride 251 * Outputs - out0, out1 252 * Details : Load 8 halfword elements in 'out0' from (psrc) 253 * Load 8 halfword elements in 'out1' from (psrc + stride) 254 */ 255 #define LD_H2(RTYPE, psrc, stride, out0, out1) do { \ 256 out0 = LD_H(RTYPE, psrc); \ 257 out1 = LD_H(RTYPE, psrc + stride); \ 258 } while (0) 259 #define LD_UH2(...) LD_H2(v8u16, __VA_ARGS__) 260 #define LD_SH2(...) LD_H2(v8i16, __VA_ARGS__) 261 262 /* Description : Load vectors with 4 word elements with stride 263 * Arguments : Inputs - psrc, stride 264 * Outputs - out0, out1, out2, out3 265 * Details : Load 4 word elements in 'out0' from (psrc + 0 * stride) 266 * Load 4 word elements in 'out1' from (psrc + 1 * stride) 267 * Load 4 word elements in 'out2' from (psrc + 2 * stride) 268 * Load 4 word elements in 'out3' from (psrc + 3 * stride) 269 */ 270 #define LD_W2(RTYPE, psrc, stride, out0, out1) do { \ 271 out0 = LD_W(RTYPE, psrc); \ 272 out1 = LD_W(RTYPE, psrc + stride); \ 273 } while (0) 274 #define LD_UW2(...) LD_W2(v4u32, __VA_ARGS__) 275 #define LD_SW2(...) LD_W2(v4i32, __VA_ARGS__) 276 277 #define LD_W3(RTYPE, psrc, stride, out0, out1, out2) do { \ 278 LD_W2(RTYPE, psrc, stride, out0, out1); \ 279 out2 = LD_W(RTYPE, psrc + 2 * stride); \ 280 } while (0) 281 #define LD_UW3(...) LD_W3(v4u32, __VA_ARGS__) 282 #define LD_SW3(...) LD_W3(v4i32, __VA_ARGS__) 283 284 #define LD_W4(RTYPE, psrc, stride, out0, out1, out2, out3) do { \ 285 LD_W2(RTYPE, psrc, stride, out0, out1); \ 286 LD_W2(RTYPE, psrc + 2 * stride, stride, out2, out3); \ 287 } while (0) 288 #define LD_UW4(...) LD_W4(v4u32, __VA_ARGS__) 289 #define LD_SW4(...) LD_W4(v4i32, __VA_ARGS__) 290 291 /* Description : Store vectors of 16 byte elements with stride 292 * Arguments : Inputs - in0, in1, pdst, stride 293 * Details : Store 16 byte elements from 'in0' to (pdst) 294 * Store 16 byte elements from 'in1' to (pdst + stride) 295 */ 296 #define ST_B2(RTYPE, in0, in1, pdst, stride) do { \ 297 ST_B(RTYPE, in0, pdst); \ 298 ST_B(RTYPE, in1, pdst + stride); \ 299 } while (0) 300 #define ST_UB2(...) ST_B2(v16u8, __VA_ARGS__) 301 #define ST_SB2(...) ST_B2(v16i8, __VA_ARGS__) 302 303 #define ST_B4(RTYPE, in0, in1, in2, in3, pdst, stride) do { \ 304 ST_B2(RTYPE, in0, in1, pdst, stride); \ 305 ST_B2(RTYPE, in2, in3, pdst + 2 * stride, stride); \ 306 } while (0) 307 #define ST_UB4(...) ST_B4(v16u8, __VA_ARGS__) 308 #define ST_SB4(...) ST_B4(v16i8, __VA_ARGS__) 309 310 #define ST_B8(RTYPE, in0, in1, in2, in3, in4, in5, in6, in7, \ 311 pdst, stride) do { \ 312 ST_B4(RTYPE, in0, in1, in2, in3, pdst, stride); \ 313 ST_B4(RTYPE, in4, in5, in6, in7, pdst + 4 * stride, stride); \ 314 } while (0) 315 #define ST_UB8(...) ST_B8(v16u8, __VA_ARGS__) 316 317 /* Description : Store vectors of 4 word elements with stride 318 * Arguments : Inputs - in0, in1, in2, in3, pdst, stride 319 * Details : Store 4 word elements from 'in0' to (pdst + 0 * stride) 320 * Store 4 word elements from 'in1' to (pdst + 1 * stride) 321 * Store 4 word elements from 'in2' to (pdst + 2 * stride) 322 * Store 4 word elements from 'in3' to (pdst + 3 * stride) 323 */ 324 #define ST_W2(RTYPE, in0, in1, pdst, stride) do { \ 325 ST_W(RTYPE, in0, pdst); \ 326 ST_W(RTYPE, in1, pdst + stride); \ 327 } while (0) 328 #define ST_UW2(...) ST_W2(v4u32, __VA_ARGS__) 329 #define ST_SW2(...) ST_W2(v4i32, __VA_ARGS__) 330 331 #define ST_W3(RTYPE, in0, in1, in2, pdst, stride) do { \ 332 ST_W2(RTYPE, in0, in1, pdst, stride); \ 333 ST_W(RTYPE, in2, pdst + 2 * stride); \ 334 } while (0) 335 #define ST_UW3(...) ST_W3(v4u32, __VA_ARGS__) 336 #define ST_SW3(...) ST_W3(v4i32, __VA_ARGS__) 337 338 #define ST_W4(RTYPE, in0, in1, in2, in3, pdst, stride) do { \ 339 ST_W2(RTYPE, in0, in1, pdst, stride); \ 340 ST_W2(RTYPE, in2, in3, pdst + 2 * stride, stride); \ 341 } while (0) 342 #define ST_UW4(...) ST_W4(v4u32, __VA_ARGS__) 343 #define ST_SW4(...) ST_W4(v4i32, __VA_ARGS__) 344 345 /* Description : Store vectors of 8 halfword elements with stride 346 * Arguments : Inputs - in0, in1, pdst, stride 347 * Details : Store 8 halfword elements from 'in0' to (pdst) 348 * Store 8 halfword elements from 'in1' to (pdst + stride) 349 */ 350 #define ST_H2(RTYPE, in0, in1, pdst, stride) do { \ 351 ST_H(RTYPE, in0, pdst); \ 352 ST_H(RTYPE, in1, pdst + stride); \ 353 } while (0) 354 #define ST_UH2(...) ST_H2(v8u16, __VA_ARGS__) 355 #define ST_SH2(...) ST_H2(v8i16, __VA_ARGS__) 356 357 /* Description : Store 2x4 byte block to destination memory from input vector 358 * Arguments : Inputs - in, stidx, pdst, stride 359 * Details : Index 'stidx' halfword element from 'in' vector is copied to 360 * the GP register and stored to (pdst) 361 * Index 'stidx+1' halfword element from 'in' vector is copied to 362 * the GP register and stored to (pdst + stride) 363 * Index 'stidx+2' halfword element from 'in' vector is copied to 364 * the GP register and stored to (pdst + 2 * stride) 365 * Index 'stidx+3' halfword element from 'in' vector is copied to 366 * the GP register and stored to (pdst + 3 * stride) 367 */ 368 #define ST2x4_UB(in, stidx, pdst, stride) do { \ 369 uint8_t* pblk_2x4_m = (uint8_t*)pdst; \ 370 const uint16_t out0_m = __msa_copy_s_h((v8i16)in, stidx); \ 371 const uint16_t out1_m = __msa_copy_s_h((v8i16)in, stidx + 1); \ 372 const uint16_t out2_m = __msa_copy_s_h((v8i16)in, stidx + 2); \ 373 const uint16_t out3_m = __msa_copy_s_h((v8i16)in, stidx + 3); \ 374 SH(out0_m, pblk_2x4_m); \ 375 pblk_2x4_m += stride; \ 376 SH(out1_m, pblk_2x4_m); \ 377 pblk_2x4_m += stride; \ 378 SH(out2_m, pblk_2x4_m); \ 379 pblk_2x4_m += stride; \ 380 SH(out3_m, pblk_2x4_m); \ 381 } while (0) 382 383 /* Description : Store 4x4 byte block to destination memory from input vector 384 * Arguments : Inputs - in0, in1, pdst, stride 385 * Details : 'Idx0' word element from input vector 'in0' is copied to the 386 * GP register and stored to (pdst) 387 * 'Idx1' word element from input vector 'in0' is copied to the 388 * GP register and stored to (pdst + stride) 389 * 'Idx2' word element from input vector 'in0' is copied to the 390 * GP register and stored to (pdst + 2 * stride) 391 * 'Idx3' word element from input vector 'in0' is copied to the 392 * GP register and stored to (pdst + 3 * stride) 393 */ 394 #define ST4x4_UB(in0, in1, idx0, idx1, idx2, idx3, pdst, stride) do { \ 395 uint8_t* const pblk_4x4_m = (uint8_t*)pdst; \ 396 const uint32_t out0_m = __msa_copy_s_w((v4i32)in0, idx0); \ 397 const uint32_t out1_m = __msa_copy_s_w((v4i32)in0, idx1); \ 398 const uint32_t out2_m = __msa_copy_s_w((v4i32)in1, idx2); \ 399 const uint32_t out3_m = __msa_copy_s_w((v4i32)in1, idx3); \ 400 SW4(out0_m, out1_m, out2_m, out3_m, pblk_4x4_m, stride); \ 401 } while (0) 402 403 #define ST4x8_UB(in0, in1, pdst, stride) do { \ 404 uint8_t* const pblk_4x8 = (uint8_t*)pdst; \ 405 ST4x4_UB(in0, in0, 0, 1, 2, 3, pblk_4x8, stride); \ 406 ST4x4_UB(in1, in1, 0, 1, 2, 3, pblk_4x8 + 4 * stride, stride); \ 407 } while (0) 408 409 /* Description : Immediate number of elements to slide 410 * Arguments : Inputs - in0, in1, slide_val 411 * Outputs - out 412 * Return Type - as per RTYPE 413 * Details : Byte elements from 'in1' vector are slid into 'in0' by 414 * value specified in the 'slide_val' 415 */ 416 #define SLDI_B(RTYPE, in0, in1, slide_val) \ 417 (RTYPE)__msa_sldi_b((v16i8)in0, (v16i8)in1, slide_val) \ 418 419 #define SLDI_UB(...) SLDI_B(v16u8, __VA_ARGS__) 420 #define SLDI_SB(...) SLDI_B(v16i8, __VA_ARGS__) 421 #define SLDI_SH(...) SLDI_B(v8i16, __VA_ARGS__) 422 423 /* Description : Shuffle byte vector elements as per mask vector 424 * Arguments : Inputs - in0, in1, in2, in3, mask0, mask1 425 * Outputs - out0, out1 426 * Return Type - as per RTYPE 427 * Details : Byte elements from 'in0' & 'in1' are copied selectively to 428 * 'out0' as per control vector 'mask0' 429 */ 430 #define VSHF_B(RTYPE, in0, in1, mask) \ 431 (RTYPE)__msa_vshf_b((v16i8)mask, (v16i8)in1, (v16i8)in0) 432 433 #define VSHF_UB(...) VSHF_B(v16u8, __VA_ARGS__) 434 #define VSHF_SB(...) VSHF_B(v16i8, __VA_ARGS__) 435 #define VSHF_UH(...) VSHF_B(v8u16, __VA_ARGS__) 436 #define VSHF_SH(...) VSHF_B(v8i16, __VA_ARGS__) 437 438 #define VSHF_B2(RTYPE, in0, in1, in2, in3, mask0, mask1, out0, out1) do { \ 439 out0 = VSHF_B(RTYPE, in0, in1, mask0); \ 440 out1 = VSHF_B(RTYPE, in2, in3, mask1); \ 441 } while (0) 442 #define VSHF_B2_UB(...) VSHF_B2(v16u8, __VA_ARGS__) 443 #define VSHF_B2_SB(...) VSHF_B2(v16i8, __VA_ARGS__) 444 #define VSHF_B2_UH(...) VSHF_B2(v8u16, __VA_ARGS__) 445 #define VSHF_B2_SH(...) VSHF_B2(v8i16, __VA_ARGS__) 446 447 /* Description : Shuffle halfword vector elements as per mask vector 448 * Arguments : Inputs - in0, in1, in2, in3, mask0, mask1 449 * Outputs - out0, out1 450 * Return Type - as per RTYPE 451 * Details : halfword elements from 'in0' & 'in1' are copied selectively to 452 * 'out0' as per control vector 'mask0' 453 */ 454 #define VSHF_H2(RTYPE, in0, in1, in2, in3, mask0, mask1, out0, out1) do { \ 455 out0 = (RTYPE)__msa_vshf_h((v8i16)mask0, (v8i16)in1, (v8i16)in0); \ 456 out1 = (RTYPE)__msa_vshf_h((v8i16)mask1, (v8i16)in3, (v8i16)in2); \ 457 } while (0) 458 #define VSHF_H2_UH(...) VSHF_H2(v8u16, __VA_ARGS__) 459 #define VSHF_H2_SH(...) VSHF_H2(v8i16, __VA_ARGS__) 460 461 /* Description : Dot product of byte vector elements 462 * Arguments : Inputs - mult0, mult1, cnst0, cnst1 463 * Outputs - out0, out1 464 * Return Type - as per RTYPE 465 * Details : Signed byte elements from 'mult0' are multiplied with 466 * signed byte elements from 'cnst0' producing a result 467 * twice the size of input i.e. signed halfword. 468 * The multiplication result of adjacent odd-even elements 469 * are added together and written to the 'out0' vector 470 */ 471 #define DOTP_SB2(RTYPE, mult0, mult1, cnst0, cnst1, out0, out1) do { \ 472 out0 = (RTYPE)__msa_dotp_s_h((v16i8)mult0, (v16i8)cnst0); \ 473 out1 = (RTYPE)__msa_dotp_s_h((v16i8)mult1, (v16i8)cnst1); \ 474 } while (0) 475 #define DOTP_SB2_SH(...) DOTP_SB2(v8i16, __VA_ARGS__) 476 477 /* Description : Dot product of halfword vector elements 478 * Arguments : Inputs - mult0, mult1, cnst0, cnst1 479 * Outputs - out0, out1 480 * Return Type - as per RTYPE 481 * Details : Signed halfword elements from 'mult0' are multiplied with 482 * signed halfword elements from 'cnst0' producing a result 483 * twice the size of input i.e. signed word. 484 * The multiplication result of adjacent odd-even elements 485 * are added together and written to the 'out0' vector 486 */ 487 #define DOTP_SH2(RTYPE, mult0, mult1, cnst0, cnst1, out0, out1) do { \ 488 out0 = (RTYPE)__msa_dotp_s_w((v8i16)mult0, (v8i16)cnst0); \ 489 out1 = (RTYPE)__msa_dotp_s_w((v8i16)mult1, (v8i16)cnst1); \ 490 } while (0) 491 #define DOTP_SH2_SW(...) DOTP_SH2(v4i32, __VA_ARGS__) 492 493 /* Description : Dot product of unsigned word vector elements 494 * Arguments : Inputs - mult0, mult1, cnst0, cnst1 495 * Outputs - out0, out1 496 * Return Type - as per RTYPE 497 * Details : Unsigned word elements from 'mult0' are multiplied with 498 * unsigned word elements from 'cnst0' producing a result 499 * twice the size of input i.e. unsigned double word. 500 * The multiplication result of adjacent odd-even elements 501 * are added together and written to the 'out0' vector 502 */ 503 #define DOTP_UW2(RTYPE, mult0, mult1, cnst0, cnst1, out0, out1) do { \ 504 out0 = (RTYPE)__msa_dotp_u_d((v4u32)mult0, (v4u32)cnst0); \ 505 out1 = (RTYPE)__msa_dotp_u_d((v4u32)mult1, (v4u32)cnst1); \ 506 } while (0) 507 #define DOTP_UW2_UD(...) DOTP_UW2(v2u64, __VA_ARGS__) 508 509 /* Description : Dot product & addition of halfword vector elements 510 * Arguments : Inputs - mult0, mult1, cnst0, cnst1 511 * Outputs - out0, out1 512 * Return Type - as per RTYPE 513 * Details : Signed halfword elements from 'mult0' are multiplied with 514 * signed halfword elements from 'cnst0' producing a result 515 * twice the size of input i.e. signed word. 516 * The multiplication result of adjacent odd-even elements 517 * are added to the 'out0' vector 518 */ 519 #define DPADD_SH2(RTYPE, mult0, mult1, cnst0, cnst1, out0, out1) do { \ 520 out0 = (RTYPE)__msa_dpadd_s_w((v4i32)out0, (v8i16)mult0, (v8i16)cnst0); \ 521 out1 = (RTYPE)__msa_dpadd_s_w((v4i32)out1, (v8i16)mult1, (v8i16)cnst1); \ 522 } while (0) 523 #define DPADD_SH2_SW(...) DPADD_SH2(v4i32, __VA_ARGS__) 524 525 /* Description : Clips all signed halfword elements of input vector 526 * between 0 & 255 527 * Arguments : Input/output - val 528 * Return Type - signed halfword 529 */ 530 #define CLIP_SH_0_255(val) do { \ 531 const v8i16 max_m = __msa_ldi_h(255); \ 532 val = __msa_maxi_s_h((v8i16)val, 0); \ 533 val = __msa_min_s_h(max_m, (v8i16)val); \ 534 } while (0) 535 536 #define CLIP_SH2_0_255(in0, in1) do { \ 537 CLIP_SH_0_255(in0); \ 538 CLIP_SH_0_255(in1); \ 539 } while (0) 540 541 #define CLIP_SH4_0_255(in0, in1, in2, in3) do { \ 542 CLIP_SH2_0_255(in0, in1); \ 543 CLIP_SH2_0_255(in2, in3); \ 544 } while (0) 545 546 /* Description : Clips all unsigned halfword elements of input vector 547 * between 0 & 255 548 * Arguments : Input - in 549 * Output - out_m 550 * Return Type - unsigned halfword 551 */ 552 #define CLIP_UH_0_255(in) do { \ 553 const v8u16 max_m = (v8u16)__msa_ldi_h(255); \ 554 in = __msa_maxi_u_h((v8u16) in, 0); \ 555 in = __msa_min_u_h((v8u16) max_m, (v8u16) in); \ 556 } while (0) 557 558 #define CLIP_UH2_0_255(in0, in1) do { \ 559 CLIP_UH_0_255(in0); \ 560 CLIP_UH_0_255(in1); \ 561 } while (0) 562 563 /* Description : Clips all signed word elements of input vector 564 * between 0 & 255 565 * Arguments : Input/output - val 566 * Return Type - signed word 567 */ 568 #define CLIP_SW_0_255(val) do { \ 569 const v4i32 max_m = __msa_ldi_w(255); \ 570 val = __msa_maxi_s_w((v4i32)val, 0); \ 571 val = __msa_min_s_w(max_m, (v4i32)val); \ 572 } while (0) 573 574 #define CLIP_SW4_0_255(in0, in1, in2, in3) do { \ 575 CLIP_SW_0_255(in0); \ 576 CLIP_SW_0_255(in1); \ 577 CLIP_SW_0_255(in2); \ 578 CLIP_SW_0_255(in3); \ 579 } while (0) 580 581 /* Description : Horizontal addition of 4 signed word elements of input vector 582 * Arguments : Input - in (signed word vector) 583 * Output - sum_m (i32 sum) 584 * Return Type - signed word (GP) 585 * Details : 4 signed word elements of 'in' vector are added together and 586 * the resulting integer sum is returned 587 */ 588 static WEBP_INLINE int32_t func_hadd_sw_s32(v4i32 in) { 589 const v2i64 res0_m = __msa_hadd_s_d((v4i32)in, (v4i32)in); 590 const v2i64 res1_m = __msa_splati_d(res0_m, 1); 591 const v2i64 out = res0_m + res1_m; 592 int32_t sum_m = __msa_copy_s_w((v4i32)out, 0); 593 return sum_m; 594 } 595 #define HADD_SW_S32(in) func_hadd_sw_s32(in) 596 597 /* Description : Horizontal addition of 8 signed halfword elements 598 * Arguments : Input - in (signed halfword vector) 599 * Output - sum_m (s32 sum) 600 * Return Type - signed word 601 * Details : 8 signed halfword elements of input vector are added 602 * together and the resulting integer sum is returned 603 */ 604 static WEBP_INLINE int32_t func_hadd_sh_s32(v8i16 in) { 605 const v4i32 res = __msa_hadd_s_w(in, in); 606 const v2i64 res0 = __msa_hadd_s_d(res, res); 607 const v2i64 res1 = __msa_splati_d(res0, 1); 608 const v2i64 res2 = res0 + res1; 609 const int32_t sum_m = __msa_copy_s_w((v4i32)res2, 0); 610 return sum_m; 611 } 612 #define HADD_SH_S32(in) func_hadd_sh_s32(in) 613 614 /* Description : Horizontal addition of 8 unsigned halfword elements 615 * Arguments : Input - in (unsigned halfword vector) 616 * Output - sum_m (u32 sum) 617 * Return Type - unsigned word 618 * Details : 8 unsigned halfword elements of input vector are added 619 * together and the resulting integer sum is returned 620 */ 621 static WEBP_INLINE uint32_t func_hadd_uh_u32(v8u16 in) { 622 uint32_t sum_m; 623 const v4u32 res_m = __msa_hadd_u_w(in, in); 624 v2u64 res0_m = __msa_hadd_u_d(res_m, res_m); 625 v2u64 res1_m = (v2u64)__msa_splati_d((v2i64)res0_m, 1); 626 res0_m = res0_m + res1_m; 627 sum_m = __msa_copy_s_w((v4i32)res0_m, 0); 628 return sum_m; 629 } 630 #define HADD_UH_U32(in) func_hadd_uh_u32(in) 631 632 /* Description : Horizontal addition of signed half word vector elements 633 Arguments : Inputs - in0, in1 634 Outputs - out0, out1 635 Return Type - as per RTYPE 636 Details : Each signed odd half word element from 'in0' is added to 637 even signed half word element from 'in0' (pairwise) and the 638 halfword result is written in 'out0' 639 */ 640 #define HADD_SH2(RTYPE, in0, in1, out0, out1) do { \ 641 out0 = (RTYPE)__msa_hadd_s_w((v8i16)in0, (v8i16)in0); \ 642 out1 = (RTYPE)__msa_hadd_s_w((v8i16)in1, (v8i16)in1); \ 643 } while (0) 644 #define HADD_SH2_SW(...) HADD_SH2(v4i32, __VA_ARGS__) 645 646 #define HADD_SH4(RTYPE, in0, in1, in2, in3, out0, out1, out2, out3) do { \ 647 HADD_SH2(RTYPE, in0, in1, out0, out1); \ 648 HADD_SH2(RTYPE, in2, in3, out2, out3); \ 649 } while (0) 650 #define HADD_SH4_SW(...) HADD_SH4(v4i32, __VA_ARGS__) 651 652 /* Description : Horizontal subtraction of unsigned byte vector elements 653 * Arguments : Inputs - in0, in1 654 * Outputs - out0, out1 655 * Return Type - as per RTYPE 656 * Details : Each unsigned odd byte element from 'in0' is subtracted from 657 * even unsigned byte element from 'in0' (pairwise) and the 658 * halfword result is written to 'out0' 659 */ 660 #define HSUB_UB2(RTYPE, in0, in1, out0, out1) do { \ 661 out0 = (RTYPE)__msa_hsub_u_h((v16u8)in0, (v16u8)in0); \ 662 out1 = (RTYPE)__msa_hsub_u_h((v16u8)in1, (v16u8)in1); \ 663 } while (0) 664 #define HSUB_UB2_UH(...) HSUB_UB2(v8u16, __VA_ARGS__) 665 #define HSUB_UB2_SH(...) HSUB_UB2(v8i16, __VA_ARGS__) 666 #define HSUB_UB2_SW(...) HSUB_UB2(v4i32, __VA_ARGS__) 667 668 /* Description : Set element n input vector to GPR value 669 * Arguments : Inputs - in0, in1, in2, in3 670 * Output - out 671 * Return Type - as per RTYPE 672 * Details : Set element 0 in vector 'out' to value specified in 'in0' 673 */ 674 #define INSERT_W2(RTYPE, in0, in1, out) do { \ 675 out = (RTYPE)__msa_insert_w((v4i32)out, 0, in0); \ 676 out = (RTYPE)__msa_insert_w((v4i32)out, 1, in1); \ 677 } while (0) 678 #define INSERT_W2_UB(...) INSERT_W2(v16u8, __VA_ARGS__) 679 #define INSERT_W2_SB(...) INSERT_W2(v16i8, __VA_ARGS__) 680 681 #define INSERT_W4(RTYPE, in0, in1, in2, in3, out) do { \ 682 out = (RTYPE)__msa_insert_w((v4i32)out, 0, in0); \ 683 out = (RTYPE)__msa_insert_w((v4i32)out, 1, in1); \ 684 out = (RTYPE)__msa_insert_w((v4i32)out, 2, in2); \ 685 out = (RTYPE)__msa_insert_w((v4i32)out, 3, in3); \ 686 } while (0) 687 #define INSERT_W4_UB(...) INSERT_W4(v16u8, __VA_ARGS__) 688 #define INSERT_W4_SB(...) INSERT_W4(v16i8, __VA_ARGS__) 689 #define INSERT_W4_SW(...) INSERT_W4(v4i32, __VA_ARGS__) 690 691 /* Description : Set element n of double word input vector to GPR value 692 * Arguments : Inputs - in0, in1 693 * Output - out 694 * Return Type - as per RTYPE 695 * Details : Set element 0 in vector 'out' to GPR value specified in 'in0' 696 * Set element 1 in vector 'out' to GPR value specified in 'in1' 697 */ 698 #define INSERT_D2(RTYPE, in0, in1, out) do { \ 699 out = (RTYPE)__msa_insert_d((v2i64)out, 0, in0); \ 700 out = (RTYPE)__msa_insert_d((v2i64)out, 1, in1); \ 701 } while (0) 702 #define INSERT_D2_UB(...) INSERT_D2(v16u8, __VA_ARGS__) 703 #define INSERT_D2_SB(...) INSERT_D2(v16i8, __VA_ARGS__) 704 705 /* Description : Interleave even byte elements from vectors 706 * Arguments : Inputs - in0, in1, in2, in3 707 * Outputs - out0, out1 708 * Return Type - as per RTYPE 709 * Details : Even byte elements of 'in0' and 'in1' are interleaved 710 * and written to 'out0' 711 */ 712 #define ILVEV_B2(RTYPE, in0, in1, in2, in3, out0, out1) do { \ 713 out0 = (RTYPE)__msa_ilvev_b((v16i8)in1, (v16i8)in0); \ 714 out1 = (RTYPE)__msa_ilvev_b((v16i8)in3, (v16i8)in2); \ 715 } while (0) 716 #define ILVEV_B2_UB(...) ILVEV_B2(v16u8, __VA_ARGS__) 717 #define ILVEV_B2_SB(...) ILVEV_B2(v16i8, __VA_ARGS__) 718 #define ILVEV_B2_UH(...) ILVEV_B2(v8u16, __VA_ARGS__) 719 #define ILVEV_B2_SH(...) ILVEV_B2(v8i16, __VA_ARGS__) 720 #define ILVEV_B2_SD(...) ILVEV_B2(v2i64, __VA_ARGS__) 721 722 /* Description : Interleave odd byte elements from vectors 723 * Arguments : Inputs - in0, in1, in2, in3 724 * Outputs - out0, out1 725 * Return Type - as per RTYPE 726 * Details : Odd byte elements of 'in0' and 'in1' are interleaved 727 * and written to 'out0' 728 */ 729 #define ILVOD_B2(RTYPE, in0, in1, in2, in3, out0, out1) do { \ 730 out0 = (RTYPE)__msa_ilvod_b((v16i8)in1, (v16i8)in0); \ 731 out1 = (RTYPE)__msa_ilvod_b((v16i8)in3, (v16i8)in2); \ 732 } while (0) 733 #define ILVOD_B2_UB(...) ILVOD_B2(v16u8, __VA_ARGS__) 734 #define ILVOD_B2_SB(...) ILVOD_B2(v16i8, __VA_ARGS__) 735 #define ILVOD_B2_UH(...) ILVOD_B2(v8u16, __VA_ARGS__) 736 #define ILVOD_B2_SH(...) ILVOD_B2(v8i16, __VA_ARGS__) 737 #define ILVOD_B2_SD(...) ILVOD_B2(v2i64, __VA_ARGS__) 738 739 /* Description : Interleave even halfword elements from vectors 740 * Arguments : Inputs - in0, in1, in2, in3 741 * Outputs - out0, out1 742 * Return Type - as per RTYPE 743 * Details : Even halfword elements of 'in0' and 'in1' are interleaved 744 * and written to 'out0' 745 */ 746 #define ILVEV_H2(RTYPE, in0, in1, in2, in3, out0, out1) do { \ 747 out0 = (RTYPE)__msa_ilvev_h((v8i16)in1, (v8i16)in0); \ 748 out1 = (RTYPE)__msa_ilvev_h((v8i16)in3, (v8i16)in2); \ 749 } while (0) 750 #define ILVEV_H2_UB(...) ILVEV_H2(v16u8, __VA_ARGS__) 751 #define ILVEV_H2_UH(...) ILVEV_H2(v8u16, __VA_ARGS__) 752 #define ILVEV_H2_SH(...) ILVEV_H2(v8i16, __VA_ARGS__) 753 #define ILVEV_H2_SW(...) ILVEV_H2(v4i32, __VA_ARGS__) 754 755 /* Description : Interleave odd halfword elements from vectors 756 * Arguments : Inputs - in0, in1, in2, in3 757 * Outputs - out0, out1 758 * Return Type - as per RTYPE 759 * Details : Odd halfword elements of 'in0' and 'in1' are interleaved 760 * and written to 'out0' 761 */ 762 #define ILVOD_H2(RTYPE, in0, in1, in2, in3, out0, out1) do { \ 763 out0 = (RTYPE)__msa_ilvod_h((v8i16)in1, (v8i16)in0); \ 764 out1 = (RTYPE)__msa_ilvod_h((v8i16)in3, (v8i16)in2); \ 765 } while (0) 766 #define ILVOD_H2_UB(...) ILVOD_H2(v16u8, __VA_ARGS__) 767 #define ILVOD_H2_UH(...) ILVOD_H2(v8u16, __VA_ARGS__) 768 #define ILVOD_H2_SH(...) ILVOD_H2(v8i16, __VA_ARGS__) 769 #define ILVOD_H2_SW(...) ILVOD_H2(v4i32, __VA_ARGS__) 770 771 /* Description : Interleave even word elements from vectors 772 * Arguments : Inputs - in0, in1, in2, in3 773 * Outputs - out0, out1 774 * Return Type - as per RTYPE 775 * Details : Even word elements of 'in0' and 'in1' are interleaved 776 * and written to 'out0' 777 */ 778 #define ILVEV_W2(RTYPE, in0, in1, in2, in3, out0, out1) do { \ 779 out0 = (RTYPE)__msa_ilvev_w((v4i32)in1, (v4i32)in0); \ 780 out1 = (RTYPE)__msa_ilvev_w((v4i32)in3, (v4i32)in2); \ 781 } while (0) 782 #define ILVEV_W2_UB(...) ILVEV_W2(v16u8, __VA_ARGS__) 783 #define ILVEV_W2_SB(...) ILVEV_W2(v16i8, __VA_ARGS__) 784 #define ILVEV_W2_UH(...) ILVEV_W2(v8u16, __VA_ARGS__) 785 #define ILVEV_W2_SD(...) ILVEV_W2(v2i64, __VA_ARGS__) 786 787 /* Description : Interleave even-odd word elements from vectors 788 * Arguments : Inputs - in0, in1, in2, in3 789 * Outputs - out0, out1 790 * Return Type - as per RTYPE 791 * Details : Even word elements of 'in0' and 'in1' are interleaved 792 * and written to 'out0' 793 * Odd word elements of 'in2' and 'in3' are interleaved 794 * and written to 'out1' 795 */ 796 #define ILVEVOD_W2(RTYPE, in0, in1, in2, in3, out0, out1) do { \ 797 out0 = (RTYPE)__msa_ilvev_w((v4i32)in1, (v4i32)in0); \ 798 out1 = (RTYPE)__msa_ilvod_w((v4i32)in3, (v4i32)in2); \ 799 } while (0) 800 #define ILVEVOD_W2_UB(...) ILVEVOD_W2(v16u8, __VA_ARGS__) 801 #define ILVEVOD_W2_UH(...) ILVEVOD_W2(v8u16, __VA_ARGS__) 802 #define ILVEVOD_W2_SH(...) ILVEVOD_W2(v8i16, __VA_ARGS__) 803 #define ILVEVOD_W2_SW(...) ILVEVOD_W2(v4i32, __VA_ARGS__) 804 805 /* Description : Interleave even-odd half-word elements from vectors 806 * Arguments : Inputs - in0, in1, in2, in3 807 * Outputs - out0, out1 808 * Return Type - as per RTYPE 809 * Details : Even half-word elements of 'in0' and 'in1' are interleaved 810 * and written to 'out0' 811 * Odd half-word elements of 'in2' and 'in3' are interleaved 812 * and written to 'out1' 813 */ 814 #define ILVEVOD_H2(RTYPE, in0, in1, in2, in3, out0, out1) do { \ 815 out0 = (RTYPE)__msa_ilvev_h((v8i16)in1, (v8i16)in0); \ 816 out1 = (RTYPE)__msa_ilvod_h((v8i16)in3, (v8i16)in2); \ 817 } while (0) 818 #define ILVEVOD_H2_UB(...) ILVEVOD_H2(v16u8, __VA_ARGS__) 819 #define ILVEVOD_H2_UH(...) ILVEVOD_H2(v8u16, __VA_ARGS__) 820 #define ILVEVOD_H2_SH(...) ILVEVOD_H2(v8i16, __VA_ARGS__) 821 #define ILVEVOD_H2_SW(...) ILVEVOD_H2(v4i32, __VA_ARGS__) 822 823 /* Description : Interleave even double word elements from vectors 824 * Arguments : Inputs - in0, in1, in2, in3 825 * Outputs - out0, out1 826 * Return Type - as per RTYPE 827 * Details : Even double word elements of 'in0' and 'in1' are interleaved 828 * and written to 'out0' 829 */ 830 #define ILVEV_D2(RTYPE, in0, in1, in2, in3, out0, out1) do { \ 831 out0 = (RTYPE)__msa_ilvev_d((v2i64)in1, (v2i64)in0); \ 832 out1 = (RTYPE)__msa_ilvev_d((v2i64)in3, (v2i64)in2); \ 833 } while (0) 834 #define ILVEV_D2_UB(...) ILVEV_D2(v16u8, __VA_ARGS__) 835 #define ILVEV_D2_SB(...) ILVEV_D2(v16i8, __VA_ARGS__) 836 #define ILVEV_D2_SW(...) ILVEV_D2(v4i32, __VA_ARGS__) 837 #define ILVEV_D2_SD(...) ILVEV_D2(v2i64, __VA_ARGS__) 838 839 /* Description : Interleave left half of byte elements from vectors 840 * Arguments : Inputs - in0, in1, in2, in3 841 * Outputs - out0, out1 842 * Return Type - as per RTYPE 843 * Details : Left half of byte elements of 'in0' and 'in1' are interleaved 844 * and written to 'out0'. 845 */ 846 #define ILVL_B2(RTYPE, in0, in1, in2, in3, out0, out1) do { \ 847 out0 = (RTYPE)__msa_ilvl_b((v16i8)in0, (v16i8)in1); \ 848 out1 = (RTYPE)__msa_ilvl_b((v16i8)in2, (v16i8)in3); \ 849 } while (0) 850 #define ILVL_B2_UB(...) ILVL_B2(v16u8, __VA_ARGS__) 851 #define ILVL_B2_SB(...) ILVL_B2(v16i8, __VA_ARGS__) 852 #define ILVL_B2_UH(...) ILVL_B2(v8u16, __VA_ARGS__) 853 #define ILVL_B2_SH(...) ILVL_B2(v8i16, __VA_ARGS__) 854 #define ILVL_B2_SW(...) ILVL_B2(v4i32, __VA_ARGS__) 855 856 /* Description : Interleave right half of byte elements from vectors 857 * Arguments : Inputs - in0, in1, in2, in3 858 * Outputs - out0, out1 859 * Return Type - as per RTYPE 860 * Details : Right half of byte elements of 'in0' and 'in1' are interleaved 861 * and written to out0. 862 */ 863 #define ILVR_B2(RTYPE, in0, in1, in2, in3, out0, out1) do { \ 864 out0 = (RTYPE)__msa_ilvr_b((v16i8)in0, (v16i8)in1); \ 865 out1 = (RTYPE)__msa_ilvr_b((v16i8)in2, (v16i8)in3); \ 866 } while (0) 867 #define ILVR_B2_UB(...) ILVR_B2(v16u8, __VA_ARGS__) 868 #define ILVR_B2_SB(...) ILVR_B2(v16i8, __VA_ARGS__) 869 #define ILVR_B2_UH(...) ILVR_B2(v8u16, __VA_ARGS__) 870 #define ILVR_B2_SH(...) ILVR_B2(v8i16, __VA_ARGS__) 871 #define ILVR_B2_SW(...) ILVR_B2(v4i32, __VA_ARGS__) 872 873 #define ILVR_B4(RTYPE, in0, in1, in2, in3, in4, in5, in6, in7, \ 874 out0, out1, out2, out3) do { \ 875 ILVR_B2(RTYPE, in0, in1, in2, in3, out0, out1); \ 876 ILVR_B2(RTYPE, in4, in5, in6, in7, out2, out3); \ 877 } while (0) 878 #define ILVR_B4_UB(...) ILVR_B4(v16u8, __VA_ARGS__) 879 #define ILVR_B4_SB(...) ILVR_B4(v16i8, __VA_ARGS__) 880 #define ILVR_B4_UH(...) ILVR_B4(v8u16, __VA_ARGS__) 881 #define ILVR_B4_SH(...) ILVR_B4(v8i16, __VA_ARGS__) 882 #define ILVR_B4_SW(...) ILVR_B4(v4i32, __VA_ARGS__) 883 884 /* Description : Interleave right half of halfword elements from vectors 885 * Arguments : Inputs - in0, in1, in2, in3 886 * Outputs - out0, out1 887 * Return Type - as per RTYPE 888 * Details : Right half of halfword elements of 'in0' and 'in1' are 889 * interleaved and written to 'out0'. 890 */ 891 #define ILVR_H2(RTYPE, in0, in1, in2, in3, out0, out1) do { \ 892 out0 = (RTYPE)__msa_ilvr_h((v8i16)in0, (v8i16)in1); \ 893 out1 = (RTYPE)__msa_ilvr_h((v8i16)in2, (v8i16)in3); \ 894 } while (0) 895 #define ILVR_H2_UB(...) ILVR_H2(v16u8, __VA_ARGS__) 896 #define ILVR_H2_SH(...) ILVR_H2(v8i16, __VA_ARGS__) 897 #define ILVR_H2_SW(...) ILVR_H2(v4i32, __VA_ARGS__) 898 899 #define ILVR_H4(RTYPE, in0, in1, in2, in3, in4, in5, in6, in7, \ 900 out0, out1, out2, out3) do { \ 901 ILVR_H2(RTYPE, in0, in1, in2, in3, out0, out1); \ 902 ILVR_H2(RTYPE, in4, in5, in6, in7, out2, out3); \ 903 } while (0) 904 #define ILVR_H4_UB(...) ILVR_H4(v16u8, __VA_ARGS__) 905 #define ILVR_H4_SH(...) ILVR_H4(v8i16, __VA_ARGS__) 906 #define ILVR_H4_SW(...) ILVR_H4(v4i32, __VA_ARGS__) 907 908 /* Description : Interleave right half of double word elements from vectors 909 * Arguments : Inputs - in0, in1, in2, in3 910 * Outputs - out0, out1 911 * Return Type - as per RTYPE 912 * Details : Right half of double word elements of 'in0' and 'in1' are 913 * interleaved and written to 'out0'. 914 */ 915 #define ILVR_D2(RTYPE, in0, in1, in2, in3, out0, out1) do { \ 916 out0 = (RTYPE)__msa_ilvr_d((v2i64)in0, (v2i64)in1); \ 917 out1 = (RTYPE)__msa_ilvr_d((v2i64)in2, (v2i64)in3); \ 918 } while (0) 919 #define ILVR_D2_UB(...) ILVR_D2(v16u8, __VA_ARGS__) 920 #define ILVR_D2_SB(...) ILVR_D2(v16i8, __VA_ARGS__) 921 #define ILVR_D2_SH(...) ILVR_D2(v8i16, __VA_ARGS__) 922 923 #define ILVR_D4(RTYPE, in0, in1, in2, in3, in4, in5, in6, in7, \ 924 out0, out1, out2, out3) do { \ 925 ILVR_D2(RTYPE, in0, in1, in2, in3, out0, out1); \ 926 ILVR_D2(RTYPE, in4, in5, in6, in7, out2, out3); \ 927 } while (0) 928 #define ILVR_D4_SB(...) ILVR_D4(v16i8, __VA_ARGS__) 929 #define ILVR_D4_UB(...) ILVR_D4(v16u8, __VA_ARGS__) 930 931 /* Description : Interleave both left and right half of input vectors 932 * Arguments : Inputs - in0, in1 933 * Outputs - out0, out1 934 * Return Type - as per RTYPE 935 * Details : Right half of byte elements from 'in0' and 'in1' are 936 * interleaved and written to 'out0' 937 */ 938 #define ILVRL_B2(RTYPE, in0, in1, out0, out1) do { \ 939 out0 = (RTYPE)__msa_ilvr_b((v16i8)in0, (v16i8)in1); \ 940 out1 = (RTYPE)__msa_ilvl_b((v16i8)in0, (v16i8)in1); \ 941 } while (0) 942 #define ILVRL_B2_UB(...) ILVRL_B2(v16u8, __VA_ARGS__) 943 #define ILVRL_B2_SB(...) ILVRL_B2(v16i8, __VA_ARGS__) 944 #define ILVRL_B2_UH(...) ILVRL_B2(v8u16, __VA_ARGS__) 945 #define ILVRL_B2_SH(...) ILVRL_B2(v8i16, __VA_ARGS__) 946 #define ILVRL_B2_SW(...) ILVRL_B2(v4i32, __VA_ARGS__) 947 948 #define ILVRL_H2(RTYPE, in0, in1, out0, out1) do { \ 949 out0 = (RTYPE)__msa_ilvr_h((v8i16)in0, (v8i16)in1); \ 950 out1 = (RTYPE)__msa_ilvl_h((v8i16)in0, (v8i16)in1); \ 951 } while (0) 952 #define ILVRL_H2_UB(...) ILVRL_H2(v16u8, __VA_ARGS__) 953 #define ILVRL_H2_SB(...) ILVRL_H2(v16i8, __VA_ARGS__) 954 #define ILVRL_H2_SH(...) ILVRL_H2(v8i16, __VA_ARGS__) 955 #define ILVRL_H2_SW(...) ILVRL_H2(v4i32, __VA_ARGS__) 956 #define ILVRL_H2_UW(...) ILVRL_H2(v4u32, __VA_ARGS__) 957 958 #define ILVRL_W2(RTYPE, in0, in1, out0, out1) do { \ 959 out0 = (RTYPE)__msa_ilvr_w((v4i32)in0, (v4i32)in1); \ 960 out1 = (RTYPE)__msa_ilvl_w((v4i32)in0, (v4i32)in1); \ 961 } while (0) 962 #define ILVRL_W2_UB(...) ILVRL_W2(v16u8, __VA_ARGS__) 963 #define ILVRL_W2_SH(...) ILVRL_W2(v8i16, __VA_ARGS__) 964 #define ILVRL_W2_SW(...) ILVRL_W2(v4i32, __VA_ARGS__) 965 #define ILVRL_W2_UW(...) ILVRL_W2(v4u32, __VA_ARGS__) 966 967 /* Description : Pack even byte elements of vector pairs 968 * Arguments : Inputs - in0, in1, in2, in3 969 * Outputs - out0, out1 970 * Return Type - as per RTYPE 971 * Details : Even byte elements of 'in0' are copied to the left half of 972 * 'out0' & even byte elements of 'in1' are copied to the right 973 * half of 'out0'. 974 */ 975 #define PCKEV_B2(RTYPE, in0, in1, in2, in3, out0, out1) do { \ 976 out0 = (RTYPE)__msa_pckev_b((v16i8)in0, (v16i8)in1); \ 977 out1 = (RTYPE)__msa_pckev_b((v16i8)in2, (v16i8)in3); \ 978 } while (0) 979 #define PCKEV_B2_SB(...) PCKEV_B2(v16i8, __VA_ARGS__) 980 #define PCKEV_B2_UB(...) PCKEV_B2(v16u8, __VA_ARGS__) 981 #define PCKEV_B2_SH(...) PCKEV_B2(v8i16, __VA_ARGS__) 982 #define PCKEV_B2_SW(...) PCKEV_B2(v4i32, __VA_ARGS__) 983 984 #define PCKEV_B4(RTYPE, in0, in1, in2, in3, in4, in5, in6, in7, \ 985 out0, out1, out2, out3) do { \ 986 PCKEV_B2(RTYPE, in0, in1, in2, in3, out0, out1); \ 987 PCKEV_B2(RTYPE, in4, in5, in6, in7, out2, out3); \ 988 } while (0) 989 #define PCKEV_B4_SB(...) PCKEV_B4(v16i8, __VA_ARGS__) 990 #define PCKEV_B4_UB(...) PCKEV_B4(v16u8, __VA_ARGS__) 991 #define PCKEV_B4_SH(...) PCKEV_B4(v8i16, __VA_ARGS__) 992 #define PCKEV_B4_SW(...) PCKEV_B4(v4i32, __VA_ARGS__) 993 994 /* Description : Pack even halfword elements of vector pairs 995 * Arguments : Inputs - in0, in1, in2, in3 996 * Outputs - out0, out1 997 * Return Type - as per RTYPE 998 * Details : Even halfword elements of 'in0' are copied to the left half of 999 * 'out0' & even halfword elements of 'in1' are copied to the 1000 * right half of 'out0'. 1001 */ 1002 #define PCKEV_H2(RTYPE, in0, in1, in2, in3, out0, out1) do { \ 1003 out0 = (RTYPE)__msa_pckev_h((v8i16)in0, (v8i16)in1); \ 1004 out1 = (RTYPE)__msa_pckev_h((v8i16)in2, (v8i16)in3); \ 1005 } while (0) 1006 #define PCKEV_H2_UH(...) PCKEV_H2(v8u16, __VA_ARGS__) 1007 #define PCKEV_H2_SH(...) PCKEV_H2(v8i16, __VA_ARGS__) 1008 #define PCKEV_H2_SW(...) PCKEV_H2(v4i32, __VA_ARGS__) 1009 #define PCKEV_H2_UW(...) PCKEV_H2(v4u32, __VA_ARGS__) 1010 1011 /* Description : Pack even word elements of vector pairs 1012 * Arguments : Inputs - in0, in1, in2, in3 1013 * Outputs - out0, out1 1014 * Return Type - as per RTYPE 1015 * Details : Even word elements of 'in0' are copied to the left half of 1016 * 'out0' & even word elements of 'in1' are copied to the 1017 * right half of 'out0'. 1018 */ 1019 #define PCKEV_W2(RTYPE, in0, in1, in2, in3, out0, out1) do { \ 1020 out0 = (RTYPE)__msa_pckev_w((v4i32)in0, (v4i32)in1); \ 1021 out1 = (RTYPE)__msa_pckev_w((v4i32)in2, (v4i32)in3); \ 1022 } while (0) 1023 #define PCKEV_W2_UH(...) PCKEV_W2(v8u16, __VA_ARGS__) 1024 #define PCKEV_W2_SH(...) PCKEV_W2(v8i16, __VA_ARGS__) 1025 #define PCKEV_W2_SW(...) PCKEV_W2(v4i32, __VA_ARGS__) 1026 #define PCKEV_W2_UW(...) PCKEV_W2(v4u32, __VA_ARGS__) 1027 1028 /* Description : Pack odd halfword elements of vector pairs 1029 * Arguments : Inputs - in0, in1, in2, in3 1030 * Outputs - out0, out1 1031 * Return Type - as per RTYPE 1032 * Details : Odd halfword elements of 'in0' are copied to the left half of 1033 * 'out0' & odd halfword elements of 'in1' are copied to the 1034 * right half of 'out0'. 1035 */ 1036 #define PCKOD_H2(RTYPE, in0, in1, in2, in3, out0, out1) do { \ 1037 out0 = (RTYPE)__msa_pckod_h((v8i16)in0, (v8i16)in1); \ 1038 out1 = (RTYPE)__msa_pckod_h((v8i16)in2, (v8i16)in3); \ 1039 } while (0) 1040 #define PCKOD_H2_UH(...) PCKOD_H2(v8u16, __VA_ARGS__) 1041 #define PCKOD_H2_SH(...) PCKOD_H2(v8i16, __VA_ARGS__) 1042 #define PCKOD_H2_SW(...) PCKOD_H2(v4i32, __VA_ARGS__) 1043 #define PCKOD_H2_UW(...) PCKOD_H2(v4u32, __VA_ARGS__) 1044 1045 /* Description : Arithmetic immediate shift right all elements of word vector 1046 * Arguments : Inputs - in0, in1, shift 1047 * Outputs - in place operation 1048 * Return Type - as per input vector RTYPE 1049 * Details : Each element of vector 'in0' is right shifted by 'shift' and 1050 * the result is written in-place. 'shift' is a GP variable. 1051 */ 1052 #define SRAI_W2(RTYPE, in0, in1, shift_val) do { \ 1053 in0 = (RTYPE)SRAI_W(in0, shift_val); \ 1054 in1 = (RTYPE)SRAI_W(in1, shift_val); \ 1055 } while (0) 1056 #define SRAI_W2_SW(...) SRAI_W2(v4i32, __VA_ARGS__) 1057 #define SRAI_W2_UW(...) SRAI_W2(v4u32, __VA_ARGS__) 1058 1059 #define SRAI_W4(RTYPE, in0, in1, in2, in3, shift_val) do { \ 1060 SRAI_W2(RTYPE, in0, in1, shift_val); \ 1061 SRAI_W2(RTYPE, in2, in3, shift_val); \ 1062 } while (0) 1063 #define SRAI_W4_SW(...) SRAI_W4(v4i32, __VA_ARGS__) 1064 #define SRAI_W4_UW(...) SRAI_W4(v4u32, __VA_ARGS__) 1065 1066 /* Description : Arithmetic shift right all elements of half-word vector 1067 * Arguments : Inputs - in0, in1, shift 1068 * Outputs - in place operation 1069 * Return Type - as per input vector RTYPE 1070 * Details : Each element of vector 'in0' is right shifted by 'shift' and 1071 * the result is written in-place. 'shift' is a GP variable. 1072 */ 1073 #define SRAI_H2(RTYPE, in0, in1, shift_val) do { \ 1074 in0 = (RTYPE)SRAI_H(in0, shift_val); \ 1075 in1 = (RTYPE)SRAI_H(in1, shift_val); \ 1076 } while (0) 1077 #define SRAI_H2_SH(...) SRAI_H2(v8i16, __VA_ARGS__) 1078 #define SRAI_H2_UH(...) SRAI_H2(v8u16, __VA_ARGS__) 1079 1080 /* Description : Arithmetic rounded shift right all elements of word vector 1081 * Arguments : Inputs - in0, in1, shift 1082 * Outputs - in place operation 1083 * Return Type - as per input vector RTYPE 1084 * Details : Each element of vector 'in0' is right shifted by 'shift' and 1085 * the result is written in-place. 'shift' is a GP variable. 1086 */ 1087 #define SRARI_W2(RTYPE, in0, in1, shift) do { \ 1088 in0 = (RTYPE)__msa_srari_w((v4i32)in0, shift); \ 1089 in1 = (RTYPE)__msa_srari_w((v4i32)in1, shift); \ 1090 } while (0) 1091 #define SRARI_W2_SW(...) SRARI_W2(v4i32, __VA_ARGS__) 1092 1093 #define SRARI_W4(RTYPE, in0, in1, in2, in3, shift) do { \ 1094 SRARI_W2(RTYPE, in0, in1, shift); \ 1095 SRARI_W2(RTYPE, in2, in3, shift); \ 1096 } while (0) 1097 #define SRARI_W4_SH(...) SRARI_W4(v8i16, __VA_ARGS__) 1098 #define SRARI_W4_UW(...) SRARI_W4(v4u32, __VA_ARGS__) 1099 #define SRARI_W4_SW(...) SRARI_W4(v4i32, __VA_ARGS__) 1100 1101 /* Description : Shift right arithmetic rounded double words 1102 * Arguments : Inputs - in0, in1, shift 1103 * Outputs - in place operation 1104 * Return Type - as per RTYPE 1105 * Details : Each element of vector 'in0' is shifted right arithmetically by 1106 * the number of bits in the corresponding element in the vector 1107 * 'shift'. The last discarded bit is added to shifted value for 1108 * rounding and the result is written in-place. 1109 * 'shift' is a vector. 1110 */ 1111 #define SRAR_D2(RTYPE, in0, in1, shift) do { \ 1112 in0 = (RTYPE)__msa_srar_d((v2i64)in0, (v2i64)shift); \ 1113 in1 = (RTYPE)__msa_srar_d((v2i64)in1, (v2i64)shift); \ 1114 } while (0) 1115 #define SRAR_D2_SW(...) SRAR_D2(v4i32, __VA_ARGS__) 1116 #define SRAR_D2_SD(...) SRAR_D2(v2i64, __VA_ARGS__) 1117 #define SRAR_D2_UD(...) SRAR_D2(v2u64, __VA_ARGS__) 1118 1119 #define SRAR_D4(RTYPE, in0, in1, in2, in3, shift) do { \ 1120 SRAR_D2(RTYPE, in0, in1, shift); \ 1121 SRAR_D2(RTYPE, in2, in3, shift); \ 1122 } while (0) 1123 #define SRAR_D4_SD(...) SRAR_D4(v2i64, __VA_ARGS__) 1124 #define SRAR_D4_UD(...) SRAR_D4(v2u64, __VA_ARGS__) 1125 1126 /* Description : Addition of 2 pairs of half-word vectors 1127 * Arguments : Inputs - in0, in1, in2, in3 1128 * Outputs - out0, out1 1129 * Details : Each element in 'in0' is added to 'in1' and result is written 1130 * to 'out0'. 1131 */ 1132 #define ADDVI_H2(RTYPE, in0, in1, in2, in3, out0, out1) do { \ 1133 out0 = (RTYPE)ADDVI_H(in0, in1); \ 1134 out1 = (RTYPE)ADDVI_H(in2, in3); \ 1135 } while (0) 1136 #define ADDVI_H2_SH(...) ADDVI_H2(v8i16, __VA_ARGS__) 1137 #define ADDVI_H2_UH(...) ADDVI_H2(v8u16, __VA_ARGS__) 1138 1139 /* Description : Addition of 2 pairs of word vectors 1140 * Arguments : Inputs - in0, in1, in2, in3 1141 * Outputs - out0, out1 1142 * Details : Each element in 'in0' is added to 'in1' and result is written 1143 * to 'out0'. 1144 */ 1145 #define ADDVI_W2(RTYPE, in0, in1, in2, in3, out0, out1) do { \ 1146 out0 = (RTYPE)ADDVI_W(in0, in1); \ 1147 out1 = (RTYPE)ADDVI_W(in2, in3); \ 1148 } while (0) 1149 #define ADDVI_W2_SW(...) ADDVI_W2(v4i32, __VA_ARGS__) 1150 1151 /* Description : Fill 2 pairs of word vectors with GP registers 1152 * Arguments : Inputs - in0, in1 1153 * Outputs - out0, out1 1154 * Details : GP register in0 is replicated in each word element of out0 1155 * GP register in1 is replicated in each word element of out1 1156 */ 1157 #define FILL_W2(RTYPE, in0, in1, out0, out1) do { \ 1158 out0 = (RTYPE)__msa_fill_w(in0); \ 1159 out1 = (RTYPE)__msa_fill_w(in1); \ 1160 } while (0) 1161 #define FILL_W2_SW(...) FILL_W2(v4i32, __VA_ARGS__) 1162 1163 /* Description : Addition of 2 pairs of vectors 1164 * Arguments : Inputs - in0, in1, in2, in3 1165 * Outputs - out0, out1 1166 * Details : Each element in 'in0' is added to 'in1' and result is written 1167 * to 'out0'. 1168 */ 1169 #define ADD2(in0, in1, in2, in3, out0, out1) do { \ 1170 out0 = in0 + in1; \ 1171 out1 = in2 + in3; \ 1172 } while (0) 1173 1174 #define ADD4(in0, in1, in2, in3, in4, in5, in6, in7, \ 1175 out0, out1, out2, out3) do { \ 1176 ADD2(in0, in1, in2, in3, out0, out1); \ 1177 ADD2(in4, in5, in6, in7, out2, out3); \ 1178 } while (0) 1179 1180 /* Description : Subtraction of 2 pairs of vectors 1181 * Arguments : Inputs - in0, in1, in2, in3 1182 * Outputs - out0, out1 1183 * Details : Each element in 'in1' is subtracted from 'in0' and result is 1184 * written to 'out0'. 1185 */ 1186 #define SUB2(in0, in1, in2, in3, out0, out1) do { \ 1187 out0 = in0 - in1; \ 1188 out1 = in2 - in3; \ 1189 } while (0) 1190 1191 #define SUB3(in0, in1, in2, in3, in4, in5, out0, out1, out2) do { \ 1192 out0 = in0 - in1; \ 1193 out1 = in2 - in3; \ 1194 out2 = in4 - in5; \ 1195 } while (0) 1196 1197 #define SUB4(in0, in1, in2, in3, in4, in5, in6, in7, \ 1198 out0, out1, out2, out3) do { \ 1199 out0 = in0 - in1; \ 1200 out1 = in2 - in3; \ 1201 out2 = in4 - in5; \ 1202 out3 = in6 - in7; \ 1203 } while (0) 1204 1205 /* Description : Addition - Subtraction of input vectors 1206 * Arguments : Inputs - in0, in1 1207 * Outputs - out0, out1 1208 * Details : Each element in 'in1' is added to 'in0' and result is 1209 * written to 'out0'. 1210 * Each element in 'in1' is subtracted from 'in0' and result is 1211 * written to 'out1'. 1212 */ 1213 #define ADDSUB2(in0, in1, out0, out1) do { \ 1214 out0 = in0 + in1; \ 1215 out1 = in0 - in1; \ 1216 } while (0) 1217 1218 /* Description : Multiplication of pairs of vectors 1219 * Arguments : Inputs - in0, in1, in2, in3 1220 * Outputs - out0, out1 1221 * Details : Each element from 'in0' is multiplied with elements from 'in1' 1222 * and the result is written to 'out0' 1223 */ 1224 #define MUL2(in0, in1, in2, in3, out0, out1) do { \ 1225 out0 = in0 * in1; \ 1226 out1 = in2 * in3; \ 1227 } while (0) 1228 1229 #define MUL4(in0, in1, in2, in3, in4, in5, in6, in7, \ 1230 out0, out1, out2, out3) do { \ 1231 MUL2(in0, in1, in2, in3, out0, out1); \ 1232 MUL2(in4, in5, in6, in7, out2, out3); \ 1233 } while (0) 1234 1235 /* Description : Sign extend halfword elements from right half of the vector 1236 * Arguments : Input - in (halfword vector) 1237 * Output - out (sign extended word vector) 1238 * Return Type - signed word 1239 * Details : Sign bit of halfword elements from input vector 'in' is 1240 * extracted and interleaved with same vector 'in0' to generate 1241 * 4 word elements keeping sign intact 1242 */ 1243 #define UNPCK_R_SH_SW(in, out) do { \ 1244 const v8i16 sign_m = __msa_clti_s_h((v8i16)in, 0); \ 1245 out = (v4i32)__msa_ilvr_h(sign_m, (v8i16)in); \ 1246 } while (0) 1247 1248 /* Description : Sign extend halfword elements from input vector and return 1249 * the result in pair of vectors 1250 * Arguments : Input - in (halfword vector) 1251 * Outputs - out0, out1 (sign extended word vectors) 1252 * Return Type - signed word 1253 * Details : Sign bit of halfword elements from input vector 'in' is 1254 * extracted and interleaved right with same vector 'in0' to 1255 * generate 4 signed word elements in 'out0' 1256 * Then interleaved left with same vector 'in0' to 1257 * generate 4 signed word elements in 'out1' 1258 */ 1259 #define UNPCK_SH_SW(in, out0, out1) do { \ 1260 const v8i16 tmp_m = __msa_clti_s_h((v8i16)in, 0); \ 1261 ILVRL_H2_SW(tmp_m, in, out0, out1); \ 1262 } while (0) 1263 1264 /* Description : Butterfly of 4 input vectors 1265 * Arguments : Inputs - in0, in1, in2, in3 1266 * Outputs - out0, out1, out2, out3 1267 * Details : Butterfly operation 1268 */ 1269 #define BUTTERFLY_4(in0, in1, in2, in3, out0, out1, out2, out3) do { \ 1270 out0 = in0 + in3; \ 1271 out1 = in1 + in2; \ 1272 out2 = in1 - in2; \ 1273 out3 = in0 - in3; \ 1274 } while (0) 1275 1276 /* Description : Transpose 16x4 block into 4x16 with byte elements in vectors 1277 * Arguments : Inputs - in0, in1, in2, in3, in4, in5, in6, in7, 1278 * in8, in9, in10, in11, in12, in13, in14, in15 1279 * Outputs - out0, out1, out2, out3 1280 * Return Type - unsigned byte 1281 */ 1282 #define TRANSPOSE16x4_UB_UB(in0, in1, in2, in3, in4, in5, in6, in7, \ 1283 in8, in9, in10, in11, in12, in13, in14, in15, \ 1284 out0, out1, out2, out3) do { \ 1285 v2i64 tmp0_m, tmp1_m, tmp2_m, tmp3_m, tmp4_m, tmp5_m; \ 1286 ILVEV_W2_SD(in0, in4, in8, in12, tmp2_m, tmp3_m); \ 1287 ILVEV_W2_SD(in1, in5, in9, in13, tmp0_m, tmp1_m); \ 1288 ILVEV_D2_UB(tmp2_m, tmp3_m, tmp0_m, tmp1_m, out1, out3); \ 1289 ILVEV_W2_SD(in2, in6, in10, in14, tmp4_m, tmp5_m); \ 1290 ILVEV_W2_SD(in3, in7, in11, in15, tmp0_m, tmp1_m); \ 1291 ILVEV_D2_SD(tmp4_m, tmp5_m, tmp0_m, tmp1_m, tmp2_m, tmp3_m); \ 1292 ILVEV_B2_SD(out1, out3, tmp2_m, tmp3_m, tmp0_m, tmp1_m); \ 1293 ILVEVOD_H2_UB(tmp0_m, tmp1_m, tmp0_m, tmp1_m, out0, out2); \ 1294 ILVOD_B2_SD(out1, out3, tmp2_m, tmp3_m, tmp0_m, tmp1_m); \ 1295 ILVEVOD_H2_UB(tmp0_m, tmp1_m, tmp0_m, tmp1_m, out1, out3); \ 1296 } while (0) 1297 1298 /* Description : Transpose 16x8 block into 8x16 with byte elements in vectors 1299 * Arguments : Inputs - in0, in1, in2, in3, in4, in5, in6, in7, 1300 * in8, in9, in10, in11, in12, in13, in14, in15 1301 * Outputs - out0, out1, out2, out3, out4, out5, out6, out7 1302 * Return Type - unsigned byte 1303 */ 1304 #define TRANSPOSE16x8_UB_UB(in0, in1, in2, in3, in4, in5, in6, in7, \ 1305 in8, in9, in10, in11, in12, in13, in14, in15, \ 1306 out0, out1, out2, out3, out4, out5, \ 1307 out6, out7) do { \ 1308 v8i16 tmp0_m, tmp1_m, tmp4_m, tmp5_m, tmp6_m, tmp7_m; \ 1309 v4i32 tmp2_m, tmp3_m; \ 1310 ILVEV_D2_UB(in0, in8, in1, in9, out7, out6); \ 1311 ILVEV_D2_UB(in2, in10, in3, in11, out5, out4); \ 1312 ILVEV_D2_UB(in4, in12, in5, in13, out3, out2); \ 1313 ILVEV_D2_UB(in6, in14, in7, in15, out1, out0); \ 1314 ILVEV_B2_SH(out7, out6, out5, out4, tmp0_m, tmp1_m); \ 1315 ILVOD_B2_SH(out7, out6, out5, out4, tmp4_m, tmp5_m); \ 1316 ILVEV_B2_UB(out3, out2, out1, out0, out5, out7); \ 1317 ILVOD_B2_SH(out3, out2, out1, out0, tmp6_m, tmp7_m); \ 1318 ILVEV_H2_SW(tmp0_m, tmp1_m, out5, out7, tmp2_m, tmp3_m); \ 1319 ILVEVOD_W2_UB(tmp2_m, tmp3_m, tmp2_m, tmp3_m, out0, out4); \ 1320 ILVOD_H2_SW(tmp0_m, tmp1_m, out5, out7, tmp2_m, tmp3_m); \ 1321 ILVEVOD_W2_UB(tmp2_m, tmp3_m, tmp2_m, tmp3_m, out2, out6); \ 1322 ILVEV_H2_SW(tmp4_m, tmp5_m, tmp6_m, tmp7_m, tmp2_m, tmp3_m); \ 1323 ILVEVOD_W2_UB(tmp2_m, tmp3_m, tmp2_m, tmp3_m, out1, out5); \ 1324 ILVOD_H2_SW(tmp4_m, tmp5_m, tmp6_m, tmp7_m, tmp2_m, tmp3_m); \ 1325 ILVEVOD_W2_UB(tmp2_m, tmp3_m, tmp2_m, tmp3_m, out3, out7); \ 1326 } while (0) 1327 1328 /* Description : Transpose 4x4 block with word elements in vectors 1329 * Arguments : Inputs - in0, in1, in2, in3 1330 * Outputs - out0, out1, out2, out3 1331 * Return Type - as per RTYPE 1332 */ 1333 #define TRANSPOSE4x4_W(RTYPE, in0, in1, in2, in3, \ 1334 out0, out1, out2, out3) do { \ 1335 v4i32 s0_m, s1_m, s2_m, s3_m; \ 1336 ILVRL_W2_SW(in1, in0, s0_m, s1_m); \ 1337 ILVRL_W2_SW(in3, in2, s2_m, s3_m); \ 1338 out0 = (RTYPE)__msa_ilvr_d((v2i64)s2_m, (v2i64)s0_m); \ 1339 out1 = (RTYPE)__msa_ilvl_d((v2i64)s2_m, (v2i64)s0_m); \ 1340 out2 = (RTYPE)__msa_ilvr_d((v2i64)s3_m, (v2i64)s1_m); \ 1341 out3 = (RTYPE)__msa_ilvl_d((v2i64)s3_m, (v2i64)s1_m); \ 1342 } while (0) 1343 #define TRANSPOSE4x4_SW_SW(...) TRANSPOSE4x4_W(v4i32, __VA_ARGS__) 1344 1345 /* Description : Add block 4x4 1346 * Arguments : Inputs - in0, in1, in2, in3, pdst, stride 1347 * Details : Least significant 4 bytes from each input vector are added to 1348 * the destination bytes, clipped between 0-255 and stored. 1349 */ 1350 #define ADDBLK_ST4x4_UB(in0, in1, in2, in3, pdst, stride) do { \ 1351 uint32_t src0_m, src1_m, src2_m, src3_m; \ 1352 v8i16 inp0_m, inp1_m, res0_m, res1_m; \ 1353 v16i8 dst0_m = { 0 }; \ 1354 v16i8 dst1_m = { 0 }; \ 1355 const v16i8 zero_m = { 0 }; \ 1356 ILVR_D2_SH(in1, in0, in3, in2, inp0_m, inp1_m); \ 1357 LW4(pdst, stride, src0_m, src1_m, src2_m, src3_m); \ 1358 INSERT_W2_SB(src0_m, src1_m, dst0_m); \ 1359 INSERT_W2_SB(src2_m, src3_m, dst1_m); \ 1360 ILVR_B2_SH(zero_m, dst0_m, zero_m, dst1_m, res0_m, res1_m); \ 1361 ADD2(res0_m, inp0_m, res1_m, inp1_m, res0_m, res1_m); \ 1362 CLIP_SH2_0_255(res0_m, res1_m); \ 1363 PCKEV_B2_SB(res0_m, res0_m, res1_m, res1_m, dst0_m, dst1_m); \ 1364 ST4x4_UB(dst0_m, dst1_m, 0, 1, 0, 1, pdst, stride); \ 1365 } while (0) 1366 1367 /* Description : Pack even byte elements, extract 0 & 2 index words from pair 1368 * of results and store 4 words in destination memory as per 1369 * stride 1370 * Arguments : Inputs - in0, in1, in2, in3, pdst, stride 1371 */ 1372 #define PCKEV_ST4x4_UB(in0, in1, in2, in3, pdst, stride) do { \ 1373 v16i8 tmp0_m, tmp1_m; \ 1374 PCKEV_B2_SB(in1, in0, in3, in2, tmp0_m, tmp1_m); \ 1375 ST4x4_UB(tmp0_m, tmp1_m, 0, 2, 0, 2, pdst, stride); \ 1376 } while (0) 1377 1378 /* Description : average with rounding (in0 + in1 + 1) / 2. 1379 * Arguments : Inputs - in0, in1, in2, in3, 1380 * Outputs - out0, out1 1381 * Return Type - as per RTYPE 1382 * Details : Each unsigned byte element from 'in0' vector is added with 1383 * each unsigned byte element from 'in1' vector. Then the average 1384 * with rounding is calculated and written to 'out0' 1385 */ 1386 #define AVER_UB2(RTYPE, in0, in1, in2, in3, out0, out1) do { \ 1387 out0 = (RTYPE)__msa_aver_u_b((v16u8)in0, (v16u8)in1); \ 1388 out1 = (RTYPE)__msa_aver_u_b((v16u8)in2, (v16u8)in3); \ 1389 } while (0) 1390 #define AVER_UB2_UB(...) AVER_UB2(v16u8, __VA_ARGS__) 1391 1392 #endif /* WEBP_DSP_MSA_MACRO_H_ */ 1393
