Home | History | Annotate | Download | only in dsp
      1 // Copyright 2016 Google Inc. All Rights Reserved.
      2 //
      3 // Use of this source code is governed by a BSD-style license
      4 // that can be found in the COPYING file in the root of the source
      5 // tree. An additional intellectual property rights grant can be found
      6 // in the file PATENTS. All contributing project authors may
      7 // be found in the AUTHORS file in the root of the source tree.
      8 // -----------------------------------------------------------------------------
      9 //
     10 // MSA common macros
     11 //
     12 // Author(s):  Prashant Patil   (prashant.patil (at) imgtec.com)
     13 
     14 #ifndef WEBP_DSP_MSA_MACRO_H_
     15 #define WEBP_DSP_MSA_MACRO_H_
     16 
     17 #include <stdint.h>
     18 #include <msa.h>
     19 
     20 #if defined(__clang__)
     21   #define CLANG_BUILD
     22 #endif
     23 
     24 #ifdef CLANG_BUILD
     25   #define ALPHAVAL  (-1)
     26   #define ADDVI_H(a, b)  __msa_addvi_h((v8i16)a, b)
     27   #define ADDVI_W(a, b)  __msa_addvi_w((v4i32)a, b)
     28   #define SRAI_B(a, b)  __msa_srai_b((v16i8)a, b)
     29   #define SRAI_H(a, b)  __msa_srai_h((v8i16)a, b)
     30   #define SRAI_W(a, b)  __msa_srai_w((v4i32)a, b)
     31   #define SRLI_H(a, b)  __msa_srli_h((v8i16)a, b)
     32   #define SLLI_B(a, b)  __msa_slli_b((v4i32)a, b)
     33   #define ANDI_B(a, b)  __msa_andi_b((v16u8)a, b)
     34   #define ORI_B(a, b)   __msa_ori_b((v16u8)a, b)
     35 #else
     36   #define ALPHAVAL  (0xff)
     37   #define ADDVI_H(a, b)  (a + b)
     38   #define ADDVI_W(a, b)  (a + b)
     39   #define SRAI_B(a, b)  (a >> b)
     40   #define SRAI_H(a, b)  (a >> b)
     41   #define SRAI_W(a, b)  (a >> b)
     42   #define SRLI_H(a, b)  (a << b)
     43   #define SLLI_B(a, b)  (a << b)
     44   #define ANDI_B(a, b)  (a & b)
     45   #define ORI_B(a, b)   (a | b)
     46 #endif
     47 
     48 #define LD_B(RTYPE, psrc) *((RTYPE*)(psrc))
     49 #define LD_UB(...) LD_B(v16u8, __VA_ARGS__)
     50 #define LD_SB(...) LD_B(v16i8, __VA_ARGS__)
     51 
     52 #define LD_H(RTYPE, psrc) *((RTYPE*)(psrc))
     53 #define LD_UH(...) LD_H(v8u16, __VA_ARGS__)
     54 #define LD_SH(...) LD_H(v8i16, __VA_ARGS__)
     55 
     56 #define LD_W(RTYPE, psrc) *((RTYPE*)(psrc))
     57 #define LD_UW(...) LD_W(v4u32, __VA_ARGS__)
     58 #define LD_SW(...) LD_W(v4i32, __VA_ARGS__)
     59 
     60 #define ST_B(RTYPE, in, pdst) *((RTYPE*)(pdst)) = in
     61 #define ST_UB(...) ST_B(v16u8, __VA_ARGS__)
     62 #define ST_SB(...) ST_B(v16i8, __VA_ARGS__)
     63 
     64 #define ST_H(RTYPE, in, pdst) *((RTYPE*)(pdst)) = in
     65 #define ST_UH(...) ST_H(v8u16, __VA_ARGS__)
     66 #define ST_SH(...) ST_H(v8i16, __VA_ARGS__)
     67 
     68 #define ST_W(RTYPE, in, pdst) *((RTYPE*)(pdst)) = in
     69 #define ST_UW(...) ST_W(v4u32, __VA_ARGS__)
     70 #define ST_SW(...) ST_W(v4i32, __VA_ARGS__)
     71 
     72 #define MSA_LOAD_FUNC(TYPE, INSTR, FUNC_NAME)             \
     73   static inline TYPE FUNC_NAME(const void* const psrc) {  \
     74     const uint8_t* const psrc_m = (const uint8_t*)psrc;   \
     75     TYPE val_m;                                           \
     76     asm volatile (                                        \
     77       "" #INSTR " %[val_m], %[psrc_m]  \n\t"              \
     78       : [val_m] "=r" (val_m)                              \
     79       : [psrc_m] "m" (*psrc_m));                          \
     80     return val_m;                                         \
     81   }
     82 
     83 #define MSA_LOAD(psrc, FUNC_NAME)  FUNC_NAME(psrc)
     84 
     85 #define MSA_STORE_FUNC(TYPE, INSTR, FUNC_NAME)               \
     86   static inline void FUNC_NAME(TYPE val, void* const pdst) { \
     87     uint8_t* const pdst_m = (uint8_t*)pdst;                  \
     88     TYPE val_m = val;                                        \
     89     asm volatile (                                           \
     90       " " #INSTR "  %[val_m],  %[pdst_m]  \n\t"              \
     91       : [pdst_m] "=m" (*pdst_m)                              \
     92       : [val_m] "r" (val_m));                                \
     93   }
     94 
     95 #define MSA_STORE(val, pdst, FUNC_NAME)  FUNC_NAME(val, pdst)
     96 
     97 #if (__mips_isa_rev >= 6)
     98   MSA_LOAD_FUNC(uint16_t, lh, msa_lh);
     99   #define LH(psrc)  MSA_LOAD(psrc, msa_lh)
    100   MSA_LOAD_FUNC(uint32_t, lw, msa_lw);
    101   #define LW(psrc)  MSA_LOAD(psrc, msa_lw)
    102   #if (__mips == 64)
    103     MSA_LOAD_FUNC(uint64_t, ld, msa_ld);
    104     #define LD(psrc)  MSA_LOAD(psrc, msa_ld)
    105   #else  // !(__mips == 64)
    106     #define LD(psrc)  ((((uint64_t)MSA_LOAD(psrc + 4, msa_lw)) << 32) | \
    107                        MSA_LOAD(psrc, msa_lw))
    108   #endif  // (__mips == 64)
    109 
    110   MSA_STORE_FUNC(uint16_t, sh, msa_sh);
    111   #define SH(val, pdst)  MSA_STORE(val, pdst, msa_sh)
    112   MSA_STORE_FUNC(uint32_t, sw, msa_sw);
    113   #define SW(val, pdst)  MSA_STORE(val, pdst, msa_sw)
    114   MSA_STORE_FUNC(uint64_t, sd, msa_sd);
    115   #define SD(val, pdst)  MSA_STORE(val, pdst, msa_sd)
    116 #else  // !(__mips_isa_rev >= 6)
    117   MSA_LOAD_FUNC(uint16_t, ulh, msa_ulh);
    118   #define LH(psrc)  MSA_LOAD(psrc, msa_ulh)
    119   MSA_LOAD_FUNC(uint32_t, ulw, msa_ulw);
    120   #define LW(psrc)  MSA_LOAD(psrc, msa_ulw)
    121   #if (__mips == 64)
    122     MSA_LOAD_FUNC(uint64_t, uld, msa_uld);
    123     #define LD(psrc)  MSA_LOAD(psrc, msa_uld)
    124   #else  // !(__mips == 64)
    125     #define LD(psrc)  ((((uint64_t)MSA_LOAD(psrc + 4, msa_ulw)) << 32) | \
    126                         MSA_LOAD(psrc, msa_ulw))
    127   #endif  // (__mips == 64)
    128 
    129   MSA_STORE_FUNC(uint16_t, ush, msa_ush);
    130   #define SH(val, pdst)  MSA_STORE(val, pdst, msa_ush)
    131   MSA_STORE_FUNC(uint32_t, usw, msa_usw);
    132   #define SW(val, pdst)  MSA_STORE(val, pdst, msa_usw)
    133   #define SD(val, pdst) do {                                               \
    134     uint8_t* const pdst_sd_m = (uint8_t*)(pdst);                           \
    135     const uint32_t val0_m = (uint32_t)(val & 0x00000000FFFFFFFF);          \
    136     const uint32_t val1_m = (uint32_t)((val >> 32) & 0x00000000FFFFFFFF);  \
    137     SW(val0_m, pdst_sd_m);                                                 \
    138     SW(val1_m, pdst_sd_m + 4);                                             \
    139   } while (0)
    140 #endif  // (__mips_isa_rev >= 6)
    141 
    142 /* Description : Load 4 words with stride
    143  * Arguments   : Inputs  - psrc, stride
    144  *               Outputs - out0, out1, out2, out3
    145  * Details     : Load word in 'out0' from (psrc)
    146  *               Load word in 'out1' from (psrc + stride)
    147  *               Load word in 'out2' from (psrc + 2 * stride)
    148  *               Load word in 'out3' from (psrc + 3 * stride)
    149  */
    150 #define LW4(psrc, stride, out0, out1, out2, out3) do {  \
    151   const uint8_t* ptmp = (const uint8_t*)psrc;           \
    152   out0 = LW(ptmp);                                      \
    153   ptmp += stride;                                       \
    154   out1 = LW(ptmp);                                      \
    155   ptmp += stride;                                       \
    156   out2 = LW(ptmp);                                      \
    157   ptmp += stride;                                       \
    158   out3 = LW(ptmp);                                      \
    159 } while (0)
    160 
    161 /* Description : Store words with stride
    162  * Arguments   : Inputs - in0, in1, in2, in3, pdst, stride
    163  * Details     : Store word from 'in0' to (pdst)
    164  *               Store word from 'in1' to (pdst + stride)
    165  *               Store word from 'in2' to (pdst + 2 * stride)
    166  *               Store word from 'in3' to (pdst + 3 * stride)
    167  */
    168 #define SW4(in0, in1, in2, in3, pdst, stride) do {  \
    169   uint8_t* ptmp = (uint8_t*)pdst;                   \
    170   SW(in0, ptmp);                                    \
    171   ptmp += stride;                                   \
    172   SW(in1, ptmp);                                    \
    173   ptmp += stride;                                   \
    174   SW(in2, ptmp);                                    \
    175   ptmp += stride;                                   \
    176   SW(in3, ptmp);                                    \
    177 } while (0)
    178 
    179 #define SW3(in0, in1, in2, pdst, stride) do {  \
    180   uint8_t* ptmp = (uint8_t*)pdst;              \
    181   SW(in0, ptmp);                               \
    182   ptmp += stride;                              \
    183   SW(in1, ptmp);                               \
    184   ptmp += stride;                              \
    185   SW(in2, ptmp);                               \
    186 } while (0)
    187 
    188 #define SW2(in0, in1, pdst, stride) do {  \
    189   uint8_t* ptmp = (uint8_t*)pdst;         \
    190   SW(in0, ptmp);                          \
    191   ptmp += stride;                         \
    192   SW(in1, ptmp);                          \
    193 } while (0)
    194 
    195 /* Description : Store 4 double words with stride
    196  * Arguments   : Inputs - in0, in1, in2, in3, pdst, stride
    197  * Details     : Store double word from 'in0' to (pdst)
    198  *               Store double word from 'in1' to (pdst + stride)
    199  *               Store double word from 'in2' to (pdst + 2 * stride)
    200  *               Store double word from 'in3' to (pdst + 3 * stride)
    201  */
    202 #define SD4(in0, in1, in2, in3, pdst, stride) do {  \
    203   uint8_t* ptmp = (uint8_t*)pdst;                   \
    204   SD(in0, ptmp);                                    \
    205   ptmp += stride;                                   \
    206   SD(in1, ptmp);                                    \
    207   ptmp += stride;                                   \
    208   SD(in2, ptmp);                                    \
    209   ptmp += stride;                                   \
    210   SD(in3, ptmp);                                    \
    211 } while (0)
    212 
    213 /* Description : Load vectors with 16 byte elements with stride
    214  * Arguments   : Inputs  - psrc, stride
    215  *               Outputs - out0, out1
    216  *               Return Type - as per RTYPE
    217  * Details     : Load 16 byte elements in 'out0' from (psrc)
    218  *               Load 16 byte elements in 'out1' from (psrc + stride)
    219  */
    220 #define LD_B2(RTYPE, psrc, stride, out0, out1) do {  \
    221   out0 = LD_B(RTYPE, psrc);                          \
    222   out1 = LD_B(RTYPE, psrc + stride);                 \
    223 } while (0)
    224 #define LD_UB2(...) LD_B2(v16u8, __VA_ARGS__)
    225 #define LD_SB2(...) LD_B2(v16i8, __VA_ARGS__)
    226 
    227 #define LD_B3(RTYPE, psrc, stride, out0, out1, out2) do {  \
    228   LD_B2(RTYPE, psrc, stride, out0, out1);                  \
    229   out2 = LD_B(RTYPE, psrc + 2 * stride);                   \
    230 } while (0)
    231 #define LD_UB3(...) LD_B3(v16u8, __VA_ARGS__)
    232 #define LD_SB3(...) LD_B3(v16i8, __VA_ARGS__)
    233 
    234 #define LD_B4(RTYPE, psrc, stride, out0, out1, out2, out3) do {  \
    235   LD_B2(RTYPE, psrc, stride, out0, out1);                        \
    236   LD_B2(RTYPE, psrc + 2 * stride , stride, out2, out3);          \
    237 } while (0)
    238 #define LD_UB4(...) LD_B4(v16u8, __VA_ARGS__)
    239 #define LD_SB4(...) LD_B4(v16i8, __VA_ARGS__)
    240 
    241 #define LD_B8(RTYPE, psrc, stride,                                  \
    242               out0, out1, out2, out3, out4, out5, out6, out7) do {  \
    243   LD_B4(RTYPE, psrc, stride, out0, out1, out2, out3);               \
    244   LD_B4(RTYPE, psrc + 4 * stride, stride, out4, out5, out6, out7);  \
    245 } while (0)
    246 #define LD_UB8(...) LD_B8(v16u8, __VA_ARGS__)
    247 #define LD_SB8(...) LD_B8(v16i8, __VA_ARGS__)
    248 
    249 /* Description : Load vectors with 8 halfword elements with stride
    250  * Arguments   : Inputs  - psrc, stride
    251  *               Outputs - out0, out1
    252  * Details     : Load 8 halfword elements in 'out0' from (psrc)
    253  *               Load 8 halfword elements in 'out1' from (psrc + stride)
    254  */
    255 #define LD_H2(RTYPE, psrc, stride, out0, out1) do {  \
    256   out0 = LD_H(RTYPE, psrc);                          \
    257   out1 = LD_H(RTYPE, psrc + stride);                 \
    258 } while (0)
    259 #define LD_UH2(...) LD_H2(v8u16, __VA_ARGS__)
    260 #define LD_SH2(...) LD_H2(v8i16, __VA_ARGS__)
    261 
    262 /* Description : Load vectors with 4 word elements with stride
    263  * Arguments   : Inputs  - psrc, stride
    264  *               Outputs - out0, out1, out2, out3
    265  * Details     : Load 4 word elements in 'out0' from (psrc + 0 * stride)
    266  *               Load 4 word elements in 'out1' from (psrc + 1 * stride)
    267  *               Load 4 word elements in 'out2' from (psrc + 2 * stride)
    268  *               Load 4 word elements in 'out3' from (psrc + 3 * stride)
    269  */
    270 #define LD_W2(RTYPE, psrc, stride, out0, out1) do {  \
    271   out0 = LD_W(RTYPE, psrc);                          \
    272   out1 = LD_W(RTYPE, psrc + stride);                 \
    273 } while (0)
    274 #define LD_UW2(...) LD_W2(v4u32, __VA_ARGS__)
    275 #define LD_SW2(...) LD_W2(v4i32, __VA_ARGS__)
    276 
    277 #define LD_W3(RTYPE, psrc, stride, out0, out1, out2) do {  \
    278   LD_W2(RTYPE, psrc, stride, out0, out1);                  \
    279   out2 = LD_W(RTYPE, psrc + 2 * stride);                   \
    280 } while (0)
    281 #define LD_UW3(...) LD_W3(v4u32, __VA_ARGS__)
    282 #define LD_SW3(...) LD_W3(v4i32, __VA_ARGS__)
    283 
    284 #define LD_W4(RTYPE, psrc, stride, out0, out1, out2, out3) do {  \
    285   LD_W2(RTYPE, psrc, stride, out0, out1);                        \
    286   LD_W2(RTYPE, psrc + 2 * stride, stride, out2, out3);           \
    287 } while (0)
    288 #define LD_UW4(...) LD_W4(v4u32, __VA_ARGS__)
    289 #define LD_SW4(...) LD_W4(v4i32, __VA_ARGS__)
    290 
    291 /* Description : Store vectors of 16 byte elements with stride
    292  * Arguments   : Inputs - in0, in1, pdst, stride
    293  * Details     : Store 16 byte elements from 'in0' to (pdst)
    294  *               Store 16 byte elements from 'in1' to (pdst + stride)
    295  */
    296 #define ST_B2(RTYPE, in0, in1, pdst, stride) do {  \
    297   ST_B(RTYPE, in0, pdst);                          \
    298   ST_B(RTYPE, in1, pdst + stride);                 \
    299 } while (0)
    300 #define ST_UB2(...) ST_B2(v16u8, __VA_ARGS__)
    301 #define ST_SB2(...) ST_B2(v16i8, __VA_ARGS__)
    302 
    303 #define ST_B4(RTYPE, in0, in1, in2, in3, pdst, stride) do {  \
    304   ST_B2(RTYPE, in0, in1, pdst, stride);                      \
    305   ST_B2(RTYPE, in2, in3, pdst + 2 * stride, stride);         \
    306 } while (0)
    307 #define ST_UB4(...) ST_B4(v16u8, __VA_ARGS__)
    308 #define ST_SB4(...) ST_B4(v16i8, __VA_ARGS__)
    309 
    310 #define ST_B8(RTYPE, in0, in1, in2, in3, in4, in5, in6, in7,    \
    311               pdst, stride) do {                                \
    312   ST_B4(RTYPE, in0, in1, in2, in3, pdst, stride);               \
    313   ST_B4(RTYPE, in4, in5, in6, in7, pdst + 4 * stride, stride);  \
    314 } while (0)
    315 #define ST_UB8(...) ST_B8(v16u8, __VA_ARGS__)
    316 
    317 /* Description : Store vectors of 4 word elements with stride
    318  * Arguments   : Inputs - in0, in1, in2, in3, pdst, stride
    319  * Details     : Store 4 word elements from 'in0' to (pdst + 0 * stride)
    320  *               Store 4 word elements from 'in1' to (pdst + 1 * stride)
    321  *               Store 4 word elements from 'in2' to (pdst + 2 * stride)
    322  *               Store 4 word elements from 'in3' to (pdst + 3 * stride)
    323  */
    324 #define ST_W2(RTYPE, in0, in1, pdst, stride) do {  \
    325   ST_W(RTYPE, in0, pdst);                          \
    326   ST_W(RTYPE, in1, pdst + stride);                 \
    327 } while (0)
    328 #define ST_UW2(...) ST_W2(v4u32, __VA_ARGS__)
    329 #define ST_SW2(...) ST_W2(v4i32, __VA_ARGS__)
    330 
    331 #define ST_W3(RTYPE, in0, in1, in2, pdst, stride) do {  \
    332   ST_W2(RTYPE, in0, in1, pdst, stride);                 \
    333   ST_W(RTYPE, in2, pdst + 2 * stride);                  \
    334 } while (0)
    335 #define ST_UW3(...) ST_W3(v4u32, __VA_ARGS__)
    336 #define ST_SW3(...) ST_W3(v4i32, __VA_ARGS__)
    337 
    338 #define ST_W4(RTYPE, in0, in1, in2, in3, pdst, stride) do {  \
    339   ST_W2(RTYPE, in0, in1, pdst, stride);                      \
    340   ST_W2(RTYPE, in2, in3, pdst + 2 * stride, stride);         \
    341 } while (0)
    342 #define ST_UW4(...) ST_W4(v4u32, __VA_ARGS__)
    343 #define ST_SW4(...) ST_W4(v4i32, __VA_ARGS__)
    344 
    345 /* Description : Store vectors of 8 halfword elements with stride
    346  * Arguments   : Inputs - in0, in1, pdst, stride
    347  * Details     : Store 8 halfword elements from 'in0' to (pdst)
    348  *               Store 8 halfword elements from 'in1' to (pdst + stride)
    349  */
    350 #define ST_H2(RTYPE, in0, in1, pdst, stride) do {  \
    351   ST_H(RTYPE, in0, pdst);                          \
    352   ST_H(RTYPE, in1, pdst + stride);                 \
    353 } while (0)
    354 #define ST_UH2(...) ST_H2(v8u16, __VA_ARGS__)
    355 #define ST_SH2(...) ST_H2(v8i16, __VA_ARGS__)
    356 
    357 /* Description : Store 2x4 byte block to destination memory from input vector
    358  * Arguments   : Inputs - in, stidx, pdst, stride
    359  * Details     : Index 'stidx' halfword element from 'in' vector is copied to
    360  *               the GP register and stored to (pdst)
    361  *               Index 'stidx+1' halfword element from 'in' vector is copied to
    362  *               the GP register and stored to (pdst + stride)
    363  *               Index 'stidx+2' halfword element from 'in' vector is copied to
    364  *               the GP register and stored to (pdst + 2 * stride)
    365  *               Index 'stidx+3' halfword element from 'in' vector is copied to
    366  *               the GP register and stored to (pdst + 3 * stride)
    367  */
    368 #define ST2x4_UB(in, stidx, pdst, stride) do {                   \
    369   uint8_t* pblk_2x4_m = (uint8_t*)pdst;                          \
    370   const uint16_t out0_m = __msa_copy_s_h((v8i16)in, stidx);      \
    371   const uint16_t out1_m = __msa_copy_s_h((v8i16)in, stidx + 1);  \
    372   const uint16_t out2_m = __msa_copy_s_h((v8i16)in, stidx + 2);  \
    373   const uint16_t out3_m = __msa_copy_s_h((v8i16)in, stidx + 3);  \
    374   SH(out0_m, pblk_2x4_m);                                        \
    375   pblk_2x4_m += stride;                                          \
    376   SH(out1_m, pblk_2x4_m);                                        \
    377   pblk_2x4_m += stride;                                          \
    378   SH(out2_m, pblk_2x4_m);                                        \
    379   pblk_2x4_m += stride;                                          \
    380   SH(out3_m, pblk_2x4_m);                                        \
    381 } while (0)
    382 
    383 /* Description : Store 4x4 byte block to destination memory from input vector
    384  * Arguments   : Inputs - in0, in1, pdst, stride
    385  * Details     : 'Idx0' word element from input vector 'in0' is copied to the
    386  *               GP register and stored to (pdst)
    387  *               'Idx1' word element from input vector 'in0' is copied to the
    388  *               GP register and stored to (pdst + stride)
    389  *               'Idx2' word element from input vector 'in0' is copied to the
    390  *               GP register and stored to (pdst + 2 * stride)
    391  *               'Idx3' word element from input vector 'in0' is copied to the
    392  *               GP register and stored to (pdst + 3 * stride)
    393  */
    394 #define ST4x4_UB(in0, in1, idx0, idx1, idx2, idx3, pdst, stride) do {  \
    395   uint8_t* const pblk_4x4_m = (uint8_t*)pdst;                          \
    396   const uint32_t out0_m = __msa_copy_s_w((v4i32)in0, idx0);            \
    397   const uint32_t out1_m = __msa_copy_s_w((v4i32)in0, idx1);            \
    398   const uint32_t out2_m = __msa_copy_s_w((v4i32)in1, idx2);            \
    399   const uint32_t out3_m = __msa_copy_s_w((v4i32)in1, idx3);            \
    400   SW4(out0_m, out1_m, out2_m, out3_m, pblk_4x4_m, stride);             \
    401 } while (0)
    402 
    403 #define ST4x8_UB(in0, in1, pdst, stride) do {                     \
    404   uint8_t* const pblk_4x8 = (uint8_t*)pdst;                       \
    405   ST4x4_UB(in0, in0, 0, 1, 2, 3, pblk_4x8, stride);               \
    406   ST4x4_UB(in1, in1, 0, 1, 2, 3, pblk_4x8 + 4 * stride, stride);  \
    407 } while (0)
    408 
    409 /* Description : Immediate number of elements to slide
    410  * Arguments   : Inputs  - in0, in1, slide_val
    411  *               Outputs - out
    412  *               Return Type - as per RTYPE
    413  * Details     : Byte elements from 'in1' vector are slid into 'in0' by
    414  *               value specified in the 'slide_val'
    415  */
    416 #define SLDI_B(RTYPE, in0, in1, slide_val)                      \
    417         (RTYPE)__msa_sldi_b((v16i8)in0, (v16i8)in1, slide_val)  \
    418 
    419 #define SLDI_UB(...) SLDI_B(v16u8, __VA_ARGS__)
    420 #define SLDI_SB(...) SLDI_B(v16i8, __VA_ARGS__)
    421 #define SLDI_SH(...) SLDI_B(v8i16, __VA_ARGS__)
    422 
    423 /* Description : Shuffle byte vector elements as per mask vector
    424  * Arguments   : Inputs  - in0, in1, in2, in3, mask0, mask1
    425  *               Outputs - out0, out1
    426  *               Return Type - as per RTYPE
    427  * Details     : Byte elements from 'in0' & 'in1' are copied selectively to
    428  *               'out0' as per control vector 'mask0'
    429  */
    430 #define VSHF_B(RTYPE, in0, in1, mask)                              \
    431         (RTYPE)__msa_vshf_b((v16i8)mask, (v16i8)in1, (v16i8)in0)
    432 
    433 #define VSHF_UB(...) VSHF_B(v16u8, __VA_ARGS__)
    434 #define VSHF_SB(...) VSHF_B(v16i8, __VA_ARGS__)
    435 #define VSHF_UH(...) VSHF_B(v8u16, __VA_ARGS__)
    436 #define VSHF_SH(...) VSHF_B(v8i16, __VA_ARGS__)
    437 
    438 #define VSHF_B2(RTYPE, in0, in1, in2, in3, mask0, mask1, out0, out1) do {  \
    439   out0 = VSHF_B(RTYPE, in0, in1, mask0);                                   \
    440   out1 = VSHF_B(RTYPE, in2, in3, mask1);                                   \
    441 } while (0)
    442 #define VSHF_B2_UB(...) VSHF_B2(v16u8, __VA_ARGS__)
    443 #define VSHF_B2_SB(...) VSHF_B2(v16i8, __VA_ARGS__)
    444 #define VSHF_B2_UH(...) VSHF_B2(v8u16, __VA_ARGS__)
    445 #define VSHF_B2_SH(...) VSHF_B2(v8i16, __VA_ARGS__)
    446 
    447 /* Description : Shuffle halfword vector elements as per mask vector
    448  * Arguments   : Inputs  - in0, in1, in2, in3, mask0, mask1
    449  *               Outputs - out0, out1
    450  *               Return Type - as per RTYPE
    451  * Details     : halfword elements from 'in0' & 'in1' are copied selectively to
    452  *               'out0' as per control vector 'mask0'
    453  */
    454 #define VSHF_H2(RTYPE, in0, in1, in2, in3, mask0, mask1, out0, out1) do {  \
    455   out0 = (RTYPE)__msa_vshf_h((v8i16)mask0, (v8i16)in1, (v8i16)in0);        \
    456   out1 = (RTYPE)__msa_vshf_h((v8i16)mask1, (v8i16)in3, (v8i16)in2);        \
    457 } while (0)
    458 #define VSHF_H2_UH(...) VSHF_H2(v8u16, __VA_ARGS__)
    459 #define VSHF_H2_SH(...) VSHF_H2(v8i16, __VA_ARGS__)
    460 
    461 /* Description : Dot product of byte vector elements
    462  * Arguments   : Inputs  - mult0, mult1, cnst0, cnst1
    463  *               Outputs - out0, out1
    464  *               Return Type - as per RTYPE
    465  * Details     : Signed byte elements from 'mult0' are multiplied with
    466  *               signed byte elements from 'cnst0' producing a result
    467  *               twice the size of input i.e. signed halfword.
    468  *               The multiplication result of adjacent odd-even elements
    469  *               are added together and written to the 'out0' vector
    470 */
    471 #define DOTP_SB2(RTYPE, mult0, mult1, cnst0, cnst1, out0, out1) do {  \
    472   out0 = (RTYPE)__msa_dotp_s_h((v16i8)mult0, (v16i8)cnst0);           \
    473   out1 = (RTYPE)__msa_dotp_s_h((v16i8)mult1, (v16i8)cnst1);           \
    474 } while (0)
    475 #define DOTP_SB2_SH(...) DOTP_SB2(v8i16, __VA_ARGS__)
    476 
    477 /* Description : Dot product of halfword vector elements
    478  * Arguments   : Inputs  - mult0, mult1, cnst0, cnst1
    479  *               Outputs - out0, out1
    480  *               Return Type - as per RTYPE
    481  * Details     : Signed halfword elements from 'mult0' are multiplied with
    482  *               signed halfword elements from 'cnst0' producing a result
    483  *               twice the size of input i.e. signed word.
    484  *               The multiplication result of adjacent odd-even elements
    485  *               are added together and written to the 'out0' vector
    486  */
    487 #define DOTP_SH2(RTYPE, mult0, mult1, cnst0, cnst1, out0, out1) do {  \
    488   out0 = (RTYPE)__msa_dotp_s_w((v8i16)mult0, (v8i16)cnst0);           \
    489   out1 = (RTYPE)__msa_dotp_s_w((v8i16)mult1, (v8i16)cnst1);           \
    490 } while (0)
    491 #define DOTP_SH2_SW(...) DOTP_SH2(v4i32, __VA_ARGS__)
    492 
    493 /* Description : Dot product of unsigned word vector elements
    494  * Arguments   : Inputs  - mult0, mult1, cnst0, cnst1
    495  *               Outputs - out0, out1
    496  *               Return Type - as per RTYPE
    497  * Details     : Unsigned word elements from 'mult0' are multiplied with
    498  *               unsigned word elements from 'cnst0' producing a result
    499  *               twice the size of input i.e. unsigned double word.
    500  *               The multiplication result of adjacent odd-even elements
    501  *               are added together and written to the 'out0' vector
    502  */
    503 #define DOTP_UW2(RTYPE, mult0, mult1, cnst0, cnst1, out0, out1) do {  \
    504   out0 = (RTYPE)__msa_dotp_u_d((v4u32)mult0, (v4u32)cnst0);           \
    505   out1 = (RTYPE)__msa_dotp_u_d((v4u32)mult1, (v4u32)cnst1);           \
    506 } while (0)
    507 #define DOTP_UW2_UD(...) DOTP_UW2(v2u64, __VA_ARGS__)
    508 
    509 /* Description : Dot product & addition of halfword vector elements
    510  * Arguments   : Inputs  - mult0, mult1, cnst0, cnst1
    511  *               Outputs - out0, out1
    512  *               Return Type - as per RTYPE
    513  * Details     : Signed halfword elements from 'mult0' are multiplied with
    514  *               signed halfword elements from 'cnst0' producing a result
    515  *               twice the size of input i.e. signed word.
    516  *               The multiplication result of adjacent odd-even elements
    517  *               are added to the 'out0' vector
    518  */
    519 #define DPADD_SH2(RTYPE, mult0, mult1, cnst0, cnst1, out0, out1) do {      \
    520   out0 = (RTYPE)__msa_dpadd_s_w((v4i32)out0, (v8i16)mult0, (v8i16)cnst0);  \
    521   out1 = (RTYPE)__msa_dpadd_s_w((v4i32)out1, (v8i16)mult1, (v8i16)cnst1);  \
    522 } while (0)
    523 #define DPADD_SH2_SW(...) DPADD_SH2(v4i32, __VA_ARGS__)
    524 
    525 /* Description : Clips all signed halfword elements of input vector
    526  *               between 0 & 255
    527  * Arguments   : Input/output  - val
    528  *               Return Type - signed halfword
    529  */
    530 #define CLIP_SH_0_255(val) do {                   \
    531   const v8i16 max_m = __msa_ldi_h(255);           \
    532   val = __msa_maxi_s_h((v8i16)val, 0);            \
    533   val = __msa_min_s_h(max_m, (v8i16)val);         \
    534 } while (0)
    535 
    536 #define CLIP_SH2_0_255(in0, in1) do {  \
    537   CLIP_SH_0_255(in0);                  \
    538   CLIP_SH_0_255(in1);                  \
    539 } while (0)
    540 
    541 #define CLIP_SH4_0_255(in0, in1, in2, in3) do {  \
    542   CLIP_SH2_0_255(in0, in1);                      \
    543   CLIP_SH2_0_255(in2, in3);                      \
    544 } while (0)
    545 
    546 /* Description : Clips all unsigned halfword elements of input vector
    547  *               between 0 & 255
    548  * Arguments   : Input  - in
    549  *               Output - out_m
    550  *               Return Type - unsigned halfword
    551  */
    552 #define CLIP_UH_0_255(in) do {                    \
    553   const v8u16 max_m = (v8u16)__msa_ldi_h(255);    \
    554   in = __msa_maxi_u_h((v8u16) in, 0);             \
    555   in = __msa_min_u_h((v8u16) max_m, (v8u16) in);  \
    556 } while (0)
    557 
    558 #define CLIP_UH2_0_255(in0, in1) do {  \
    559   CLIP_UH_0_255(in0);                  \
    560   CLIP_UH_0_255(in1);                  \
    561 } while (0)
    562 
    563 /* Description : Clips all signed word elements of input vector
    564  *               between 0 & 255
    565  * Arguments   : Input/output  - val
    566  *               Return Type - signed word
    567  */
    568 #define CLIP_SW_0_255(val) do {                   \
    569   const v4i32 max_m = __msa_ldi_w(255);           \
    570   val = __msa_maxi_s_w((v4i32)val, 0);            \
    571   val = __msa_min_s_w(max_m, (v4i32)val);         \
    572 } while (0)
    573 
    574 #define CLIP_SW4_0_255(in0, in1, in2, in3) do {   \
    575   CLIP_SW_0_255(in0);                             \
    576   CLIP_SW_0_255(in1);                             \
    577   CLIP_SW_0_255(in2);                             \
    578   CLIP_SW_0_255(in3);                             \
    579 } while (0)
    580 
    581 /* Description : Horizontal addition of 4 signed word elements of input vector
    582  * Arguments   : Input  - in       (signed word vector)
    583  *               Output - sum_m    (i32 sum)
    584  *               Return Type - signed word (GP)
    585  * Details     : 4 signed word elements of 'in' vector are added together and
    586  *               the resulting integer sum is returned
    587  */
    588 static WEBP_INLINE int32_t func_hadd_sw_s32(v4i32 in) {
    589   const v2i64 res0_m = __msa_hadd_s_d((v4i32)in, (v4i32)in);
    590   const v2i64 res1_m = __msa_splati_d(res0_m, 1);
    591   const v2i64 out = res0_m + res1_m;
    592   int32_t sum_m = __msa_copy_s_w((v4i32)out, 0);
    593   return sum_m;
    594 }
    595 #define HADD_SW_S32(in) func_hadd_sw_s32(in)
    596 
    597 /* Description : Horizontal addition of 8 signed halfword elements
    598  * Arguments   : Input  - in       (signed halfword vector)
    599  *               Output - sum_m    (s32 sum)
    600  *               Return Type - signed word
    601  * Details     : 8 signed halfword elements of input vector are added
    602  *               together and the resulting integer sum is returned
    603  */
    604 static WEBP_INLINE int32_t func_hadd_sh_s32(v8i16 in) {
    605   const v4i32 res = __msa_hadd_s_w(in, in);
    606   const v2i64 res0 = __msa_hadd_s_d(res, res);
    607   const v2i64 res1 = __msa_splati_d(res0, 1);
    608   const v2i64 res2 = res0 + res1;
    609   const int32_t sum_m = __msa_copy_s_w((v4i32)res2, 0);
    610   return sum_m;
    611 }
    612 #define HADD_SH_S32(in) func_hadd_sh_s32(in)
    613 
    614 /* Description : Horizontal addition of 8 unsigned halfword elements
    615  * Arguments   : Input  - in       (unsigned halfword vector)
    616  *               Output - sum_m    (u32 sum)
    617  *               Return Type - unsigned word
    618  * Details     : 8 unsigned halfword elements of input vector are added
    619  *               together and the resulting integer sum is returned
    620  */
    621 static WEBP_INLINE uint32_t func_hadd_uh_u32(v8u16 in) {
    622   uint32_t sum_m;
    623   const v4u32 res_m = __msa_hadd_u_w(in, in);
    624   v2u64 res0_m = __msa_hadd_u_d(res_m, res_m);
    625   v2u64 res1_m = (v2u64)__msa_splati_d((v2i64)res0_m, 1);
    626   res0_m = res0_m + res1_m;
    627   sum_m = __msa_copy_s_w((v4i32)res0_m, 0);
    628   return sum_m;
    629 }
    630 #define HADD_UH_U32(in) func_hadd_uh_u32(in)
    631 
    632 /* Description : Horizontal addition of signed half word vector elements
    633    Arguments   : Inputs  - in0, in1
    634                  Outputs - out0, out1
    635                  Return Type - as per RTYPE
    636    Details     : Each signed odd half word element from 'in0' is added to
    637                  even signed half word element from 'in0' (pairwise) and the
    638                  halfword result is written in 'out0'
    639 */
    640 #define HADD_SH2(RTYPE, in0, in1, out0, out1) do {       \
    641   out0 = (RTYPE)__msa_hadd_s_w((v8i16)in0, (v8i16)in0);  \
    642   out1 = (RTYPE)__msa_hadd_s_w((v8i16)in1, (v8i16)in1);  \
    643 } while (0)
    644 #define HADD_SH2_SW(...) HADD_SH2(v4i32, __VA_ARGS__)
    645 
    646 #define HADD_SH4(RTYPE, in0, in1, in2, in3, out0, out1, out2, out3) do {  \
    647   HADD_SH2(RTYPE, in0, in1, out0, out1);                                  \
    648   HADD_SH2(RTYPE, in2, in3, out2, out3);                                  \
    649 } while (0)
    650 #define HADD_SH4_SW(...) HADD_SH4(v4i32, __VA_ARGS__)
    651 
    652 /* Description : Horizontal subtraction of unsigned byte vector elements
    653  * Arguments   : Inputs  - in0, in1
    654  *               Outputs - out0, out1
    655  *               Return Type - as per RTYPE
    656  * Details     : Each unsigned odd byte element from 'in0' is subtracted from
    657  *               even unsigned byte element from 'in0' (pairwise) and the
    658  *               halfword result is written to 'out0'
    659  */
    660 #define HSUB_UB2(RTYPE, in0, in1, out0, out1) do {       \
    661   out0 = (RTYPE)__msa_hsub_u_h((v16u8)in0, (v16u8)in0);  \
    662   out1 = (RTYPE)__msa_hsub_u_h((v16u8)in1, (v16u8)in1);  \
    663 } while (0)
    664 #define HSUB_UB2_UH(...) HSUB_UB2(v8u16, __VA_ARGS__)
    665 #define HSUB_UB2_SH(...) HSUB_UB2(v8i16, __VA_ARGS__)
    666 #define HSUB_UB2_SW(...) HSUB_UB2(v4i32, __VA_ARGS__)
    667 
    668 /* Description : Set element n input vector to GPR value
    669  * Arguments   : Inputs - in0, in1, in2, in3
    670  *               Output - out
    671  *               Return Type - as per RTYPE
    672  * Details     : Set element 0 in vector 'out' to value specified in 'in0'
    673  */
    674 #define INSERT_W2(RTYPE, in0, in1, out) do {        \
    675   out = (RTYPE)__msa_insert_w((v4i32)out, 0, in0);  \
    676   out = (RTYPE)__msa_insert_w((v4i32)out, 1, in1);  \
    677 } while (0)
    678 #define INSERT_W2_UB(...) INSERT_W2(v16u8, __VA_ARGS__)
    679 #define INSERT_W2_SB(...) INSERT_W2(v16i8, __VA_ARGS__)
    680 
    681 #define INSERT_W4(RTYPE, in0, in1, in2, in3, out) do {  \
    682   out = (RTYPE)__msa_insert_w((v4i32)out, 0, in0);      \
    683   out = (RTYPE)__msa_insert_w((v4i32)out, 1, in1);      \
    684   out = (RTYPE)__msa_insert_w((v4i32)out, 2, in2);      \
    685   out = (RTYPE)__msa_insert_w((v4i32)out, 3, in3);      \
    686 } while (0)
    687 #define INSERT_W4_UB(...) INSERT_W4(v16u8, __VA_ARGS__)
    688 #define INSERT_W4_SB(...) INSERT_W4(v16i8, __VA_ARGS__)
    689 #define INSERT_W4_SW(...) INSERT_W4(v4i32, __VA_ARGS__)
    690 
    691 /* Description : Set element n of double word input vector to GPR value
    692  * Arguments   : Inputs - in0, in1
    693  *               Output - out
    694  *               Return Type - as per RTYPE
    695  * Details     : Set element 0 in vector 'out' to GPR value specified in 'in0'
    696  *               Set element 1 in vector 'out' to GPR value specified in 'in1'
    697  */
    698 #define INSERT_D2(RTYPE, in0, in1, out) do {        \
    699   out = (RTYPE)__msa_insert_d((v2i64)out, 0, in0);  \
    700   out = (RTYPE)__msa_insert_d((v2i64)out, 1, in1);  \
    701 } while (0)
    702 #define INSERT_D2_UB(...) INSERT_D2(v16u8, __VA_ARGS__)
    703 #define INSERT_D2_SB(...) INSERT_D2(v16i8, __VA_ARGS__)
    704 
    705 /* Description : Interleave even byte elements from vectors
    706  * Arguments   : Inputs  - in0, in1, in2, in3
    707  *               Outputs - out0, out1
    708  *               Return Type - as per RTYPE
    709  * Details     : Even byte elements of 'in0' and 'in1' are interleaved
    710  *               and written to 'out0'
    711  */
    712 #define ILVEV_B2(RTYPE, in0, in1, in2, in3, out0, out1) do {  \
    713   out0 = (RTYPE)__msa_ilvev_b((v16i8)in1, (v16i8)in0);        \
    714   out1 = (RTYPE)__msa_ilvev_b((v16i8)in3, (v16i8)in2);        \
    715 } while (0)
    716 #define ILVEV_B2_UB(...) ILVEV_B2(v16u8, __VA_ARGS__)
    717 #define ILVEV_B2_SB(...) ILVEV_B2(v16i8, __VA_ARGS__)
    718 #define ILVEV_B2_UH(...) ILVEV_B2(v8u16, __VA_ARGS__)
    719 #define ILVEV_B2_SH(...) ILVEV_B2(v8i16, __VA_ARGS__)
    720 #define ILVEV_B2_SD(...) ILVEV_B2(v2i64, __VA_ARGS__)
    721 
    722 /* Description : Interleave odd byte elements from vectors
    723  * Arguments   : Inputs  - in0, in1, in2, in3
    724  *               Outputs - out0, out1
    725  *               Return Type - as per RTYPE
    726  * Details     : Odd byte elements of 'in0' and 'in1' are interleaved
    727  *               and written to 'out0'
    728  */
    729 #define ILVOD_B2(RTYPE, in0, in1, in2, in3, out0, out1) do {  \
    730   out0 = (RTYPE)__msa_ilvod_b((v16i8)in1, (v16i8)in0);        \
    731   out1 = (RTYPE)__msa_ilvod_b((v16i8)in3, (v16i8)in2);        \
    732 } while (0)
    733 #define ILVOD_B2_UB(...) ILVOD_B2(v16u8, __VA_ARGS__)
    734 #define ILVOD_B2_SB(...) ILVOD_B2(v16i8, __VA_ARGS__)
    735 #define ILVOD_B2_UH(...) ILVOD_B2(v8u16, __VA_ARGS__)
    736 #define ILVOD_B2_SH(...) ILVOD_B2(v8i16, __VA_ARGS__)
    737 #define ILVOD_B2_SD(...) ILVOD_B2(v2i64, __VA_ARGS__)
    738 
    739 /* Description : Interleave even halfword elements from vectors
    740  * Arguments   : Inputs  - in0, in1, in2, in3
    741  *               Outputs - out0, out1
    742  *               Return Type - as per RTYPE
    743  * Details     : Even halfword elements of 'in0' and 'in1' are interleaved
    744  *               and written to 'out0'
    745  */
    746 #define ILVEV_H2(RTYPE, in0, in1, in2, in3, out0, out1) do {  \
    747   out0 = (RTYPE)__msa_ilvev_h((v8i16)in1, (v8i16)in0);        \
    748   out1 = (RTYPE)__msa_ilvev_h((v8i16)in3, (v8i16)in2);        \
    749 } while (0)
    750 #define ILVEV_H2_UB(...) ILVEV_H2(v16u8, __VA_ARGS__)
    751 #define ILVEV_H2_UH(...) ILVEV_H2(v8u16, __VA_ARGS__)
    752 #define ILVEV_H2_SH(...) ILVEV_H2(v8i16, __VA_ARGS__)
    753 #define ILVEV_H2_SW(...) ILVEV_H2(v4i32, __VA_ARGS__)
    754 
    755 /* Description : Interleave odd halfword elements from vectors
    756  * Arguments   : Inputs  - in0, in1, in2, in3
    757  *               Outputs - out0, out1
    758  *               Return Type - as per RTYPE
    759  * Details     : Odd halfword elements of 'in0' and 'in1' are interleaved
    760  *               and written to 'out0'
    761  */
    762 #define ILVOD_H2(RTYPE, in0, in1, in2, in3, out0, out1) do {  \
    763   out0 = (RTYPE)__msa_ilvod_h((v8i16)in1, (v8i16)in0);        \
    764   out1 = (RTYPE)__msa_ilvod_h((v8i16)in3, (v8i16)in2);        \
    765 } while (0)
    766 #define ILVOD_H2_UB(...) ILVOD_H2(v16u8, __VA_ARGS__)
    767 #define ILVOD_H2_UH(...) ILVOD_H2(v8u16, __VA_ARGS__)
    768 #define ILVOD_H2_SH(...) ILVOD_H2(v8i16, __VA_ARGS__)
    769 #define ILVOD_H2_SW(...) ILVOD_H2(v4i32, __VA_ARGS__)
    770 
    771 /* Description : Interleave even word elements from vectors
    772  * Arguments   : Inputs  - in0, in1, in2, in3
    773  *               Outputs - out0, out1
    774  *               Return Type - as per RTYPE
    775  * Details     : Even word elements of 'in0' and 'in1' are interleaved
    776  *               and written to 'out0'
    777  */
    778 #define ILVEV_W2(RTYPE, in0, in1, in2, in3, out0, out1) do {  \
    779   out0 = (RTYPE)__msa_ilvev_w((v4i32)in1, (v4i32)in0);        \
    780   out1 = (RTYPE)__msa_ilvev_w((v4i32)in3, (v4i32)in2);        \
    781 } while (0)
    782 #define ILVEV_W2_UB(...) ILVEV_W2(v16u8, __VA_ARGS__)
    783 #define ILVEV_W2_SB(...) ILVEV_W2(v16i8, __VA_ARGS__)
    784 #define ILVEV_W2_UH(...) ILVEV_W2(v8u16, __VA_ARGS__)
    785 #define ILVEV_W2_SD(...) ILVEV_W2(v2i64, __VA_ARGS__)
    786 
    787 /* Description : Interleave even-odd word elements from vectors
    788  * Arguments   : Inputs  - in0, in1, in2, in3
    789  *               Outputs - out0, out1
    790  *               Return Type - as per RTYPE
    791  * Details     : Even word elements of 'in0' and 'in1' are interleaved
    792  *               and written to 'out0'
    793  *               Odd word elements of 'in2' and 'in3' are interleaved
    794  *               and written to 'out1'
    795  */
    796 #define ILVEVOD_W2(RTYPE, in0, in1, in2, in3, out0, out1) do {  \
    797   out0 = (RTYPE)__msa_ilvev_w((v4i32)in1, (v4i32)in0);          \
    798   out1 = (RTYPE)__msa_ilvod_w((v4i32)in3, (v4i32)in2);          \
    799 } while (0)
    800 #define ILVEVOD_W2_UB(...) ILVEVOD_W2(v16u8, __VA_ARGS__)
    801 #define ILVEVOD_W2_UH(...) ILVEVOD_W2(v8u16, __VA_ARGS__)
    802 #define ILVEVOD_W2_SH(...) ILVEVOD_W2(v8i16, __VA_ARGS__)
    803 #define ILVEVOD_W2_SW(...) ILVEVOD_W2(v4i32, __VA_ARGS__)
    804 
    805 /* Description : Interleave even-odd half-word elements from vectors
    806  * Arguments   : Inputs  - in0, in1, in2, in3
    807  *               Outputs - out0, out1
    808  *               Return Type - as per RTYPE
    809  * Details     : Even half-word elements of 'in0' and 'in1' are interleaved
    810  *               and written to 'out0'
    811  *               Odd half-word elements of 'in2' and 'in3' are interleaved
    812  *               and written to 'out1'
    813  */
    814 #define ILVEVOD_H2(RTYPE, in0, in1, in2, in3, out0, out1) do {  \
    815   out0 = (RTYPE)__msa_ilvev_h((v8i16)in1, (v8i16)in0);          \
    816   out1 = (RTYPE)__msa_ilvod_h((v8i16)in3, (v8i16)in2);          \
    817 } while (0)
    818 #define ILVEVOD_H2_UB(...) ILVEVOD_H2(v16u8, __VA_ARGS__)
    819 #define ILVEVOD_H2_UH(...) ILVEVOD_H2(v8u16, __VA_ARGS__)
    820 #define ILVEVOD_H2_SH(...) ILVEVOD_H2(v8i16, __VA_ARGS__)
    821 #define ILVEVOD_H2_SW(...) ILVEVOD_H2(v4i32, __VA_ARGS__)
    822 
    823 /* Description : Interleave even double word elements from vectors
    824  * Arguments   : Inputs  - in0, in1, in2, in3
    825  *               Outputs - out0, out1
    826  *               Return Type - as per RTYPE
    827  * Details     : Even double word elements of 'in0' and 'in1' are interleaved
    828  *               and written to 'out0'
    829  */
    830 #define ILVEV_D2(RTYPE, in0, in1, in2, in3, out0, out1) do {  \
    831   out0 = (RTYPE)__msa_ilvev_d((v2i64)in1, (v2i64)in0);        \
    832   out1 = (RTYPE)__msa_ilvev_d((v2i64)in3, (v2i64)in2);        \
    833 } while (0)
    834 #define ILVEV_D2_UB(...) ILVEV_D2(v16u8, __VA_ARGS__)
    835 #define ILVEV_D2_SB(...) ILVEV_D2(v16i8, __VA_ARGS__)
    836 #define ILVEV_D2_SW(...) ILVEV_D2(v4i32, __VA_ARGS__)
    837 #define ILVEV_D2_SD(...) ILVEV_D2(v2i64, __VA_ARGS__)
    838 
    839 /* Description : Interleave left half of byte elements from vectors
    840  * Arguments   : Inputs  - in0, in1, in2, in3
    841  *               Outputs - out0, out1
    842  *               Return Type - as per RTYPE
    843  * Details     : Left half of byte elements of 'in0' and 'in1' are interleaved
    844  *               and written to 'out0'.
    845  */
    846 #define ILVL_B2(RTYPE, in0, in1, in2, in3, out0, out1) do {  \
    847   out0 = (RTYPE)__msa_ilvl_b((v16i8)in0, (v16i8)in1);        \
    848   out1 = (RTYPE)__msa_ilvl_b((v16i8)in2, (v16i8)in3);        \
    849 } while (0)
    850 #define ILVL_B2_UB(...) ILVL_B2(v16u8, __VA_ARGS__)
    851 #define ILVL_B2_SB(...) ILVL_B2(v16i8, __VA_ARGS__)
    852 #define ILVL_B2_UH(...) ILVL_B2(v8u16, __VA_ARGS__)
    853 #define ILVL_B2_SH(...) ILVL_B2(v8i16, __VA_ARGS__)
    854 #define ILVL_B2_SW(...) ILVL_B2(v4i32, __VA_ARGS__)
    855 
    856 /* Description : Interleave right half of byte elements from vectors
    857  * Arguments   : Inputs  - in0, in1, in2, in3
    858  *               Outputs - out0, out1
    859  *               Return Type - as per RTYPE
    860  * Details     : Right half of byte elements of 'in0' and 'in1' are interleaved
    861  *               and written to out0.
    862  */
    863 #define ILVR_B2(RTYPE, in0, in1, in2, in3, out0, out1) do {  \
    864   out0 = (RTYPE)__msa_ilvr_b((v16i8)in0, (v16i8)in1);        \
    865   out1 = (RTYPE)__msa_ilvr_b((v16i8)in2, (v16i8)in3);        \
    866 } while (0)
    867 #define ILVR_B2_UB(...) ILVR_B2(v16u8, __VA_ARGS__)
    868 #define ILVR_B2_SB(...) ILVR_B2(v16i8, __VA_ARGS__)
    869 #define ILVR_B2_UH(...) ILVR_B2(v8u16, __VA_ARGS__)
    870 #define ILVR_B2_SH(...) ILVR_B2(v8i16, __VA_ARGS__)
    871 #define ILVR_B2_SW(...) ILVR_B2(v4i32, __VA_ARGS__)
    872 
    873 #define ILVR_B4(RTYPE, in0, in1, in2, in3, in4, in5, in6, in7,  \
    874                 out0, out1, out2, out3) do {                    \
    875   ILVR_B2(RTYPE, in0, in1, in2, in3, out0, out1);               \
    876   ILVR_B2(RTYPE, in4, in5, in6, in7, out2, out3);               \
    877 } while (0)
    878 #define ILVR_B4_UB(...) ILVR_B4(v16u8, __VA_ARGS__)
    879 #define ILVR_B4_SB(...) ILVR_B4(v16i8, __VA_ARGS__)
    880 #define ILVR_B4_UH(...) ILVR_B4(v8u16, __VA_ARGS__)
    881 #define ILVR_B4_SH(...) ILVR_B4(v8i16, __VA_ARGS__)
    882 #define ILVR_B4_SW(...) ILVR_B4(v4i32, __VA_ARGS__)
    883 
    884 /* Description : Interleave right half of halfword elements from vectors
    885  * Arguments   : Inputs  - in0, in1, in2, in3
    886  *               Outputs - out0, out1
    887  *               Return Type - as per RTYPE
    888  * Details     : Right half of halfword elements of 'in0' and 'in1' are
    889  *               interleaved and written to 'out0'.
    890  */
    891 #define ILVR_H2(RTYPE, in0, in1, in2, in3, out0, out1) do {  \
    892   out0 = (RTYPE)__msa_ilvr_h((v8i16)in0, (v8i16)in1);        \
    893   out1 = (RTYPE)__msa_ilvr_h((v8i16)in2, (v8i16)in3);        \
    894 } while (0)
    895 #define ILVR_H2_UB(...) ILVR_H2(v16u8, __VA_ARGS__)
    896 #define ILVR_H2_SH(...) ILVR_H2(v8i16, __VA_ARGS__)
    897 #define ILVR_H2_SW(...) ILVR_H2(v4i32, __VA_ARGS__)
    898 
    899 #define ILVR_H4(RTYPE, in0, in1, in2, in3, in4, in5, in6, in7,  \
    900                 out0, out1, out2, out3) do {                    \
    901   ILVR_H2(RTYPE, in0, in1, in2, in3, out0, out1);               \
    902   ILVR_H2(RTYPE, in4, in5, in6, in7, out2, out3);               \
    903 } while (0)
    904 #define ILVR_H4_UB(...) ILVR_H4(v16u8, __VA_ARGS__)
    905 #define ILVR_H4_SH(...) ILVR_H4(v8i16, __VA_ARGS__)
    906 #define ILVR_H4_SW(...) ILVR_H4(v4i32, __VA_ARGS__)
    907 
    908 /* Description : Interleave right half of double word elements from vectors
    909  * Arguments   : Inputs  - in0, in1, in2, in3
    910  *               Outputs - out0, out1
    911  *               Return Type - as per RTYPE
    912  * Details     : Right half of double word elements of 'in0' and 'in1' are
    913  *               interleaved and written to 'out0'.
    914  */
    915 #define ILVR_D2(RTYPE, in0, in1, in2, in3, out0, out1) do {  \
    916   out0 = (RTYPE)__msa_ilvr_d((v2i64)in0, (v2i64)in1);        \
    917   out1 = (RTYPE)__msa_ilvr_d((v2i64)in2, (v2i64)in3);        \
    918 } while (0)
    919 #define ILVR_D2_UB(...) ILVR_D2(v16u8, __VA_ARGS__)
    920 #define ILVR_D2_SB(...) ILVR_D2(v16i8, __VA_ARGS__)
    921 #define ILVR_D2_SH(...) ILVR_D2(v8i16, __VA_ARGS__)
    922 
    923 #define ILVR_D4(RTYPE, in0, in1, in2, in3, in4, in5, in6, in7,  \
    924                 out0, out1, out2, out3) do {                    \
    925   ILVR_D2(RTYPE, in0, in1, in2, in3, out0, out1);               \
    926   ILVR_D2(RTYPE, in4, in5, in6, in7, out2, out3);               \
    927 } while (0)
    928 #define ILVR_D4_SB(...) ILVR_D4(v16i8, __VA_ARGS__)
    929 #define ILVR_D4_UB(...) ILVR_D4(v16u8, __VA_ARGS__)
    930 
    931 /* Description : Interleave both left and right half of input vectors
    932  * Arguments   : Inputs  - in0, in1
    933  *               Outputs - out0, out1
    934  *               Return Type - as per RTYPE
    935  * Details     : Right half of byte elements from 'in0' and 'in1' are
    936  *               interleaved and written to 'out0'
    937  */
    938 #define ILVRL_B2(RTYPE, in0, in1, out0, out1) do {     \
    939   out0 = (RTYPE)__msa_ilvr_b((v16i8)in0, (v16i8)in1);  \
    940   out1 = (RTYPE)__msa_ilvl_b((v16i8)in0, (v16i8)in1);  \
    941 } while (0)
    942 #define ILVRL_B2_UB(...) ILVRL_B2(v16u8, __VA_ARGS__)
    943 #define ILVRL_B2_SB(...) ILVRL_B2(v16i8, __VA_ARGS__)
    944 #define ILVRL_B2_UH(...) ILVRL_B2(v8u16, __VA_ARGS__)
    945 #define ILVRL_B2_SH(...) ILVRL_B2(v8i16, __VA_ARGS__)
    946 #define ILVRL_B2_SW(...) ILVRL_B2(v4i32, __VA_ARGS__)
    947 
    948 #define ILVRL_H2(RTYPE, in0, in1, out0, out1) do {     \
    949   out0 = (RTYPE)__msa_ilvr_h((v8i16)in0, (v8i16)in1);  \
    950   out1 = (RTYPE)__msa_ilvl_h((v8i16)in0, (v8i16)in1);  \
    951 } while (0)
    952 #define ILVRL_H2_UB(...) ILVRL_H2(v16u8, __VA_ARGS__)
    953 #define ILVRL_H2_SB(...) ILVRL_H2(v16i8, __VA_ARGS__)
    954 #define ILVRL_H2_SH(...) ILVRL_H2(v8i16, __VA_ARGS__)
    955 #define ILVRL_H2_SW(...) ILVRL_H2(v4i32, __VA_ARGS__)
    956 #define ILVRL_H2_UW(...) ILVRL_H2(v4u32, __VA_ARGS__)
    957 
    958 #define ILVRL_W2(RTYPE, in0, in1, out0, out1) do {     \
    959   out0 = (RTYPE)__msa_ilvr_w((v4i32)in0, (v4i32)in1);  \
    960   out1 = (RTYPE)__msa_ilvl_w((v4i32)in0, (v4i32)in1);  \
    961 } while (0)
    962 #define ILVRL_W2_UB(...) ILVRL_W2(v16u8, __VA_ARGS__)
    963 #define ILVRL_W2_SH(...) ILVRL_W2(v8i16, __VA_ARGS__)
    964 #define ILVRL_W2_SW(...) ILVRL_W2(v4i32, __VA_ARGS__)
    965 #define ILVRL_W2_UW(...) ILVRL_W2(v4u32, __VA_ARGS__)
    966 
    967 /* Description : Pack even byte elements of vector pairs
    968  *  Arguments   : Inputs  - in0, in1, in2, in3
    969  *                Outputs - out0, out1
    970  *                Return Type - as per RTYPE
    971  *  Details     : Even byte elements of 'in0' are copied to the left half of
    972  *                'out0' & even byte elements of 'in1' are copied to the right
    973  *                half of 'out0'.
    974  */
    975 #define PCKEV_B2(RTYPE, in0, in1, in2, in3, out0, out1) do {  \
    976   out0 = (RTYPE)__msa_pckev_b((v16i8)in0, (v16i8)in1);        \
    977   out1 = (RTYPE)__msa_pckev_b((v16i8)in2, (v16i8)in3);        \
    978 } while (0)
    979 #define PCKEV_B2_SB(...) PCKEV_B2(v16i8, __VA_ARGS__)
    980 #define PCKEV_B2_UB(...) PCKEV_B2(v16u8, __VA_ARGS__)
    981 #define PCKEV_B2_SH(...) PCKEV_B2(v8i16, __VA_ARGS__)
    982 #define PCKEV_B2_SW(...) PCKEV_B2(v4i32, __VA_ARGS__)
    983 
    984 #define PCKEV_B4(RTYPE, in0, in1, in2, in3, in4, in5, in6, in7,  \
    985                  out0, out1, out2, out3) do {                    \
    986   PCKEV_B2(RTYPE, in0, in1, in2, in3, out0, out1);               \
    987   PCKEV_B2(RTYPE, in4, in5, in6, in7, out2, out3);               \
    988 } while (0)
    989 #define PCKEV_B4_SB(...) PCKEV_B4(v16i8, __VA_ARGS__)
    990 #define PCKEV_B4_UB(...) PCKEV_B4(v16u8, __VA_ARGS__)
    991 #define PCKEV_B4_SH(...) PCKEV_B4(v8i16, __VA_ARGS__)
    992 #define PCKEV_B4_SW(...) PCKEV_B4(v4i32, __VA_ARGS__)
    993 
    994 /* Description : Pack even halfword elements of vector pairs
    995  * Arguments   : Inputs  - in0, in1, in2, in3
    996  *               Outputs - out0, out1
    997  *               Return Type - as per RTYPE
    998  * Details     : Even halfword elements of 'in0' are copied to the left half of
    999  *               'out0' & even halfword elements of 'in1' are copied to the
   1000  *               right half of 'out0'.
   1001  */
   1002 #define PCKEV_H2(RTYPE, in0, in1, in2, in3, out0, out1) do {  \
   1003   out0 = (RTYPE)__msa_pckev_h((v8i16)in0, (v8i16)in1);        \
   1004   out1 = (RTYPE)__msa_pckev_h((v8i16)in2, (v8i16)in3);        \
   1005 } while (0)
   1006 #define PCKEV_H2_UH(...) PCKEV_H2(v8u16, __VA_ARGS__)
   1007 #define PCKEV_H2_SH(...) PCKEV_H2(v8i16, __VA_ARGS__)
   1008 #define PCKEV_H2_SW(...) PCKEV_H2(v4i32, __VA_ARGS__)
   1009 #define PCKEV_H2_UW(...) PCKEV_H2(v4u32, __VA_ARGS__)
   1010 
   1011 /* Description : Pack even word elements of vector pairs
   1012  * Arguments   : Inputs  - in0, in1, in2, in3
   1013  *               Outputs - out0, out1
   1014  *               Return Type - as per RTYPE
   1015  * Details     : Even word elements of 'in0' are copied to the left half of
   1016  *               'out0' & even word elements of 'in1' are copied to the
   1017  *               right half of 'out0'.
   1018  */
   1019 #define PCKEV_W2(RTYPE, in0, in1, in2, in3, out0, out1) do {  \
   1020   out0 = (RTYPE)__msa_pckev_w((v4i32)in0, (v4i32)in1);        \
   1021   out1 = (RTYPE)__msa_pckev_w((v4i32)in2, (v4i32)in3);        \
   1022 } while (0)
   1023 #define PCKEV_W2_UH(...) PCKEV_W2(v8u16, __VA_ARGS__)
   1024 #define PCKEV_W2_SH(...) PCKEV_W2(v8i16, __VA_ARGS__)
   1025 #define PCKEV_W2_SW(...) PCKEV_W2(v4i32, __VA_ARGS__)
   1026 #define PCKEV_W2_UW(...) PCKEV_W2(v4u32, __VA_ARGS__)
   1027 
   1028 /* Description : Pack odd halfword elements of vector pairs
   1029  * Arguments   : Inputs  - in0, in1, in2, in3
   1030  *               Outputs - out0, out1
   1031  *               Return Type - as per RTYPE
   1032  * Details     : Odd halfword elements of 'in0' are copied to the left half of
   1033  *               'out0' & odd halfword elements of 'in1' are copied to the
   1034  *               right half of 'out0'.
   1035  */
   1036 #define PCKOD_H2(RTYPE, in0, in1, in2, in3, out0, out1) do {  \
   1037   out0 = (RTYPE)__msa_pckod_h((v8i16)in0, (v8i16)in1);        \
   1038   out1 = (RTYPE)__msa_pckod_h((v8i16)in2, (v8i16)in3);        \
   1039 } while (0)
   1040 #define PCKOD_H2_UH(...) PCKOD_H2(v8u16, __VA_ARGS__)
   1041 #define PCKOD_H2_SH(...) PCKOD_H2(v8i16, __VA_ARGS__)
   1042 #define PCKOD_H2_SW(...) PCKOD_H2(v4i32, __VA_ARGS__)
   1043 #define PCKOD_H2_UW(...) PCKOD_H2(v4u32, __VA_ARGS__)
   1044 
   1045 /* Description : Arithmetic immediate shift right all elements of word vector
   1046  * Arguments   : Inputs  - in0, in1, shift
   1047  *               Outputs - in place operation
   1048  *               Return Type - as per input vector RTYPE
   1049  * Details     : Each element of vector 'in0' is right shifted by 'shift' and
   1050  *               the result is written in-place. 'shift' is a GP variable.
   1051  */
   1052 #define SRAI_W2(RTYPE, in0, in1, shift_val) do {  \
   1053   in0 = (RTYPE)SRAI_W(in0, shift_val);            \
   1054   in1 = (RTYPE)SRAI_W(in1, shift_val);            \
   1055 } while (0)
   1056 #define SRAI_W2_SW(...) SRAI_W2(v4i32, __VA_ARGS__)
   1057 #define SRAI_W2_UW(...) SRAI_W2(v4u32, __VA_ARGS__)
   1058 
   1059 #define SRAI_W4(RTYPE, in0, in1, in2, in3, shift_val) do {  \
   1060   SRAI_W2(RTYPE, in0, in1, shift_val);                      \
   1061   SRAI_W2(RTYPE, in2, in3, shift_val);                      \
   1062 } while (0)
   1063 #define SRAI_W4_SW(...) SRAI_W4(v4i32, __VA_ARGS__)
   1064 #define SRAI_W4_UW(...) SRAI_W4(v4u32, __VA_ARGS__)
   1065 
   1066 /* Description : Arithmetic shift right all elements of half-word vector
   1067  * Arguments   : Inputs  - in0, in1, shift
   1068  *               Outputs - in place operation
   1069  *               Return Type - as per input vector RTYPE
   1070  * Details     : Each element of vector 'in0' is right shifted by 'shift' and
   1071  *               the result is written in-place. 'shift' is a GP variable.
   1072  */
   1073 #define SRAI_H2(RTYPE, in0, in1, shift_val) do {  \
   1074   in0 = (RTYPE)SRAI_H(in0, shift_val);            \
   1075   in1 = (RTYPE)SRAI_H(in1, shift_val);            \
   1076 } while (0)
   1077 #define SRAI_H2_SH(...) SRAI_H2(v8i16, __VA_ARGS__)
   1078 #define SRAI_H2_UH(...) SRAI_H2(v8u16, __VA_ARGS__)
   1079 
   1080 /* Description : Arithmetic rounded shift right all elements of word vector
   1081  * Arguments   : Inputs  - in0, in1, shift
   1082  *               Outputs - in place operation
   1083  *               Return Type - as per input vector RTYPE
   1084  * Details     : Each element of vector 'in0' is right shifted by 'shift' and
   1085  *               the result is written in-place. 'shift' is a GP variable.
   1086  */
   1087 #define SRARI_W2(RTYPE, in0, in1, shift) do {     \
   1088   in0 = (RTYPE)__msa_srari_w((v4i32)in0, shift);  \
   1089   in1 = (RTYPE)__msa_srari_w((v4i32)in1, shift);  \
   1090 } while (0)
   1091 #define SRARI_W2_SW(...) SRARI_W2(v4i32, __VA_ARGS__)
   1092 
   1093 #define SRARI_W4(RTYPE, in0, in1, in2, in3, shift) do {  \
   1094   SRARI_W2(RTYPE, in0, in1, shift);                      \
   1095   SRARI_W2(RTYPE, in2, in3, shift);                      \
   1096 } while (0)
   1097 #define SRARI_W4_SH(...) SRARI_W4(v8i16, __VA_ARGS__)
   1098 #define SRARI_W4_UW(...) SRARI_W4(v4u32, __VA_ARGS__)
   1099 #define SRARI_W4_SW(...) SRARI_W4(v4i32, __VA_ARGS__)
   1100 
   1101 /* Description : Shift right arithmetic rounded double words
   1102  * Arguments   : Inputs  - in0, in1, shift
   1103  *               Outputs - in place operation
   1104  *               Return Type - as per RTYPE
   1105  * Details     : Each element of vector 'in0' is shifted right arithmetically by
   1106  *               the number of bits in the corresponding element in the vector
   1107  *               'shift'. The last discarded bit is added to shifted value for
   1108  *               rounding and the result is written in-place.
   1109  *               'shift' is a vector.
   1110  */
   1111 #define SRAR_D2(RTYPE, in0, in1, shift) do {            \
   1112   in0 = (RTYPE)__msa_srar_d((v2i64)in0, (v2i64)shift);  \
   1113   in1 = (RTYPE)__msa_srar_d((v2i64)in1, (v2i64)shift);  \
   1114 } while (0)
   1115 #define SRAR_D2_SW(...) SRAR_D2(v4i32, __VA_ARGS__)
   1116 #define SRAR_D2_SD(...) SRAR_D2(v2i64, __VA_ARGS__)
   1117 #define SRAR_D2_UD(...) SRAR_D2(v2u64, __VA_ARGS__)
   1118 
   1119 #define SRAR_D4(RTYPE, in0, in1, in2, in3, shift) do {  \
   1120   SRAR_D2(RTYPE, in0, in1, shift);                      \
   1121   SRAR_D2(RTYPE, in2, in3, shift);                      \
   1122 } while (0)
   1123 #define SRAR_D4_SD(...) SRAR_D4(v2i64, __VA_ARGS__)
   1124 #define SRAR_D4_UD(...) SRAR_D4(v2u64, __VA_ARGS__)
   1125 
   1126 /* Description : Addition of 2 pairs of half-word vectors
   1127  * Arguments   : Inputs  - in0, in1, in2, in3
   1128  *               Outputs - out0, out1
   1129  * Details     : Each element in 'in0' is added to 'in1' and result is written
   1130  *               to 'out0'.
   1131  */
   1132 #define ADDVI_H2(RTYPE, in0, in1, in2, in3, out0, out1) do {  \
   1133   out0 = (RTYPE)ADDVI_H(in0, in1);                            \
   1134   out1 = (RTYPE)ADDVI_H(in2, in3);                            \
   1135 } while (0)
   1136 #define ADDVI_H2_SH(...) ADDVI_H2(v8i16, __VA_ARGS__)
   1137 #define ADDVI_H2_UH(...) ADDVI_H2(v8u16, __VA_ARGS__)
   1138 
   1139 /* Description : Addition of 2 pairs of word vectors
   1140  * Arguments   : Inputs  - in0, in1, in2, in3
   1141  *               Outputs - out0, out1
   1142  * Details     : Each element in 'in0' is added to 'in1' and result is written
   1143  *               to 'out0'.
   1144  */
   1145 #define ADDVI_W2(RTYPE, in0, in1, in2, in3, out0, out1) do {  \
   1146   out0 = (RTYPE)ADDVI_W(in0, in1);                            \
   1147   out1 = (RTYPE)ADDVI_W(in2, in3);                            \
   1148 } while (0)
   1149 #define ADDVI_W2_SW(...) ADDVI_W2(v4i32, __VA_ARGS__)
   1150 
   1151 /* Description : Fill 2 pairs of word vectors with GP registers
   1152  * Arguments   : Inputs  - in0, in1
   1153  *               Outputs - out0, out1
   1154  * Details     : GP register in0 is replicated in each word element of out0
   1155  *               GP register in1 is replicated in each word element of out1
   1156  */
   1157 #define FILL_W2(RTYPE, in0, in1, out0, out1) do {  \
   1158   out0 = (RTYPE)__msa_fill_w(in0);                 \
   1159   out1 = (RTYPE)__msa_fill_w(in1);                 \
   1160 } while (0)
   1161 #define FILL_W2_SW(...) FILL_W2(v4i32, __VA_ARGS__)
   1162 
   1163 /* Description : Addition of 2 pairs of vectors
   1164  * Arguments   : Inputs  - in0, in1, in2, in3
   1165  *               Outputs - out0, out1
   1166  * Details     : Each element in 'in0' is added to 'in1' and result is written
   1167  *               to 'out0'.
   1168  */
   1169 #define ADD2(in0, in1, in2, in3, out0, out1) do {  \
   1170   out0 = in0 + in1;                                \
   1171   out1 = in2 + in3;                                \
   1172 } while (0)
   1173 
   1174 #define ADD4(in0, in1, in2, in3, in4, in5, in6, in7,  \
   1175              out0, out1, out2, out3) do {             \
   1176   ADD2(in0, in1, in2, in3, out0, out1);               \
   1177   ADD2(in4, in5, in6, in7, out2, out3);               \
   1178 } while (0)
   1179 
   1180 /* Description : Subtraction of 2 pairs of vectors
   1181  * Arguments   : Inputs  - in0, in1, in2, in3
   1182  *               Outputs - out0, out1
   1183  * Details     : Each element in 'in1' is subtracted from 'in0' and result is
   1184  *               written to 'out0'.
   1185  */
   1186 #define SUB2(in0, in1, in2, in3, out0, out1) do {  \
   1187   out0 = in0 - in1;                                \
   1188   out1 = in2 - in3;                                \
   1189 } while (0)
   1190 
   1191 #define SUB3(in0, in1, in2, in3, in4, in5, out0, out1, out2) do {  \
   1192   out0 = in0 - in1;                                                \
   1193   out1 = in2 - in3;                                                \
   1194   out2 = in4 - in5;                                                \
   1195 } while (0)
   1196 
   1197 #define SUB4(in0, in1, in2, in3, in4, in5, in6, in7,  \
   1198              out0, out1, out2, out3) do {             \
   1199   out0 = in0 - in1;                                   \
   1200   out1 = in2 - in3;                                   \
   1201   out2 = in4 - in5;                                   \
   1202   out3 = in6 - in7;                                   \
   1203 } while (0)
   1204 
   1205 /* Description : Addition - Subtraction of input vectors
   1206  * Arguments   : Inputs  - in0, in1
   1207  *               Outputs - out0, out1
   1208  * Details     : Each element in 'in1' is added to 'in0' and result is
   1209  *               written to 'out0'.
   1210  *               Each element in 'in1' is subtracted from 'in0' and result is
   1211  *               written to 'out1'.
   1212  */
   1213 #define ADDSUB2(in0, in1, out0, out1) do {  \
   1214   out0 = in0 + in1;                         \
   1215   out1 = in0 - in1;                         \
   1216 } while (0)
   1217 
   1218 /* Description : Multiplication of pairs of vectors
   1219  * Arguments   : Inputs  - in0, in1, in2, in3
   1220  *               Outputs - out0, out1
   1221  * Details     : Each element from 'in0' is multiplied with elements from 'in1'
   1222  *               and the result is written to 'out0'
   1223  */
   1224 #define MUL2(in0, in1, in2, in3, out0, out1) do {  \
   1225   out0 = in0 * in1;                                \
   1226   out1 = in2 * in3;                                \
   1227 } while (0)
   1228 
   1229 #define MUL4(in0, in1, in2, in3, in4, in5, in6, in7,  \
   1230              out0, out1, out2, out3) do {             \
   1231   MUL2(in0, in1, in2, in3, out0, out1);               \
   1232   MUL2(in4, in5, in6, in7, out2, out3);               \
   1233 } while (0)
   1234 
   1235 /* Description : Sign extend halfword elements from right half of the vector
   1236  * Arguments   : Input  - in    (halfword vector)
   1237  *               Output - out   (sign extended word vector)
   1238  *               Return Type - signed word
   1239  * Details     : Sign bit of halfword elements from input vector 'in' is
   1240  *               extracted and interleaved with same vector 'in0' to generate
   1241  *               4 word elements keeping sign intact
   1242  */
   1243 #define UNPCK_R_SH_SW(in, out) do {                   \
   1244   const v8i16 sign_m = __msa_clti_s_h((v8i16)in, 0);  \
   1245   out = (v4i32)__msa_ilvr_h(sign_m, (v8i16)in);       \
   1246 } while (0)
   1247 
   1248 /* Description : Sign extend halfword elements from input vector and return
   1249  *               the result in pair of vectors
   1250  * Arguments   : Input   - in            (halfword vector)
   1251  *               Outputs - out0, out1   (sign extended word vectors)
   1252  *               Return Type - signed word
   1253  * Details     : Sign bit of halfword elements from input vector 'in' is
   1254  *               extracted and interleaved right with same vector 'in0' to
   1255  *               generate 4 signed word elements in 'out0'
   1256  *               Then interleaved left with same vector 'in0' to
   1257  *               generate 4 signed word elements in 'out1'
   1258  */
   1259 #define UNPCK_SH_SW(in, out0, out1) do {              \
   1260   const v8i16 tmp_m = __msa_clti_s_h((v8i16)in, 0);   \
   1261   ILVRL_H2_SW(tmp_m, in, out0, out1);                 \
   1262 } while (0)
   1263 
   1264 /* Description : Butterfly of 4 input vectors
   1265  * Arguments   : Inputs  - in0, in1, in2, in3
   1266  *               Outputs - out0, out1, out2, out3
   1267  * Details     : Butterfly operation
   1268  */
   1269 #define BUTTERFLY_4(in0, in1, in2, in3, out0, out1, out2, out3) do {  \
   1270   out0 = in0 + in3;                                                   \
   1271   out1 = in1 + in2;                                                   \
   1272   out2 = in1 - in2;                                                   \
   1273   out3 = in0 - in3;                                                   \
   1274 } while (0)
   1275 
   1276 /* Description : Transpose 16x4 block into 4x16 with byte elements in vectors
   1277  * Arguments   : Inputs  - in0, in1, in2, in3, in4, in5, in6, in7,
   1278  *                         in8, in9, in10, in11, in12, in13, in14, in15
   1279  *               Outputs - out0, out1, out2, out3
   1280  *               Return Type - unsigned byte
   1281  */
   1282 #define TRANSPOSE16x4_UB_UB(in0, in1, in2, in3, in4, in5, in6, in7,        \
   1283                             in8, in9, in10, in11, in12, in13, in14, in15,  \
   1284                             out0, out1, out2, out3) do {                   \
   1285   v2i64 tmp0_m, tmp1_m, tmp2_m, tmp3_m, tmp4_m, tmp5_m;                    \
   1286   ILVEV_W2_SD(in0, in4, in8, in12, tmp2_m, tmp3_m);                        \
   1287   ILVEV_W2_SD(in1, in5, in9, in13, tmp0_m, tmp1_m);                        \
   1288   ILVEV_D2_UB(tmp2_m, tmp3_m, tmp0_m, tmp1_m, out1, out3);                 \
   1289   ILVEV_W2_SD(in2, in6, in10, in14, tmp4_m, tmp5_m);                       \
   1290   ILVEV_W2_SD(in3, in7, in11, in15, tmp0_m, tmp1_m);                       \
   1291   ILVEV_D2_SD(tmp4_m, tmp5_m, tmp0_m, tmp1_m, tmp2_m, tmp3_m);             \
   1292   ILVEV_B2_SD(out1, out3, tmp2_m, tmp3_m, tmp0_m, tmp1_m);                 \
   1293   ILVEVOD_H2_UB(tmp0_m, tmp1_m, tmp0_m, tmp1_m, out0, out2);               \
   1294   ILVOD_B2_SD(out1, out3, tmp2_m, tmp3_m, tmp0_m, tmp1_m);                 \
   1295   ILVEVOD_H2_UB(tmp0_m, tmp1_m, tmp0_m, tmp1_m, out1, out3);               \
   1296 } while (0)
   1297 
   1298 /* Description : Transpose 16x8 block into 8x16 with byte elements in vectors
   1299  * Arguments   : Inputs  - in0, in1, in2, in3, in4, in5, in6, in7,
   1300  *                         in8, in9, in10, in11, in12, in13, in14, in15
   1301  *               Outputs - out0, out1, out2, out3, out4, out5, out6, out7
   1302  *               Return Type - unsigned byte
   1303  */
   1304 #define TRANSPOSE16x8_UB_UB(in0, in1, in2, in3, in4, in5, in6, in7,        \
   1305                             in8, in9, in10, in11, in12, in13, in14, in15,  \
   1306                             out0, out1, out2, out3, out4, out5,            \
   1307                             out6, out7) do {                               \
   1308   v8i16 tmp0_m, tmp1_m, tmp4_m, tmp5_m, tmp6_m, tmp7_m;                    \
   1309   v4i32 tmp2_m, tmp3_m;                                                    \
   1310   ILVEV_D2_UB(in0, in8, in1, in9, out7, out6);                             \
   1311   ILVEV_D2_UB(in2, in10, in3, in11, out5, out4);                           \
   1312   ILVEV_D2_UB(in4, in12, in5, in13, out3, out2);                           \
   1313   ILVEV_D2_UB(in6, in14, in7, in15, out1, out0);                           \
   1314   ILVEV_B2_SH(out7, out6, out5, out4, tmp0_m, tmp1_m);                     \
   1315   ILVOD_B2_SH(out7, out6, out5, out4, tmp4_m, tmp5_m);                     \
   1316   ILVEV_B2_UB(out3, out2, out1, out0, out5, out7);                         \
   1317   ILVOD_B2_SH(out3, out2, out1, out0, tmp6_m, tmp7_m);                     \
   1318   ILVEV_H2_SW(tmp0_m, tmp1_m, out5, out7, tmp2_m, tmp3_m);                 \
   1319   ILVEVOD_W2_UB(tmp2_m, tmp3_m, tmp2_m, tmp3_m, out0, out4);               \
   1320   ILVOD_H2_SW(tmp0_m, tmp1_m, out5, out7, tmp2_m, tmp3_m);                 \
   1321   ILVEVOD_W2_UB(tmp2_m, tmp3_m, tmp2_m, tmp3_m, out2, out6);               \
   1322   ILVEV_H2_SW(tmp4_m, tmp5_m, tmp6_m, tmp7_m, tmp2_m, tmp3_m);             \
   1323   ILVEVOD_W2_UB(tmp2_m, tmp3_m, tmp2_m, tmp3_m, out1, out5);               \
   1324   ILVOD_H2_SW(tmp4_m, tmp5_m, tmp6_m, tmp7_m, tmp2_m, tmp3_m);             \
   1325   ILVEVOD_W2_UB(tmp2_m, tmp3_m, tmp2_m, tmp3_m, out3, out7);               \
   1326 } while (0)
   1327 
   1328 /* Description : Transpose 4x4 block with word elements in vectors
   1329  * Arguments   : Inputs  - in0, in1, in2, in3
   1330  *                Outputs - out0, out1, out2, out3
   1331  *                Return Type - as per RTYPE
   1332  */
   1333 #define TRANSPOSE4x4_W(RTYPE, in0, in1, in2, in3,                            \
   1334                        out0, out1, out2, out3) do {                          \
   1335   v4i32 s0_m, s1_m, s2_m, s3_m;                                              \
   1336   ILVRL_W2_SW(in1, in0, s0_m, s1_m);                                         \
   1337   ILVRL_W2_SW(in3, in2, s2_m, s3_m);                                         \
   1338   out0 = (RTYPE)__msa_ilvr_d((v2i64)s2_m, (v2i64)s0_m);                      \
   1339   out1 = (RTYPE)__msa_ilvl_d((v2i64)s2_m, (v2i64)s0_m);                      \
   1340   out2 = (RTYPE)__msa_ilvr_d((v2i64)s3_m, (v2i64)s1_m);                      \
   1341   out3 = (RTYPE)__msa_ilvl_d((v2i64)s3_m, (v2i64)s1_m);                      \
   1342 } while (0)
   1343 #define TRANSPOSE4x4_SW_SW(...) TRANSPOSE4x4_W(v4i32, __VA_ARGS__)
   1344 
   1345 /* Description : Add block 4x4
   1346  * Arguments   : Inputs - in0, in1, in2, in3, pdst, stride
   1347  * Details     : Least significant 4 bytes from each input vector are added to
   1348  *               the destination bytes, clipped between 0-255 and stored.
   1349  */
   1350 #define ADDBLK_ST4x4_UB(in0, in1, in2, in3, pdst, stride) do {  \
   1351   uint32_t src0_m, src1_m, src2_m, src3_m;                      \
   1352   v8i16 inp0_m, inp1_m, res0_m, res1_m;                         \
   1353   v16i8 dst0_m = { 0 };                                         \
   1354   v16i8 dst1_m = { 0 };                                         \
   1355   const v16i8 zero_m = { 0 };                                   \
   1356   ILVR_D2_SH(in1, in0, in3, in2, inp0_m, inp1_m);               \
   1357   LW4(pdst, stride, src0_m, src1_m, src2_m, src3_m);            \
   1358   INSERT_W2_SB(src0_m, src1_m, dst0_m);                         \
   1359   INSERT_W2_SB(src2_m, src3_m, dst1_m);                         \
   1360   ILVR_B2_SH(zero_m, dst0_m, zero_m, dst1_m, res0_m, res1_m);   \
   1361   ADD2(res0_m, inp0_m, res1_m, inp1_m, res0_m, res1_m);         \
   1362   CLIP_SH2_0_255(res0_m, res1_m);                               \
   1363   PCKEV_B2_SB(res0_m, res0_m, res1_m, res1_m, dst0_m, dst1_m);  \
   1364   ST4x4_UB(dst0_m, dst1_m, 0, 1, 0, 1, pdst, stride);           \
   1365 } while (0)
   1366 
   1367 /* Description : Pack even byte elements, extract 0 & 2 index words from pair
   1368  *               of results and store 4 words in destination memory as per
   1369  *               stride
   1370  * Arguments   : Inputs - in0, in1, in2, in3, pdst, stride
   1371  */
   1372 #define PCKEV_ST4x4_UB(in0, in1, in2, in3, pdst, stride) do {  \
   1373   v16i8 tmp0_m, tmp1_m;                                        \
   1374   PCKEV_B2_SB(in1, in0, in3, in2, tmp0_m, tmp1_m);             \
   1375   ST4x4_UB(tmp0_m, tmp1_m, 0, 2, 0, 2, pdst, stride);          \
   1376 } while (0)
   1377 
   1378 /* Description : average with rounding (in0 + in1 + 1) / 2.
   1379  * Arguments   : Inputs  - in0, in1, in2, in3,
   1380  *               Outputs - out0, out1
   1381  *               Return Type - as per RTYPE
   1382  * Details     : Each unsigned byte element from 'in0' vector is added with
   1383  *               each unsigned byte element from 'in1' vector. Then the average
   1384  *               with rounding is calculated and written to 'out0'
   1385  */
   1386 #define AVER_UB2(RTYPE, in0, in1, in2, in3, out0, out1) do {  \
   1387   out0 = (RTYPE)__msa_aver_u_b((v16u8)in0, (v16u8)in1);       \
   1388   out1 = (RTYPE)__msa_aver_u_b((v16u8)in2, (v16u8)in3);       \
   1389 } while (0)
   1390 #define AVER_UB2_UB(...) AVER_UB2(v16u8, __VA_ARGS__)
   1391 
   1392 #endif  /* WEBP_DSP_MSA_MACRO_H_ */
   1393