Home | History | Annotate | Download | only in mips
      1 /*
      2  *  Copyright (c) 2015 The WebM project authors. All Rights Reserved.
      3  *
      4  *  Use of this source code is governed by a BSD-style license
      5  *  that can be found in the LICENSE file in the root of the source
      6  *  tree. An additional intellectual property rights grant can be found
      7  *  in the file PATENTS.  All contributing project authors may
      8  *  be found in the AUTHORS file in the root of the source tree.
      9  */
     10 
     11 #include "vpx_dsp/mips/inv_txfm_msa.h"
     12 
     13 void vpx_idct16_1d_rows_msa(const int16_t *input, int16_t *output) {
     14   v8i16 loc0, loc1, loc2, loc3;
     15   v8i16 reg0, reg2, reg4, reg6, reg8, reg10, reg12, reg14;
     16   v8i16 reg3, reg13, reg11, reg5, reg7, reg9, reg1, reg15;
     17   v8i16 tmp5, tmp6, tmp7;
     18 
     19   LD_SH8(input, 16, reg0, reg1, reg2, reg3, reg4, reg5, reg6, reg7);
     20   input += 8;
     21   LD_SH8(input, 16, reg8, reg9, reg10, reg11, reg12, reg13, reg14, reg15);
     22 
     23   TRANSPOSE8x8_SH_SH(reg0, reg1, reg2, reg3, reg4, reg5, reg6, reg7,
     24                      reg0, reg1, reg2, reg3, reg4, reg5, reg6, reg7);
     25   TRANSPOSE8x8_SH_SH(reg8, reg9, reg10, reg11, reg12, reg13, reg14, reg15,
     26                      reg8, reg9, reg10, reg11, reg12, reg13, reg14, reg15);
     27   DOTP_CONST_PAIR(reg2, reg14, cospi_28_64, cospi_4_64, reg2, reg14);
     28   DOTP_CONST_PAIR(reg10, reg6, cospi_12_64, cospi_20_64, reg10, reg6);
     29   BUTTERFLY_4(reg2, reg14, reg6, reg10, loc0, loc1, reg14, reg2);
     30   DOTP_CONST_PAIR(reg14, reg2, cospi_16_64, cospi_16_64, loc2, loc3);
     31   DOTP_CONST_PAIR(reg0, reg8, cospi_16_64, cospi_16_64, reg0, reg8);
     32   DOTP_CONST_PAIR(reg4, reg12, cospi_24_64, cospi_8_64, reg4, reg12);
     33   BUTTERFLY_4(reg8, reg0, reg4, reg12, reg2, reg6, reg10, reg14);
     34   SUB4(reg2, loc1, reg14, loc0, reg6, loc3, reg10, loc2, reg0, reg12, reg4,
     35        reg8);
     36   ADD4(reg2, loc1, reg14, loc0, reg6, loc3, reg10, loc2, reg2, reg14, reg6,
     37        reg10);
     38 
     39   /* stage 2 */
     40   DOTP_CONST_PAIR(reg1, reg15, cospi_30_64, cospi_2_64, reg1, reg15);
     41   DOTP_CONST_PAIR(reg9, reg7, cospi_14_64, cospi_18_64, loc2, loc3);
     42 
     43   reg9 = reg1 - loc2;
     44   reg1 = reg1 + loc2;
     45   reg7 = reg15 - loc3;
     46   reg15 = reg15 + loc3;
     47 
     48   DOTP_CONST_PAIR(reg5, reg11, cospi_22_64, cospi_10_64, reg5, reg11);
     49   DOTP_CONST_PAIR(reg13, reg3, cospi_6_64, cospi_26_64, loc0, loc1);
     50   BUTTERFLY_4(loc0, loc1, reg11, reg5, reg13, reg3, reg11, reg5);
     51 
     52   loc1 = reg15 + reg3;
     53   reg3 = reg15 - reg3;
     54   loc2 = reg2 + loc1;
     55   reg15 = reg2 - loc1;
     56 
     57   loc1 = reg1 + reg13;
     58   reg13 = reg1 - reg13;
     59   loc0 = reg0 + loc1;
     60   loc1 = reg0 - loc1;
     61   tmp6 = loc0;
     62   tmp7 = loc1;
     63   reg0 = loc2;
     64 
     65   DOTP_CONST_PAIR(reg7, reg9, cospi_24_64, cospi_8_64, reg7, reg9);
     66   DOTP_CONST_PAIR((-reg5), (-reg11), cospi_8_64, cospi_24_64, reg5, reg11);
     67 
     68   loc0 = reg9 + reg5;
     69   reg5 = reg9 - reg5;
     70   reg2 = reg6 + loc0;
     71   reg1 = reg6 - loc0;
     72 
     73   loc0 = reg7 + reg11;
     74   reg11 = reg7 - reg11;
     75   loc1 = reg4 + loc0;
     76   loc2 = reg4 - loc0;
     77   tmp5 = loc1;
     78 
     79   DOTP_CONST_PAIR(reg5, reg11, cospi_16_64, cospi_16_64, reg5, reg11);
     80   BUTTERFLY_4(reg8, reg10, reg11, reg5, loc0, reg4, reg9, loc1);
     81 
     82   reg10 = loc0;
     83   reg11 = loc1;
     84 
     85   DOTP_CONST_PAIR(reg3, reg13, cospi_16_64, cospi_16_64, reg3, reg13);
     86   BUTTERFLY_4(reg12, reg14, reg13, reg3, reg8, reg6, reg7, reg5);
     87 
     88   reg13 = loc2;
     89 
     90   /* Transpose and store the output */
     91   reg12 = tmp5;
     92   reg14 = tmp6;
     93   reg3 = tmp7;
     94 
     95   /* transpose block */
     96   TRANSPOSE8x8_SH_SH(reg0, reg2, reg4, reg6, reg8, reg10, reg12, reg14,
     97                      reg0, reg2, reg4, reg6, reg8, reg10, reg12, reg14);
     98   ST_SH8(reg0, reg2, reg4, reg6, reg8, reg10, reg12, reg14, output, 16);
     99 
    100   /* transpose block */
    101   TRANSPOSE8x8_SH_SH(reg3, reg13, reg11, reg5, reg7, reg9, reg1, reg15,
    102                      reg3, reg13, reg11, reg5, reg7, reg9, reg1, reg15);
    103   ST_SH8(reg3, reg13, reg11, reg5, reg7, reg9, reg1, reg15, (output + 8), 16);
    104 }
    105 
    106 void vpx_idct16_1d_columns_addblk_msa(int16_t *input, uint8_t *dst,
    107                                       int32_t dst_stride) {
    108   v8i16 loc0, loc1, loc2, loc3;
    109   v8i16 reg0, reg2, reg4, reg6, reg8, reg10, reg12, reg14;
    110   v8i16 reg3, reg13, reg11, reg5, reg7, reg9, reg1, reg15;
    111   v8i16 tmp5, tmp6, tmp7;
    112 
    113   /* load up 8x8 */
    114   LD_SH8(input, 16, reg0, reg1, reg2, reg3, reg4, reg5, reg6, reg7);
    115   input += 8 * 16;
    116   /* load bottom 8x8 */
    117   LD_SH8(input, 16, reg8, reg9, reg10, reg11, reg12, reg13, reg14, reg15);
    118 
    119   DOTP_CONST_PAIR(reg2, reg14, cospi_28_64, cospi_4_64, reg2, reg14);
    120   DOTP_CONST_PAIR(reg10, reg6, cospi_12_64, cospi_20_64, reg10, reg6);
    121   BUTTERFLY_4(reg2, reg14, reg6, reg10, loc0, loc1, reg14, reg2);
    122   DOTP_CONST_PAIR(reg14, reg2, cospi_16_64, cospi_16_64, loc2, loc3);
    123   DOTP_CONST_PAIR(reg0, reg8, cospi_16_64, cospi_16_64, reg0, reg8);
    124   DOTP_CONST_PAIR(reg4, reg12, cospi_24_64, cospi_8_64, reg4, reg12);
    125   BUTTERFLY_4(reg8, reg0, reg4, reg12, reg2, reg6, reg10, reg14);
    126 
    127   reg0 = reg2 - loc1;
    128   reg2 = reg2 + loc1;
    129   reg12 = reg14 - loc0;
    130   reg14 = reg14 + loc0;
    131   reg4 = reg6 - loc3;
    132   reg6 = reg6 + loc3;
    133   reg8 = reg10 - loc2;
    134   reg10 = reg10 + loc2;
    135 
    136   /* stage 2 */
    137   DOTP_CONST_PAIR(reg1, reg15, cospi_30_64, cospi_2_64, reg1, reg15);
    138   DOTP_CONST_PAIR(reg9, reg7, cospi_14_64, cospi_18_64, loc2, loc3);
    139 
    140   reg9 = reg1 - loc2;
    141   reg1 = reg1 + loc2;
    142   reg7 = reg15 - loc3;
    143   reg15 = reg15 + loc3;
    144 
    145   DOTP_CONST_PAIR(reg5, reg11, cospi_22_64, cospi_10_64, reg5, reg11);
    146   DOTP_CONST_PAIR(reg13, reg3, cospi_6_64, cospi_26_64, loc0, loc1);
    147   BUTTERFLY_4(loc0, loc1, reg11, reg5, reg13, reg3, reg11, reg5);
    148 
    149   loc1 = reg15 + reg3;
    150   reg3 = reg15 - reg3;
    151   loc2 = reg2 + loc1;
    152   reg15 = reg2 - loc1;
    153 
    154   loc1 = reg1 + reg13;
    155   reg13 = reg1 - reg13;
    156   loc0 = reg0 + loc1;
    157   loc1 = reg0 - loc1;
    158   tmp6 = loc0;
    159   tmp7 = loc1;
    160   reg0 = loc2;
    161 
    162   DOTP_CONST_PAIR(reg7, reg9, cospi_24_64, cospi_8_64, reg7, reg9);
    163   DOTP_CONST_PAIR((-reg5), (-reg11), cospi_8_64, cospi_24_64, reg5, reg11);
    164 
    165   loc0 = reg9 + reg5;
    166   reg5 = reg9 - reg5;
    167   reg2 = reg6 + loc0;
    168   reg1 = reg6 - loc0;
    169 
    170   loc0 = reg7 + reg11;
    171   reg11 = reg7 - reg11;
    172   loc1 = reg4 + loc0;
    173   loc2 = reg4 - loc0;
    174   tmp5 = loc1;
    175 
    176   DOTP_CONST_PAIR(reg5, reg11, cospi_16_64, cospi_16_64, reg5, reg11);
    177   BUTTERFLY_4(reg8, reg10, reg11, reg5, loc0, reg4, reg9, loc1);
    178 
    179   reg10 = loc0;
    180   reg11 = loc1;
    181 
    182   DOTP_CONST_PAIR(reg3, reg13, cospi_16_64, cospi_16_64, reg3, reg13);
    183   BUTTERFLY_4(reg12, reg14, reg13, reg3, reg8, reg6, reg7, reg5);
    184   reg13 = loc2;
    185 
    186   /* Transpose and store the output */
    187   reg12 = tmp5;
    188   reg14 = tmp6;
    189   reg3 = tmp7;
    190 
    191   SRARI_H4_SH(reg0, reg2, reg4, reg6, 6);
    192   VP9_ADDBLK_ST8x4_UB(dst, dst_stride, reg0, reg2, reg4, reg6);
    193   dst += (4 * dst_stride);
    194   SRARI_H4_SH(reg8, reg10, reg12, reg14, 6);
    195   VP9_ADDBLK_ST8x4_UB(dst, dst_stride, reg8, reg10, reg12, reg14);
    196   dst += (4 * dst_stride);
    197   SRARI_H4_SH(reg3, reg13, reg11, reg5, 6);
    198   VP9_ADDBLK_ST8x4_UB(dst, dst_stride, reg3, reg13, reg11, reg5);
    199   dst += (4 * dst_stride);
    200   SRARI_H4_SH(reg7, reg9, reg1, reg15, 6);
    201   VP9_ADDBLK_ST8x4_UB(dst, dst_stride, reg7, reg9, reg1, reg15);
    202 }
    203 
    204 void vpx_idct16x16_256_add_msa(const int16_t *input, uint8_t *dst,
    205                                int32_t dst_stride) {
    206   int32_t i;
    207   DECLARE_ALIGNED(32, int16_t, out_arr[16 * 16]);
    208   int16_t *out = out_arr;
    209 
    210   /* transform rows */
    211   for (i = 0; i < 2; ++i) {
    212     /* process 16 * 8 block */
    213     vpx_idct16_1d_rows_msa((input + (i << 7)), (out + (i << 7)));
    214   }
    215 
    216   /* transform columns */
    217   for (i = 0; i < 2; ++i) {
    218     /* process 8 * 16 block */
    219     vpx_idct16_1d_columns_addblk_msa((out + (i << 3)), (dst + (i << 3)),
    220                                      dst_stride);
    221   }
    222 }
    223 
    224 void vpx_idct16x16_10_add_msa(const int16_t *input, uint8_t *dst,
    225                               int32_t dst_stride) {
    226   uint8_t i;
    227   DECLARE_ALIGNED(32, int16_t, out_arr[16 * 16]);
    228   int16_t *out = out_arr;
    229 
    230   /* process 16 * 8 block */
    231   vpx_idct16_1d_rows_msa(input, out);
    232 
    233   /* short case just considers top 4 rows as valid output */
    234   out += 4 * 16;
    235   for (i = 12; i--;) {
    236     __asm__ __volatile__ (
    237         "sw     $zero,   0(%[out])     \n\t"
    238         "sw     $zero,   4(%[out])     \n\t"
    239         "sw     $zero,   8(%[out])     \n\t"
    240         "sw     $zero,  12(%[out])     \n\t"
    241         "sw     $zero,  16(%[out])     \n\t"
    242         "sw     $zero,  20(%[out])     \n\t"
    243         "sw     $zero,  24(%[out])     \n\t"
    244         "sw     $zero,  28(%[out])     \n\t"
    245 
    246         :
    247         : [out] "r" (out)
    248     );
    249 
    250     out += 16;
    251   }
    252 
    253   out = out_arr;
    254 
    255   /* transform columns */
    256   for (i = 0; i < 2; ++i) {
    257     /* process 8 * 16 block */
    258     vpx_idct16_1d_columns_addblk_msa((out + (i << 3)), (dst + (i << 3)),
    259                                      dst_stride);
    260   }
    261 }
    262 
    263 void vpx_idct16x16_1_add_msa(const int16_t *input, uint8_t *dst,
    264                              int32_t dst_stride) {
    265   uint8_t i;
    266   int16_t out;
    267   v8i16 vec, res0, res1, res2, res3, res4, res5, res6, res7;
    268   v16u8 dst0, dst1, dst2, dst3, tmp0, tmp1, tmp2, tmp3;
    269 
    270   out = ROUND_POWER_OF_TWO((input[0] * cospi_16_64), DCT_CONST_BITS);
    271   out = ROUND_POWER_OF_TWO((out * cospi_16_64), DCT_CONST_BITS);
    272   out = ROUND_POWER_OF_TWO(out, 6);
    273 
    274   vec = __msa_fill_h(out);
    275 
    276   for (i = 4; i--;) {
    277     LD_UB4(dst, dst_stride, dst0, dst1, dst2, dst3);
    278     UNPCK_UB_SH(dst0, res0, res4);
    279     UNPCK_UB_SH(dst1, res1, res5);
    280     UNPCK_UB_SH(dst2, res2, res6);
    281     UNPCK_UB_SH(dst3, res3, res7);
    282     ADD4(res0, vec, res1, vec, res2, vec, res3, vec, res0, res1, res2, res3);
    283     ADD4(res4, vec, res5, vec, res6, vec, res7, vec, res4, res5, res6, res7);
    284     CLIP_SH4_0_255(res0, res1, res2, res3);
    285     CLIP_SH4_0_255(res4, res5, res6, res7);
    286     PCKEV_B4_UB(res4, res0, res5, res1, res6, res2, res7, res3,
    287                 tmp0, tmp1, tmp2, tmp3);
    288     ST_UB4(tmp0, tmp1, tmp2, tmp3, dst, dst_stride);
    289     dst += (4 * dst_stride);
    290   }
    291 }
    292 
    293 void vpx_iadst16_1d_rows_msa(const int16_t *input, int16_t *output) {
    294   v8i16 r0, r1, r2, r3, r4, r5, r6, r7, r8, r9, r10, r11, r12, r13, r14, r15;
    295   v8i16 l0, l1, l2, l3, l4, l5, l6, l7, l8, l9, l10, l11, l12, l13, l14, l15;
    296 
    297   /* load input data */
    298   LD_SH16(input, 8,
    299           l0, l8, l1, l9, l2, l10, l3, l11, l4, l12, l5, l13, l6, l14, l7, l15);
    300   TRANSPOSE8x8_SH_SH(l0, l1, l2, l3, l4, l5, l6, l7,
    301                      l0, l1, l2, l3, l4, l5, l6, l7);
    302   TRANSPOSE8x8_SH_SH(l8, l9, l10, l11, l12, l13, l14, l15,
    303                      l8, l9, l10, l11, l12, l13, l14, l15);
    304 
    305   /* ADST in horizontal */
    306   VP9_IADST8x16_1D(l0, l1, l2, l3, l4, l5, l6, l7,
    307                    l8, l9, l10, l11, l12, l13, l14, l15,
    308                    r0, r1, r2, r3, r4, r5, r6, r7,
    309                    r8, r9, r10, r11, r12, r13, r14, r15);
    310 
    311   l1 = -r8;
    312   l3 = -r4;
    313   l13 = -r13;
    314   l15 = -r1;
    315 
    316   TRANSPOSE8x8_SH_SH(r0, l1, r12, l3, r6, r14, r10, r2,
    317                      l0, l1, l2, l3, l4, l5, l6, l7);
    318   ST_SH8(l0, l1, l2, l3, l4, l5, l6, l7, output, 16);
    319   TRANSPOSE8x8_SH_SH(r3, r11, r15, r7, r5, l13, r9, l15,
    320                      l8, l9, l10, l11, l12, l13, l14, l15);
    321   ST_SH8(l8, l9, l10, l11, l12, l13, l14, l15, (output + 8), 16);
    322 }
    323 
    324 void vpx_iadst16_1d_columns_addblk_msa(int16_t *input, uint8_t *dst,
    325                                        int32_t dst_stride) {
    326   v8i16 v0, v2, v4, v6, k0, k1, k2, k3;
    327   v8i16 r0, r1, r2, r3, r4, r5, r6, r7, r8, r9, r10, r11, r12, r13, r14, r15;
    328   v8i16 out0, out1, out2, out3, out4, out5, out6, out7;
    329   v8i16 out8, out9, out10, out11, out12, out13, out14, out15;
    330   v8i16 g0, g1, g2, g3, g4, g5, g6, g7, g8, g9, g10, g11, g12, g13, g14, g15;
    331   v8i16 h0, h1, h2, h3, h4, h5, h6, h7, h8, h9, h10, h11;
    332   v8i16 res0, res1, res2, res3, res4, res5, res6, res7;
    333   v8i16 res8, res9, res10, res11, res12, res13, res14, res15;
    334   v16u8 dst0, dst1, dst2, dst3, dst4, dst5, dst6, dst7;
    335   v16u8 dst8, dst9, dst10, dst11, dst12, dst13, dst14, dst15;
    336   v16i8 zero = { 0 };
    337 
    338   r0 = LD_SH(input + 0 * 16);
    339   r3 = LD_SH(input + 3 * 16);
    340   r4 = LD_SH(input + 4 * 16);
    341   r7 = LD_SH(input + 7 * 16);
    342   r8 = LD_SH(input + 8 * 16);
    343   r11 = LD_SH(input + 11 * 16);
    344   r12 = LD_SH(input + 12 * 16);
    345   r15 = LD_SH(input + 15 * 16);
    346 
    347   /* stage 1 */
    348   k0 = VP9_SET_COSPI_PAIR(cospi_1_64, cospi_31_64);
    349   k1 = VP9_SET_COSPI_PAIR(cospi_31_64, -cospi_1_64);
    350   k2 = VP9_SET_COSPI_PAIR(cospi_17_64, cospi_15_64);
    351   k3 = VP9_SET_COSPI_PAIR(cospi_15_64, -cospi_17_64);
    352   MADD_BF(r15, r0, r7, r8, k0, k1, k2, k3, g0, g1, g2, g3);
    353   k0 = VP9_SET_COSPI_PAIR(cospi_9_64, cospi_23_64);
    354   k1 = VP9_SET_COSPI_PAIR(cospi_23_64, -cospi_9_64);
    355   k2 = VP9_SET_COSPI_PAIR(cospi_25_64, cospi_7_64);
    356   k3 = VP9_SET_COSPI_PAIR(cospi_7_64, -cospi_25_64);
    357   MADD_BF(r11, r4, r3, r12, k0, k1, k2, k3, g8, g9, g10, g11);
    358   BUTTERFLY_4(g0, g2, g10, g8, h8, h9, v2, v0);
    359   k0 = VP9_SET_COSPI_PAIR(cospi_4_64, cospi_28_64);
    360   k1 = VP9_SET_COSPI_PAIR(cospi_28_64, -cospi_4_64);
    361   k2 = VP9_SET_COSPI_PAIR(-cospi_28_64, cospi_4_64);
    362   MADD_BF(g1, g3, g9, g11, k0, k1, k2, k0, h0, h1, h2, h3);
    363 
    364   r1 = LD_SH(input + 1 * 16);
    365   r2 = LD_SH(input + 2 * 16);
    366   r5 = LD_SH(input + 5 * 16);
    367   r6 = LD_SH(input + 6 * 16);
    368   r9 = LD_SH(input + 9 * 16);
    369   r10 = LD_SH(input + 10 * 16);
    370   r13 = LD_SH(input + 13 * 16);
    371   r14 = LD_SH(input + 14 * 16);
    372 
    373   k0 = VP9_SET_COSPI_PAIR(cospi_5_64, cospi_27_64);
    374   k1 = VP9_SET_COSPI_PAIR(cospi_27_64, -cospi_5_64);
    375   k2 = VP9_SET_COSPI_PAIR(cospi_21_64, cospi_11_64);
    376   k3 = VP9_SET_COSPI_PAIR(cospi_11_64, -cospi_21_64);
    377   MADD_BF(r13, r2, r5, r10, k0, k1, k2, k3, g4, g5, g6, g7);
    378   k0 = VP9_SET_COSPI_PAIR(cospi_13_64, cospi_19_64);
    379   k1 = VP9_SET_COSPI_PAIR(cospi_19_64, -cospi_13_64);
    380   k2 = VP9_SET_COSPI_PAIR(cospi_29_64, cospi_3_64);
    381   k3 = VP9_SET_COSPI_PAIR(cospi_3_64, -cospi_29_64);
    382   MADD_BF(r9, r6, r1, r14, k0, k1, k2, k3, g12, g13, g14, g15);
    383   BUTTERFLY_4(g4, g6, g14, g12, h10, h11, v6, v4);
    384   BUTTERFLY_4(h8, h9, h11, h10, out0, out1, h11, h10);
    385   out1 = -out1;
    386   SRARI_H2_SH(out0, out1, 6);
    387   dst0 = LD_UB(dst + 0 * dst_stride);
    388   dst1 = LD_UB(dst + 15 * dst_stride);
    389   ILVR_B2_SH(zero, dst0, zero, dst1, res0, res1);
    390   ADD2(res0, out0, res1, out1, res0, res1);
    391   CLIP_SH2_0_255(res0, res1);
    392   PCKEV_B2_SH(res0, res0, res1, res1, res0, res1);
    393   ST8x1_UB(res0, dst);
    394   ST8x1_UB(res1, dst + 15 * dst_stride);
    395 
    396   k0 = VP9_SET_COSPI_PAIR(cospi_12_64, cospi_20_64);
    397   k1 = VP9_SET_COSPI_PAIR(-cospi_20_64, cospi_12_64);
    398   k2 = VP9_SET_COSPI_PAIR(cospi_20_64, -cospi_12_64);
    399   MADD_BF(g7, g5, g15, g13, k0, k1, k2, k0, h4, h5, h6, h7);
    400   BUTTERFLY_4(h0, h2, h6, h4, out8, out9, out11, out10);
    401   out8 = -out8;
    402 
    403   SRARI_H2_SH(out8, out9, 6);
    404   dst8 = LD_UB(dst + 1 * dst_stride);
    405   dst9 = LD_UB(dst + 14 * dst_stride);
    406   ILVR_B2_SH(zero, dst8, zero, dst9, res8, res9);
    407   ADD2(res8, out8, res9, out9, res8, res9);
    408   CLIP_SH2_0_255(res8, res9);
    409   PCKEV_B2_SH(res8, res8, res9, res9, res8, res9);
    410   ST8x1_UB(res8, dst + dst_stride);
    411   ST8x1_UB(res9, dst + 14 * dst_stride);
    412 
    413   k0 = VP9_SET_COSPI_PAIR(cospi_8_64, cospi_24_64);
    414   k1 = VP9_SET_COSPI_PAIR(cospi_24_64, -cospi_8_64);
    415   k2 = VP9_SET_COSPI_PAIR(-cospi_24_64, cospi_8_64);
    416   MADD_BF(v0, v2, v4, v6, k0, k1, k2, k0, out4, out6, out5, out7);
    417   out4 = -out4;
    418   SRARI_H2_SH(out4, out5, 6);
    419   dst4 = LD_UB(dst + 3 * dst_stride);
    420   dst5 = LD_UB(dst + 12 * dst_stride);
    421   ILVR_B2_SH(zero, dst4, zero, dst5, res4, res5);
    422   ADD2(res4, out4, res5, out5, res4, res5);
    423   CLIP_SH2_0_255(res4, res5);
    424   PCKEV_B2_SH(res4, res4, res5, res5, res4, res5);
    425   ST8x1_UB(res4, dst + 3 * dst_stride);
    426   ST8x1_UB(res5, dst + 12 * dst_stride);
    427 
    428   MADD_BF(h1, h3, h5, h7, k0, k1, k2, k0, out12, out14, out13, out15);
    429   out13 = -out13;
    430   SRARI_H2_SH(out12, out13, 6);
    431   dst12 = LD_UB(dst + 2 * dst_stride);
    432   dst13 = LD_UB(dst + 13 * dst_stride);
    433   ILVR_B2_SH(zero, dst12, zero, dst13, res12, res13);
    434   ADD2(res12, out12, res13, out13, res12, res13);
    435   CLIP_SH2_0_255(res12, res13);
    436   PCKEV_B2_SH(res12, res12, res13, res13, res12, res13);
    437   ST8x1_UB(res12, dst + 2 * dst_stride);
    438   ST8x1_UB(res13, dst + 13 * dst_stride);
    439 
    440   k0 = VP9_SET_COSPI_PAIR(cospi_16_64, cospi_16_64);
    441   k3 = VP9_SET_COSPI_PAIR(-cospi_16_64, cospi_16_64);
    442   MADD_SHORT(out6, out7, k0, k3, out6, out7);
    443   SRARI_H2_SH(out6, out7, 6);
    444   dst6 = LD_UB(dst + 4 * dst_stride);
    445   dst7 = LD_UB(dst + 11 * dst_stride);
    446   ILVR_B2_SH(zero, dst6, zero, dst7, res6, res7);
    447   ADD2(res6, out6, res7, out7, res6, res7);
    448   CLIP_SH2_0_255(res6, res7);
    449   PCKEV_B2_SH(res6, res6, res7, res7, res6, res7);
    450   ST8x1_UB(res6, dst + 4 * dst_stride);
    451   ST8x1_UB(res7, dst + 11 * dst_stride);
    452 
    453   MADD_SHORT(out10, out11, k0, k3, out10, out11);
    454   SRARI_H2_SH(out10, out11, 6);
    455   dst10 = LD_UB(dst + 6 * dst_stride);
    456   dst11 = LD_UB(dst + 9 * dst_stride);
    457   ILVR_B2_SH(zero, dst10, zero, dst11, res10, res11);
    458   ADD2(res10, out10, res11, out11, res10, res11);
    459   CLIP_SH2_0_255(res10, res11);
    460   PCKEV_B2_SH(res10, res10, res11, res11, res10, res11);
    461   ST8x1_UB(res10, dst + 6 * dst_stride);
    462   ST8x1_UB(res11, dst + 9 * dst_stride);
    463 
    464   k1 = VP9_SET_COSPI_PAIR(-cospi_16_64, -cospi_16_64);
    465   k2 = VP9_SET_COSPI_PAIR(cospi_16_64, -cospi_16_64);
    466   MADD_SHORT(h10, h11, k1, k2, out2, out3);
    467   SRARI_H2_SH(out2, out3, 6);
    468   dst2 = LD_UB(dst + 7 * dst_stride);
    469   dst3 = LD_UB(dst + 8 * dst_stride);
    470   ILVR_B2_SH(zero, dst2, zero, dst3, res2, res3);
    471   ADD2(res2, out2, res3, out3, res2, res3);
    472   CLIP_SH2_0_255(res2, res3);
    473   PCKEV_B2_SH(res2, res2, res3, res3, res2, res3);
    474   ST8x1_UB(res2, dst + 7 * dst_stride);
    475   ST8x1_UB(res3, dst + 8 * dst_stride);
    476 
    477   MADD_SHORT(out14, out15, k1, k2, out14, out15);
    478   SRARI_H2_SH(out14, out15, 6);
    479   dst14 = LD_UB(dst + 5 * dst_stride);
    480   dst15 = LD_UB(dst + 10 * dst_stride);
    481   ILVR_B2_SH(zero, dst14, zero, dst15, res14, res15);
    482   ADD2(res14, out14, res15, out15, res14, res15);
    483   CLIP_SH2_0_255(res14, res15);
    484   PCKEV_B2_SH(res14, res14, res15, res15, res14, res15);
    485   ST8x1_UB(res14, dst + 5 * dst_stride);
    486   ST8x1_UB(res15, dst + 10 * dst_stride);
    487 }
    488