1 // Copyright 2016 Google Inc. All Rights Reserved. 2 // 3 // Use of this source code is governed by a BSD-style license 4 // that can be found in the COPYING file in the root of the source 5 // tree. An additional intellectual property rights grant can be found 6 // in the file PATENTS. All contributing project authors may 7 // be found in the AUTHORS file in the root of the source tree. 8 // ----------------------------------------------------------------------------- 9 // 10 // MSA version of dsp functions 11 // 12 // Author(s): Prashant Patil (prashant.patil (at) imgtec.com) 13 14 15 #include "src/dsp/dsp.h" 16 17 #if defined(WEBP_USE_MSA) 18 19 #include "src/dsp/msa_macro.h" 20 21 //------------------------------------------------------------------------------ 22 // Transforms 23 24 #define IDCT_1D_W(in0, in1, in2, in3, out0, out1, out2, out3) { \ 25 v4i32 a1_m, b1_m, c1_m, d1_m; \ 26 v4i32 c_tmp1_m, c_tmp2_m, d_tmp1_m, d_tmp2_m; \ 27 const v4i32 cospi8sqrt2minus1 = __msa_fill_w(20091); \ 28 const v4i32 sinpi8sqrt2 = __msa_fill_w(35468); \ 29 \ 30 a1_m = in0 + in2; \ 31 b1_m = in0 - in2; \ 32 c_tmp1_m = (in1 * sinpi8sqrt2) >> 16; \ 33 c_tmp2_m = in3 + ((in3 * cospi8sqrt2minus1) >> 16); \ 34 c1_m = c_tmp1_m - c_tmp2_m; \ 35 d_tmp1_m = in1 + ((in1 * cospi8sqrt2minus1) >> 16); \ 36 d_tmp2_m = (in3 * sinpi8sqrt2) >> 16; \ 37 d1_m = d_tmp1_m + d_tmp2_m; \ 38 BUTTERFLY_4(a1_m, b1_m, c1_m, d1_m, out0, out1, out2, out3); \ 39 } 40 #define MULT1(a) ((((a) * 20091) >> 16) + (a)) 41 #define MULT2(a) (((a) * 35468) >> 16) 42 43 static void TransformOne(const int16_t* in, uint8_t* dst) { 44 v8i16 input0, input1; 45 v4i32 in0, in1, in2, in3, hz0, hz1, hz2, hz3, vt0, vt1, vt2, vt3; 46 v4i32 res0, res1, res2, res3; 47 const v16i8 zero = { 0 }; 48 v16i8 dest0, dest1, dest2, dest3; 49 50 LD_SH2(in, 8, input0, input1); 51 UNPCK_SH_SW(input0, in0, in1); 52 UNPCK_SH_SW(input1, in2, in3); 53 IDCT_1D_W(in0, in1, in2, in3, hz0, hz1, hz2, hz3); 54 TRANSPOSE4x4_SW_SW(hz0, hz1, hz2, hz3, hz0, hz1, hz2, hz3); 55 IDCT_1D_W(hz0, hz1, hz2, hz3, vt0, vt1, vt2, vt3); 56 SRARI_W4_SW(vt0, vt1, vt2, vt3, 3); 57 TRANSPOSE4x4_SW_SW(vt0, vt1, vt2, vt3, vt0, vt1, vt2, vt3); 58 LD_SB4(dst, BPS, dest0, dest1, dest2, dest3); 59 ILVR_B4_SW(zero, dest0, zero, dest1, zero, dest2, zero, dest3, 60 res0, res1, res2, res3); 61 ILVR_H4_SW(zero, res0, zero, res1, zero, res2, zero, res3, 62 res0, res1, res2, res3); 63 ADD4(res0, vt0, res1, vt1, res2, vt2, res3, vt3, res0, res1, res2, res3); 64 CLIP_SW4_0_255(res0, res1, res2, res3); 65 PCKEV_B2_SW(res0, res1, res2, res3, vt0, vt1); 66 res0 = (v4i32)__msa_pckev_b((v16i8)vt0, (v16i8)vt1); 67 ST4x4_UB(res0, res0, 3, 2, 1, 0, dst, BPS); 68 } 69 70 static void TransformTwo(const int16_t* in, uint8_t* dst, int do_two) { 71 TransformOne(in, dst); 72 if (do_two) { 73 TransformOne(in + 16, dst + 4); 74 } 75 } 76 77 static void TransformWHT(const int16_t* in, int16_t* out) { 78 v8i16 input0, input1; 79 const v8i16 mask0 = { 0, 1, 2, 3, 8, 9, 10, 11 }; 80 const v8i16 mask1 = { 4, 5, 6, 7, 12, 13, 14, 15 }; 81 const v8i16 mask2 = { 0, 4, 8, 12, 1, 5, 9, 13 }; 82 const v8i16 mask3 = { 3, 7, 11, 15, 2, 6, 10, 14 }; 83 v8i16 tmp0, tmp1, tmp2, tmp3; 84 v8i16 out0, out1; 85 86 LD_SH2(in, 8, input0, input1); 87 input1 = SLDI_SH(input1, input1, 8); 88 tmp0 = input0 + input1; 89 tmp1 = input0 - input1; 90 VSHF_H2_SH(tmp0, tmp1, tmp0, tmp1, mask0, mask1, tmp2, tmp3); 91 out0 = tmp2 + tmp3; 92 out1 = tmp2 - tmp3; 93 VSHF_H2_SH(out0, out1, out0, out1, mask2, mask3, input0, input1); 94 tmp0 = input0 + input1; 95 tmp1 = input0 - input1; 96 VSHF_H2_SH(tmp0, tmp1, tmp0, tmp1, mask0, mask1, tmp2, tmp3); 97 tmp0 = tmp2 + tmp3; 98 tmp1 = tmp2 - tmp3; 99 ADDVI_H2_SH(tmp0, 3, tmp1, 3, out0, out1); 100 SRAI_H2_SH(out0, out1, 3); 101 out[0] = __msa_copy_s_h(out0, 0); 102 out[16] = __msa_copy_s_h(out0, 4); 103 out[32] = __msa_copy_s_h(out1, 0); 104 out[48] = __msa_copy_s_h(out1, 4); 105 out[64] = __msa_copy_s_h(out0, 1); 106 out[80] = __msa_copy_s_h(out0, 5); 107 out[96] = __msa_copy_s_h(out1, 1); 108 out[112] = __msa_copy_s_h(out1, 5); 109 out[128] = __msa_copy_s_h(out0, 2); 110 out[144] = __msa_copy_s_h(out0, 6); 111 out[160] = __msa_copy_s_h(out1, 2); 112 out[176] = __msa_copy_s_h(out1, 6); 113 out[192] = __msa_copy_s_h(out0, 3); 114 out[208] = __msa_copy_s_h(out0, 7); 115 out[224] = __msa_copy_s_h(out1, 3); 116 out[240] = __msa_copy_s_h(out1, 7); 117 } 118 119 static void TransformDC(const int16_t* in, uint8_t* dst) { 120 const int DC = (in[0] + 4) >> 3; 121 const v8i16 tmp0 = __msa_fill_h(DC); 122 ADDBLK_ST4x4_UB(tmp0, tmp0, tmp0, tmp0, dst, BPS); 123 } 124 125 static void TransformAC3(const int16_t* in, uint8_t* dst) { 126 const int a = in[0] + 4; 127 const int c4 = MULT2(in[4]); 128 const int d4 = MULT1(in[4]); 129 const int in2 = MULT2(in[1]); 130 const int in3 = MULT1(in[1]); 131 v4i32 tmp0 = { 0 }; 132 v4i32 out0 = __msa_fill_w(a + d4); 133 v4i32 out1 = __msa_fill_w(a + c4); 134 v4i32 out2 = __msa_fill_w(a - c4); 135 v4i32 out3 = __msa_fill_w(a - d4); 136 v4i32 res0, res1, res2, res3; 137 const v4i32 zero = { 0 }; 138 v16u8 dest0, dest1, dest2, dest3; 139 140 INSERT_W4_SW(in3, in2, -in2, -in3, tmp0); 141 ADD4(out0, tmp0, out1, tmp0, out2, tmp0, out3, tmp0, 142 out0, out1, out2, out3); 143 SRAI_W4_SW(out0, out1, out2, out3, 3); 144 LD_UB4(dst, BPS, dest0, dest1, dest2, dest3); 145 ILVR_B4_SW(zero, dest0, zero, dest1, zero, dest2, zero, dest3, 146 res0, res1, res2, res3); 147 ILVR_H4_SW(zero, res0, zero, res1, zero, res2, zero, res3, 148 res0, res1, res2, res3); 149 ADD4(res0, out0, res1, out1, res2, out2, res3, out3, res0, res1, res2, res3); 150 CLIP_SW4_0_255(res0, res1, res2, res3); 151 PCKEV_B2_SW(res0, res1, res2, res3, out0, out1); 152 res0 = (v4i32)__msa_pckev_b((v16i8)out0, (v16i8)out1); 153 ST4x4_UB(res0, res0, 3, 2, 1, 0, dst, BPS); 154 } 155 156 //------------------------------------------------------------------------------ 157 // Edge filtering functions 158 159 #define FLIP_SIGN2(in0, in1, out0, out1) { \ 160 out0 = (v16i8)__msa_xori_b(in0, 0x80); \ 161 out1 = (v16i8)__msa_xori_b(in1, 0x80); \ 162 } 163 164 #define FLIP_SIGN4(in0, in1, in2, in3, out0, out1, out2, out3) { \ 165 FLIP_SIGN2(in0, in1, out0, out1); \ 166 FLIP_SIGN2(in2, in3, out2, out3); \ 167 } 168 169 #define FILT_VAL(q0_m, p0_m, mask, filt) do { \ 170 v16i8 q0_sub_p0; \ 171 q0_sub_p0 = __msa_subs_s_b(q0_m, p0_m); \ 172 filt = __msa_adds_s_b(filt, q0_sub_p0); \ 173 filt = __msa_adds_s_b(filt, q0_sub_p0); \ 174 filt = __msa_adds_s_b(filt, q0_sub_p0); \ 175 filt = filt & mask; \ 176 } while (0) 177 178 #define FILT2(q_m, p_m, q, p) do { \ 179 u_r = SRAI_H(temp1, 7); \ 180 u_r = __msa_sat_s_h(u_r, 7); \ 181 u_l = SRAI_H(temp3, 7); \ 182 u_l = __msa_sat_s_h(u_l, 7); \ 183 u = __msa_pckev_b((v16i8)u_l, (v16i8)u_r); \ 184 q_m = __msa_subs_s_b(q_m, u); \ 185 p_m = __msa_adds_s_b(p_m, u); \ 186 q = __msa_xori_b((v16u8)q_m, 0x80); \ 187 p = __msa_xori_b((v16u8)p_m, 0x80); \ 188 } while (0) 189 190 #define LPF_FILTER4_4W(p1, p0, q0, q1, mask, hev) do { \ 191 v16i8 p1_m, p0_m, q0_m, q1_m; \ 192 v16i8 filt, t1, t2; \ 193 const v16i8 cnst4b = __msa_ldi_b(4); \ 194 const v16i8 cnst3b = __msa_ldi_b(3); \ 195 \ 196 FLIP_SIGN4(p1, p0, q0, q1, p1_m, p0_m, q0_m, q1_m); \ 197 filt = __msa_subs_s_b(p1_m, q1_m); \ 198 filt = filt & hev; \ 199 FILT_VAL(q0_m, p0_m, mask, filt); \ 200 t1 = __msa_adds_s_b(filt, cnst4b); \ 201 t1 = SRAI_B(t1, 3); \ 202 t2 = __msa_adds_s_b(filt, cnst3b); \ 203 t2 = SRAI_B(t2, 3); \ 204 q0_m = __msa_subs_s_b(q0_m, t1); \ 205 q0 = __msa_xori_b((v16u8)q0_m, 0x80); \ 206 p0_m = __msa_adds_s_b(p0_m, t2); \ 207 p0 = __msa_xori_b((v16u8)p0_m, 0x80); \ 208 filt = __msa_srari_b(t1, 1); \ 209 hev = __msa_xori_b(hev, 0xff); \ 210 filt = filt & hev; \ 211 q1_m = __msa_subs_s_b(q1_m, filt); \ 212 q1 = __msa_xori_b((v16u8)q1_m, 0x80); \ 213 p1_m = __msa_adds_s_b(p1_m, filt); \ 214 p1 = __msa_xori_b((v16u8)p1_m, 0x80); \ 215 } while (0) 216 217 #define LPF_MBFILTER(p2, p1, p0, q0, q1, q2, mask, hev) do { \ 218 v16i8 p2_m, p1_m, p0_m, q2_m, q1_m, q0_m; \ 219 v16i8 u, filt, t1, t2, filt_sign; \ 220 v8i16 filt_r, filt_l, u_r, u_l; \ 221 v8i16 temp0, temp1, temp2, temp3; \ 222 const v16i8 cnst4b = __msa_ldi_b(4); \ 223 const v16i8 cnst3b = __msa_ldi_b(3); \ 224 const v8i16 cnst9h = __msa_ldi_h(9); \ 225 const v8i16 cnst63h = __msa_ldi_h(63); \ 226 \ 227 FLIP_SIGN4(p1, p0, q0, q1, p1_m, p0_m, q0_m, q1_m); \ 228 filt = __msa_subs_s_b(p1_m, q1_m); \ 229 FILT_VAL(q0_m, p0_m, mask, filt); \ 230 FLIP_SIGN2(p2, q2, p2_m, q2_m); \ 231 t2 = filt & hev; \ 232 /* filt_val &= ~hev */ \ 233 hev = __msa_xori_b(hev, 0xff); \ 234 filt = filt & hev; \ 235 t1 = __msa_adds_s_b(t2, cnst4b); \ 236 t1 = SRAI_B(t1, 3); \ 237 t2 = __msa_adds_s_b(t2, cnst3b); \ 238 t2 = SRAI_B(t2, 3); \ 239 q0_m = __msa_subs_s_b(q0_m, t1); \ 240 p0_m = __msa_adds_s_b(p0_m, t2); \ 241 filt_sign = __msa_clti_s_b(filt, 0); \ 242 ILVRL_B2_SH(filt_sign, filt, filt_r, filt_l); \ 243 /* update q2/p2 */ \ 244 temp0 = filt_r * cnst9h; \ 245 temp1 = temp0 + cnst63h; \ 246 temp2 = filt_l * cnst9h; \ 247 temp3 = temp2 + cnst63h; \ 248 FILT2(q2_m, p2_m, q2, p2); \ 249 /* update q1/p1 */ \ 250 temp1 = temp1 + temp0; \ 251 temp3 = temp3 + temp2; \ 252 FILT2(q1_m, p1_m, q1, p1); \ 253 /* update q0/p0 */ \ 254 temp1 = temp1 + temp0; \ 255 temp3 = temp3 + temp2; \ 256 FILT2(q0_m, p0_m, q0, p0); \ 257 } while (0) 258 259 #define LPF_MASK_HEV(p3_in, p2_in, p1_in, p0_in, \ 260 q0_in, q1_in, q2_in, q3_in, \ 261 limit_in, b_limit_in, thresh_in, \ 262 hev_out, mask_out) do { \ 263 v16u8 p3_asub_p2_m, p2_asub_p1_m, p1_asub_p0_m, q1_asub_q0_m; \ 264 v16u8 p1_asub_q1_m, p0_asub_q0_m, q3_asub_q2_m, q2_asub_q1_m; \ 265 v16u8 flat_out; \ 266 \ 267 /* absolute subtraction of pixel values */ \ 268 p3_asub_p2_m = __msa_asub_u_b(p3_in, p2_in); \ 269 p2_asub_p1_m = __msa_asub_u_b(p2_in, p1_in); \ 270 p1_asub_p0_m = __msa_asub_u_b(p1_in, p0_in); \ 271 q1_asub_q0_m = __msa_asub_u_b(q1_in, q0_in); \ 272 q2_asub_q1_m = __msa_asub_u_b(q2_in, q1_in); \ 273 q3_asub_q2_m = __msa_asub_u_b(q3_in, q2_in); \ 274 p0_asub_q0_m = __msa_asub_u_b(p0_in, q0_in); \ 275 p1_asub_q1_m = __msa_asub_u_b(p1_in, q1_in); \ 276 /* calculation of hev */ \ 277 flat_out = __msa_max_u_b(p1_asub_p0_m, q1_asub_q0_m); \ 278 hev_out = (thresh_in < flat_out); \ 279 /* calculation of mask */ \ 280 p0_asub_q0_m = __msa_adds_u_b(p0_asub_q0_m, p0_asub_q0_m); \ 281 p1_asub_q1_m = SRAI_B(p1_asub_q1_m, 1); \ 282 p0_asub_q0_m = __msa_adds_u_b(p0_asub_q0_m, p1_asub_q1_m); \ 283 mask_out = (b_limit_in < p0_asub_q0_m); \ 284 mask_out = __msa_max_u_b(flat_out, mask_out); \ 285 p3_asub_p2_m = __msa_max_u_b(p3_asub_p2_m, p2_asub_p1_m); \ 286 mask_out = __msa_max_u_b(p3_asub_p2_m, mask_out); \ 287 q2_asub_q1_m = __msa_max_u_b(q2_asub_q1_m, q3_asub_q2_m); \ 288 mask_out = __msa_max_u_b(q2_asub_q1_m, mask_out); \ 289 mask_out = (limit_in < mask_out); \ 290 mask_out = __msa_xori_b(mask_out, 0xff); \ 291 } while (0) 292 293 #define ST6x1_UB(in0, in0_idx, in1, in1_idx, pdst, stride) do { \ 294 const uint16_t tmp0_h = __msa_copy_s_h((v8i16)in1, in1_idx); \ 295 const uint32_t tmp0_w = __msa_copy_s_w((v4i32)in0, in0_idx); \ 296 SW(tmp0_w, pdst); \ 297 SH(tmp0_h, pdst + stride); \ 298 } while (0) 299 300 #define ST6x4_UB(in0, start_in0_idx, in1, start_in1_idx, pdst, stride) do { \ 301 uint8_t* ptmp1 = (uint8_t*)pdst; \ 302 ST6x1_UB(in0, start_in0_idx, in1, start_in1_idx, ptmp1, 4); \ 303 ptmp1 += stride; \ 304 ST6x1_UB(in0, start_in0_idx + 1, in1, start_in1_idx + 1, ptmp1, 4); \ 305 ptmp1 += stride; \ 306 ST6x1_UB(in0, start_in0_idx + 2, in1, start_in1_idx + 2, ptmp1, 4); \ 307 ptmp1 += stride; \ 308 ST6x1_UB(in0, start_in0_idx + 3, in1, start_in1_idx + 3, ptmp1, 4); \ 309 } while (0) 310 311 #define LPF_SIMPLE_FILT(p1_in, p0_in, q0_in, q1_in, mask) do { \ 312 v16i8 p1_m, p0_m, q0_m, q1_m, filt, filt1, filt2; \ 313 const v16i8 cnst4b = __msa_ldi_b(4); \ 314 const v16i8 cnst3b = __msa_ldi_b(3); \ 315 \ 316 FLIP_SIGN4(p1_in, p0_in, q0_in, q1_in, p1_m, p0_m, q0_m, q1_m); \ 317 filt = __msa_subs_s_b(p1_m, q1_m); \ 318 FILT_VAL(q0_m, p0_m, mask, filt); \ 319 filt1 = __msa_adds_s_b(filt, cnst4b); \ 320 filt1 = SRAI_B(filt1, 3); \ 321 filt2 = __msa_adds_s_b(filt, cnst3b); \ 322 filt2 = SRAI_B(filt2, 3); \ 323 q0_m = __msa_subs_s_b(q0_m, filt1); \ 324 p0_m = __msa_adds_s_b(p0_m, filt2); \ 325 q0_in = __msa_xori_b((v16u8)q0_m, 0x80); \ 326 p0_in = __msa_xori_b((v16u8)p0_m, 0x80); \ 327 } while (0) 328 329 #define LPF_SIMPLE_MASK(p1, p0, q0, q1, b_limit, mask) do { \ 330 v16u8 p1_a_sub_q1, p0_a_sub_q0; \ 331 \ 332 p0_a_sub_q0 = __msa_asub_u_b(p0, q0); \ 333 p1_a_sub_q1 = __msa_asub_u_b(p1, q1); \ 334 p1_a_sub_q1 = (v16u8)__msa_srli_b((v16i8)p1_a_sub_q1, 1); \ 335 p0_a_sub_q0 = __msa_adds_u_b(p0_a_sub_q0, p0_a_sub_q0); \ 336 mask = __msa_adds_u_b(p0_a_sub_q0, p1_a_sub_q1); \ 337 mask = (mask <= b_limit); \ 338 } while (0) 339 340 static void VFilter16(uint8_t* src, int stride, 341 int b_limit_in, int limit_in, int thresh_in) { 342 uint8_t* ptemp = src - 4 * stride; 343 v16u8 p3, p2, p1, p0, q3, q2, q1, q0; 344 v16u8 mask, hev; 345 const v16u8 thresh = (v16u8)__msa_fill_b(thresh_in); 346 const v16u8 limit = (v16u8)__msa_fill_b(limit_in); 347 const v16u8 b_limit = (v16u8)__msa_fill_b(b_limit_in); 348 349 LD_UB8(ptemp, stride, p3, p2, p1, p0, q0, q1, q2, q3); 350 LPF_MASK_HEV(p3, p2, p1, p0, q0, q1, q2, q3, limit, b_limit, thresh, 351 hev, mask); 352 LPF_MBFILTER(p2, p1, p0, q0, q1, q2, mask, hev); 353 ptemp = src - 3 * stride; 354 ST_UB4(p2, p1, p0, q0, ptemp, stride); 355 ptemp += (4 * stride); 356 ST_UB2(q1, q2, ptemp, stride); 357 } 358 359 static void HFilter16(uint8_t* src, int stride, 360 int b_limit_in, int limit_in, int thresh_in) { 361 uint8_t* ptmp = src - 4; 362 v16u8 p3, p2, p1, p0, q3, q2, q1, q0; 363 v16u8 mask, hev; 364 v16u8 row0, row1, row2, row3, row4, row5, row6, row7, row8; 365 v16u8 row9, row10, row11, row12, row13, row14, row15; 366 v8i16 tmp0, tmp1, tmp2, tmp3, tmp4, tmp5, tmp6, tmp7; 367 const v16u8 b_limit = (v16u8)__msa_fill_b(b_limit_in); 368 const v16u8 limit = (v16u8)__msa_fill_b(limit_in); 369 const v16u8 thresh = (v16u8)__msa_fill_b(thresh_in); 370 371 LD_UB8(ptmp, stride, row0, row1, row2, row3, row4, row5, row6, row7); 372 ptmp += (8 * stride); 373 LD_UB8(ptmp, stride, row8, row9, row10, row11, row12, row13, row14, row15); 374 TRANSPOSE16x8_UB_UB(row0, row1, row2, row3, row4, row5, row6, row7, 375 row8, row9, row10, row11, row12, row13, row14, row15, 376 p3, p2, p1, p0, q0, q1, q2, q3); 377 LPF_MASK_HEV(p3, p2, p1, p0, q0, q1, q2, q3, limit, b_limit, thresh, 378 hev, mask); 379 LPF_MBFILTER(p2, p1, p0, q0, q1, q2, mask, hev); 380 ILVR_B2_SH(p1, p2, q0, p0, tmp0, tmp1); 381 ILVRL_H2_SH(tmp1, tmp0, tmp3, tmp4); 382 ILVL_B2_SH(p1, p2, q0, p0, tmp0, tmp1); 383 ILVRL_H2_SH(tmp1, tmp0, tmp6, tmp7); 384 ILVRL_B2_SH(q2, q1, tmp2, tmp5); 385 ptmp = src - 3; 386 ST6x1_UB(tmp3, 0, tmp2, 0, ptmp, 4); 387 ptmp += stride; 388 ST6x1_UB(tmp3, 1, tmp2, 1, ptmp, 4); 389 ptmp += stride; 390 ST6x1_UB(tmp3, 2, tmp2, 2, ptmp, 4); 391 ptmp += stride; 392 ST6x1_UB(tmp3, 3, tmp2, 3, ptmp, 4); 393 ptmp += stride; 394 ST6x1_UB(tmp4, 0, tmp2, 4, ptmp, 4); 395 ptmp += stride; 396 ST6x1_UB(tmp4, 1, tmp2, 5, ptmp, 4); 397 ptmp += stride; 398 ST6x1_UB(tmp4, 2, tmp2, 6, ptmp, 4); 399 ptmp += stride; 400 ST6x1_UB(tmp4, 3, tmp2, 7, ptmp, 4); 401 ptmp += stride; 402 ST6x1_UB(tmp6, 0, tmp5, 0, ptmp, 4); 403 ptmp += stride; 404 ST6x1_UB(tmp6, 1, tmp5, 1, ptmp, 4); 405 ptmp += stride; 406 ST6x1_UB(tmp6, 2, tmp5, 2, ptmp, 4); 407 ptmp += stride; 408 ST6x1_UB(tmp6, 3, tmp5, 3, ptmp, 4); 409 ptmp += stride; 410 ST6x1_UB(tmp7, 0, tmp5, 4, ptmp, 4); 411 ptmp += stride; 412 ST6x1_UB(tmp7, 1, tmp5, 5, ptmp, 4); 413 ptmp += stride; 414 ST6x1_UB(tmp7, 2, tmp5, 6, ptmp, 4); 415 ptmp += stride; 416 ST6x1_UB(tmp7, 3, tmp5, 7, ptmp, 4); 417 } 418 419 // on three inner edges 420 static void VFilterHorEdge16i(uint8_t* src, int stride, 421 int b_limit, int limit, int thresh) { 422 v16u8 mask, hev; 423 v16u8 p3, p2, p1, p0, q3, q2, q1, q0; 424 const v16u8 thresh0 = (v16u8)__msa_fill_b(thresh); 425 const v16u8 b_limit0 = (v16u8)__msa_fill_b(b_limit); 426 const v16u8 limit0 = (v16u8)__msa_fill_b(limit); 427 428 LD_UB8((src - 4 * stride), stride, p3, p2, p1, p0, q0, q1, q2, q3); 429 LPF_MASK_HEV(p3, p2, p1, p0, q0, q1, q2, q3, limit0, b_limit0, thresh0, 430 hev, mask); 431 LPF_FILTER4_4W(p1, p0, q0, q1, mask, hev); 432 ST_UB4(p1, p0, q0, q1, (src - 2 * stride), stride); 433 } 434 435 static void VFilter16i(uint8_t* src_y, int stride, 436 int b_limit, int limit, int thresh) { 437 VFilterHorEdge16i(src_y + 4 * stride, stride, b_limit, limit, thresh); 438 VFilterHorEdge16i(src_y + 8 * stride, stride, b_limit, limit, thresh); 439 VFilterHorEdge16i(src_y + 12 * stride, stride, b_limit, limit, thresh); 440 } 441 442 static void HFilterVertEdge16i(uint8_t* src, int stride, 443 int b_limit, int limit, int thresh) { 444 v16u8 mask, hev; 445 v16u8 p3, p2, p1, p0, q3, q2, q1, q0; 446 v16u8 row0, row1, row2, row3, row4, row5, row6, row7; 447 v16u8 row8, row9, row10, row11, row12, row13, row14, row15; 448 v8i16 tmp0, tmp1, tmp2, tmp3, tmp4, tmp5; 449 const v16u8 thresh0 = (v16u8)__msa_fill_b(thresh); 450 const v16u8 b_limit0 = (v16u8)__msa_fill_b(b_limit); 451 const v16u8 limit0 = (v16u8)__msa_fill_b(limit); 452 453 LD_UB8(src - 4, stride, row0, row1, row2, row3, row4, row5, row6, row7); 454 LD_UB8(src - 4 + (8 * stride), stride, 455 row8, row9, row10, row11, row12, row13, row14, row15); 456 TRANSPOSE16x8_UB_UB(row0, row1, row2, row3, row4, row5, row6, row7, 457 row8, row9, row10, row11, row12, row13, row14, row15, 458 p3, p2, p1, p0, q0, q1, q2, q3); 459 LPF_MASK_HEV(p3, p2, p1, p0, q0, q1, q2, q3, limit0, b_limit0, thresh0, 460 hev, mask); 461 LPF_FILTER4_4W(p1, p0, q0, q1, mask, hev); 462 ILVR_B2_SH(p0, p1, q1, q0, tmp0, tmp1); 463 ILVRL_H2_SH(tmp1, tmp0, tmp2, tmp3); 464 ILVL_B2_SH(p0, p1, q1, q0, tmp0, tmp1); 465 ILVRL_H2_SH(tmp1, tmp0, tmp4, tmp5); 466 src -= 2; 467 ST4x8_UB(tmp2, tmp3, src, stride); 468 src += (8 * stride); 469 ST4x8_UB(tmp4, tmp5, src, stride); 470 } 471 472 static void HFilter16i(uint8_t* src_y, int stride, 473 int b_limit, int limit, int thresh) { 474 HFilterVertEdge16i(src_y + 4, stride, b_limit, limit, thresh); 475 HFilterVertEdge16i(src_y + 8, stride, b_limit, limit, thresh); 476 HFilterVertEdge16i(src_y + 12, stride, b_limit, limit, thresh); 477 } 478 479 // 8-pixels wide variants, for chroma filtering 480 static void VFilter8(uint8_t* src_u, uint8_t* src_v, int stride, 481 int b_limit_in, int limit_in, int thresh_in) { 482 uint8_t* ptmp_src_u = src_u - 4 * stride; 483 uint8_t* ptmp_src_v = src_v - 4 * stride; 484 uint64_t p2_d, p1_d, p0_d, q0_d, q1_d, q2_d; 485 v16u8 p3, p2, p1, p0, q3, q2, q1, q0, mask, hev; 486 v16u8 p3_u, p2_u, p1_u, p0_u, q3_u, q2_u, q1_u, q0_u; 487 v16u8 p3_v, p2_v, p1_v, p0_v, q3_v, q2_v, q1_v, q0_v; 488 const v16u8 b_limit = (v16u8)__msa_fill_b(b_limit_in); 489 const v16u8 limit = (v16u8)__msa_fill_b(limit_in); 490 const v16u8 thresh = (v16u8)__msa_fill_b(thresh_in); 491 492 LD_UB8(ptmp_src_u, stride, p3_u, p2_u, p1_u, p0_u, q0_u, q1_u, q2_u, q3_u); 493 LD_UB8(ptmp_src_v, stride, p3_v, p2_v, p1_v, p0_v, q0_v, q1_v, q2_v, q3_v); 494 ILVR_D4_UB(p3_v, p3_u, p2_v, p2_u, p1_v, p1_u, p0_v, p0_u, p3, p2, p1, p0); 495 ILVR_D4_UB(q0_v, q0_u, q1_v, q1_u, q2_v, q2_u, q3_v, q3_u, q0, q1, q2, q3); 496 LPF_MASK_HEV(p3, p2, p1, p0, q0, q1, q2, q3, limit, b_limit, thresh, 497 hev, mask); 498 LPF_MBFILTER(p2, p1, p0, q0, q1, q2, mask, hev); 499 p2_d = __msa_copy_s_d((v2i64)p2, 0); 500 p1_d = __msa_copy_s_d((v2i64)p1, 0); 501 p0_d = __msa_copy_s_d((v2i64)p0, 0); 502 q0_d = __msa_copy_s_d((v2i64)q0, 0); 503 q1_d = __msa_copy_s_d((v2i64)q1, 0); 504 q2_d = __msa_copy_s_d((v2i64)q2, 0); 505 ptmp_src_u += stride; 506 SD4(p2_d, p1_d, p0_d, q0_d, ptmp_src_u, stride); 507 ptmp_src_u += (4 * stride); 508 SD(q1_d, ptmp_src_u); 509 ptmp_src_u += stride; 510 SD(q2_d, ptmp_src_u); 511 p2_d = __msa_copy_s_d((v2i64)p2, 1); 512 p1_d = __msa_copy_s_d((v2i64)p1, 1); 513 p0_d = __msa_copy_s_d((v2i64)p0, 1); 514 q0_d = __msa_copy_s_d((v2i64)q0, 1); 515 q1_d = __msa_copy_s_d((v2i64)q1, 1); 516 q2_d = __msa_copy_s_d((v2i64)q2, 1); 517 ptmp_src_v += stride; 518 SD4(p2_d, p1_d, p0_d, q0_d, ptmp_src_v, stride); 519 ptmp_src_v += (4 * stride); 520 SD(q1_d, ptmp_src_v); 521 ptmp_src_v += stride; 522 SD(q2_d, ptmp_src_v); 523 } 524 525 static void HFilter8(uint8_t* src_u, uint8_t* src_v, int stride, 526 int b_limit_in, int limit_in, int thresh_in) { 527 uint8_t* ptmp_src_u = src_u - 4; 528 uint8_t* ptmp_src_v = src_v - 4; 529 v16u8 p3, p2, p1, p0, q3, q2, q1, q0, mask, hev; 530 v16u8 row0, row1, row2, row3, row4, row5, row6, row7, row8; 531 v16u8 row9, row10, row11, row12, row13, row14, row15; 532 v8i16 tmp0, tmp1, tmp2, tmp3, tmp4, tmp5, tmp6, tmp7; 533 const v16u8 b_limit = (v16u8)__msa_fill_b(b_limit_in); 534 const v16u8 limit = (v16u8)__msa_fill_b(limit_in); 535 const v16u8 thresh = (v16u8)__msa_fill_b(thresh_in); 536 537 LD_UB8(ptmp_src_u, stride, row0, row1, row2, row3, row4, row5, row6, row7); 538 LD_UB8(ptmp_src_v, stride, 539 row8, row9, row10, row11, row12, row13, row14, row15); 540 TRANSPOSE16x8_UB_UB(row0, row1, row2, row3, row4, row5, row6, row7, 541 row8, row9, row10, row11, row12, row13, row14, row15, 542 p3, p2, p1, p0, q0, q1, q2, q3); 543 LPF_MASK_HEV(p3, p2, p1, p0, q0, q1, q2, q3, limit, b_limit, thresh, 544 hev, mask); 545 LPF_MBFILTER(p2, p1, p0, q0, q1, q2, mask, hev); 546 ILVR_B2_SH(p1, p2, q0, p0, tmp0, tmp1); 547 ILVRL_H2_SH(tmp1, tmp0, tmp3, tmp4); 548 ILVL_B2_SH(p1, p2, q0, p0, tmp0, tmp1); 549 ILVRL_H2_SH(tmp1, tmp0, tmp6, tmp7); 550 ILVRL_B2_SH(q2, q1, tmp2, tmp5); 551 ptmp_src_u += 1; 552 ST6x4_UB(tmp3, 0, tmp2, 0, ptmp_src_u, stride); 553 ptmp_src_u += 4 * stride; 554 ST6x4_UB(tmp4, 0, tmp2, 4, ptmp_src_u, stride); 555 ptmp_src_v += 1; 556 ST6x4_UB(tmp6, 0, tmp5, 0, ptmp_src_v, stride); 557 ptmp_src_v += 4 * stride; 558 ST6x4_UB(tmp7, 0, tmp5, 4, ptmp_src_v, stride); 559 } 560 561 static void VFilter8i(uint8_t* src_u, uint8_t* src_v, int stride, 562 int b_limit_in, int limit_in, int thresh_in) { 563 uint64_t p1_d, p0_d, q0_d, q1_d; 564 v16u8 p3, p2, p1, p0, q3, q2, q1, q0, mask, hev; 565 v16u8 p3_u, p2_u, p1_u, p0_u, q3_u, q2_u, q1_u, q0_u; 566 v16u8 p3_v, p2_v, p1_v, p0_v, q3_v, q2_v, q1_v, q0_v; 567 const v16u8 thresh = (v16u8)__msa_fill_b(thresh_in); 568 const v16u8 limit = (v16u8)__msa_fill_b(limit_in); 569 const v16u8 b_limit = (v16u8)__msa_fill_b(b_limit_in); 570 571 LD_UB8(src_u, stride, p3_u, p2_u, p1_u, p0_u, q0_u, q1_u, q2_u, q3_u); 572 src_u += (5 * stride); 573 LD_UB8(src_v, stride, p3_v, p2_v, p1_v, p0_v, q0_v, q1_v, q2_v, q3_v); 574 src_v += (5 * stride); 575 ILVR_D4_UB(p3_v, p3_u, p2_v, p2_u, p1_v, p1_u, p0_v, p0_u, p3, p2, p1, p0); 576 ILVR_D4_UB(q0_v, q0_u, q1_v, q1_u, q2_v, q2_u, q3_v, q3_u, q0, q1, q2, q3); 577 LPF_MASK_HEV(p3, p2, p1, p0, q0, q1, q2, q3, limit, b_limit, thresh, 578 hev, mask); 579 LPF_FILTER4_4W(p1, p0, q0, q1, mask, hev); 580 p1_d = __msa_copy_s_d((v2i64)p1, 0); 581 p0_d = __msa_copy_s_d((v2i64)p0, 0); 582 q0_d = __msa_copy_s_d((v2i64)q0, 0); 583 q1_d = __msa_copy_s_d((v2i64)q1, 0); 584 SD4(q1_d, q0_d, p0_d, p1_d, src_u, -stride); 585 p1_d = __msa_copy_s_d((v2i64)p1, 1); 586 p0_d = __msa_copy_s_d((v2i64)p0, 1); 587 q0_d = __msa_copy_s_d((v2i64)q0, 1); 588 q1_d = __msa_copy_s_d((v2i64)q1, 1); 589 SD4(q1_d, q0_d, p0_d, p1_d, src_v, -stride); 590 } 591 592 static void HFilter8i(uint8_t* src_u, uint8_t* src_v, int stride, 593 int b_limit_in, int limit_in, int thresh_in) { 594 v16u8 p3, p2, p1, p0, q3, q2, q1, q0, mask, hev; 595 v16u8 row0, row1, row2, row3, row4, row5, row6, row7, row8; 596 v16u8 row9, row10, row11, row12, row13, row14, row15; 597 v4i32 tmp0, tmp1, tmp2, tmp3, tmp4, tmp5; 598 const v16u8 thresh = (v16u8)__msa_fill_b(thresh_in); 599 const v16u8 limit = (v16u8)__msa_fill_b(limit_in); 600 const v16u8 b_limit = (v16u8)__msa_fill_b(b_limit_in); 601 602 LD_UB8(src_u, stride, row0, row1, row2, row3, row4, row5, row6, row7); 603 LD_UB8(src_v, stride, 604 row8, row9, row10, row11, row12, row13, row14, row15); 605 TRANSPOSE16x8_UB_UB(row0, row1, row2, row3, row4, row5, row6, row7, 606 row8, row9, row10, row11, row12, row13, row14, row15, 607 p3, p2, p1, p0, q0, q1, q2, q3); 608 LPF_MASK_HEV(p3, p2, p1, p0, q0, q1, q2, q3, limit, b_limit, thresh, 609 hev, mask); 610 LPF_FILTER4_4W(p1, p0, q0, q1, mask, hev); 611 ILVR_B2_SW(p0, p1, q1, q0, tmp0, tmp1); 612 ILVRL_H2_SW(tmp1, tmp0, tmp2, tmp3); 613 ILVL_B2_SW(p0, p1, q1, q0, tmp0, tmp1); 614 ILVRL_H2_SW(tmp1, tmp0, tmp4, tmp5); 615 src_u += 2; 616 ST4x4_UB(tmp2, tmp2, 0, 1, 2, 3, src_u, stride); 617 src_u += 4 * stride; 618 ST4x4_UB(tmp3, tmp3, 0, 1, 2, 3, src_u, stride); 619 src_v += 2; 620 ST4x4_UB(tmp4, tmp4, 0, 1, 2, 3, src_v, stride); 621 src_v += 4 * stride; 622 ST4x4_UB(tmp5, tmp5, 0, 1, 2, 3, src_v, stride); 623 } 624 625 static void SimpleVFilter16(uint8_t* src, int stride, int b_limit_in) { 626 v16u8 p1, p0, q1, q0, mask; 627 const v16u8 b_limit = (v16u8)__msa_fill_b(b_limit_in); 628 629 LD_UB4(src - 2 * stride, stride, p1, p0, q0, q1); 630 LPF_SIMPLE_MASK(p1, p0, q0, q1, b_limit, mask); 631 LPF_SIMPLE_FILT(p1, p0, q0, q1, mask); 632 ST_UB2(p0, q0, src - stride, stride); 633 } 634 635 static void SimpleHFilter16(uint8_t* src, int stride, int b_limit_in) { 636 v16u8 p1, p0, q1, q0, mask, row0, row1, row2, row3, row4, row5, row6, row7; 637 v16u8 row8, row9, row10, row11, row12, row13, row14, row15; 638 v8i16 tmp0, tmp1; 639 const v16u8 b_limit = (v16u8)__msa_fill_b(b_limit_in); 640 uint8_t* ptemp_src = src - 2; 641 642 LD_UB8(ptemp_src, stride, row0, row1, row2, row3, row4, row5, row6, row7); 643 LD_UB8(ptemp_src + 8 * stride, stride, 644 row8, row9, row10, row11, row12, row13, row14, row15); 645 TRANSPOSE16x4_UB_UB(row0, row1, row2, row3, row4, row5, row6, row7, 646 row8, row9, row10, row11, row12, row13, row14, row15, 647 p1, p0, q0, q1); 648 LPF_SIMPLE_MASK(p1, p0, q0, q1, b_limit, mask); 649 LPF_SIMPLE_FILT(p1, p0, q0, q1, mask); 650 ILVRL_B2_SH(q0, p0, tmp1, tmp0); 651 ptemp_src += 1; 652 ST2x4_UB(tmp1, 0, ptemp_src, stride); 653 ptemp_src += 4 * stride; 654 ST2x4_UB(tmp1, 4, ptemp_src, stride); 655 ptemp_src += 4 * stride; 656 ST2x4_UB(tmp0, 0, ptemp_src, stride); 657 ptemp_src += 4 * stride; 658 ST2x4_UB(tmp0, 4, ptemp_src, stride); 659 ptemp_src += 4 * stride; 660 } 661 662 static void SimpleVFilter16i(uint8_t* src_y, int stride, int b_limit_in) { 663 SimpleVFilter16(src_y + 4 * stride, stride, b_limit_in); 664 SimpleVFilter16(src_y + 8 * stride, stride, b_limit_in); 665 SimpleVFilter16(src_y + 12 * stride, stride, b_limit_in); 666 } 667 668 static void SimpleHFilter16i(uint8_t* src_y, int stride, int b_limit_in) { 669 SimpleHFilter16(src_y + 4, stride, b_limit_in); 670 SimpleHFilter16(src_y + 8, stride, b_limit_in); 671 SimpleHFilter16(src_y + 12, stride, b_limit_in); 672 } 673 674 //------------------------------------------------------------------------------ 675 // Intra predictions 676 //------------------------------------------------------------------------------ 677 678 // 4x4 679 680 static void DC4(uint8_t* dst) { // DC 681 uint32_t dc = 4; 682 int i; 683 for (i = 0; i < 4; ++i) dc += dst[i - BPS] + dst[-1 + i * BPS]; 684 dc >>= 3; 685 dc = dc | (dc << 8) | (dc << 16) | (dc << 24); 686 SW4(dc, dc, dc, dc, dst, BPS); 687 } 688 689 static void TM4(uint8_t* dst) { 690 const uint8_t* const ptemp = dst - BPS - 1; 691 v8i16 T, d, r0, r1, r2, r3; 692 const v16i8 zero = { 0 }; 693 const v8i16 TL = (v8i16)__msa_fill_h(ptemp[0 * BPS]); 694 const v8i16 L0 = (v8i16)__msa_fill_h(ptemp[1 * BPS]); 695 const v8i16 L1 = (v8i16)__msa_fill_h(ptemp[2 * BPS]); 696 const v8i16 L2 = (v8i16)__msa_fill_h(ptemp[3 * BPS]); 697 const v8i16 L3 = (v8i16)__msa_fill_h(ptemp[4 * BPS]); 698 const v16u8 T1 = LD_UB(ptemp + 1); 699 700 T = (v8i16)__msa_ilvr_b(zero, (v16i8)T1); 701 d = T - TL; 702 ADD4(d, L0, d, L1, d, L2, d, L3, r0, r1, r2, r3); 703 CLIP_SH4_0_255(r0, r1, r2, r3); 704 PCKEV_ST4x4_UB(r0, r1, r2, r3, dst, BPS); 705 } 706 707 static void VE4(uint8_t* dst) { // vertical 708 const uint8_t* const ptop = dst - BPS - 1; 709 const uint32_t val0 = LW(ptop + 0); 710 const uint32_t val1 = LW(ptop + 4); 711 uint32_t out; 712 v16u8 A = { 0 }, B, C, AC, B2, R; 713 714 INSERT_W2_UB(val0, val1, A); 715 B = SLDI_UB(A, A, 1); 716 C = SLDI_UB(A, A, 2); 717 AC = __msa_ave_u_b(A, C); 718 B2 = __msa_ave_u_b(B, B); 719 R = __msa_aver_u_b(AC, B2); 720 out = __msa_copy_s_w((v4i32)R, 0); 721 SW4(out, out, out, out, dst, BPS); 722 } 723 724 static void RD4(uint8_t* dst) { // Down-right 725 const uint8_t* const ptop = dst - 1 - BPS; 726 uint32_t val0 = LW(ptop + 0); 727 uint32_t val1 = LW(ptop + 4); 728 uint32_t val2, val3; 729 v16u8 A, B, C, AC, B2, R, A1 = { 0 }; 730 731 INSERT_W2_UB(val0, val1, A1); 732 A = SLDI_UB(A1, A1, 12); 733 A = (v16u8)__msa_insert_b((v16i8)A, 3, ptop[1 * BPS]); 734 A = (v16u8)__msa_insert_b((v16i8)A, 2, ptop[2 * BPS]); 735 A = (v16u8)__msa_insert_b((v16i8)A, 1, ptop[3 * BPS]); 736 A = (v16u8)__msa_insert_b((v16i8)A, 0, ptop[4 * BPS]); 737 B = SLDI_UB(A, A, 1); 738 C = SLDI_UB(A, A, 2); 739 AC = __msa_ave_u_b(A, C); 740 B2 = __msa_ave_u_b(B, B); 741 R = __msa_aver_u_b(AC, B2); 742 val3 = __msa_copy_s_w((v4i32)R, 0); 743 R = SLDI_UB(R, R, 1); 744 val2 = __msa_copy_s_w((v4i32)R, 0); 745 R = SLDI_UB(R, R, 1); 746 val1 = __msa_copy_s_w((v4i32)R, 0); 747 R = SLDI_UB(R, R, 1); 748 val0 = __msa_copy_s_w((v4i32)R, 0); 749 SW4(val0, val1, val2, val3, dst, BPS); 750 } 751 752 static void LD4(uint8_t* dst) { // Down-Left 753 const uint8_t* const ptop = dst - BPS; 754 uint32_t val0 = LW(ptop + 0); 755 uint32_t val1 = LW(ptop + 4); 756 uint32_t val2, val3; 757 v16u8 A = { 0 }, B, C, AC, B2, R; 758 759 INSERT_W2_UB(val0, val1, A); 760 B = SLDI_UB(A, A, 1); 761 C = SLDI_UB(A, A, 2); 762 C = (v16u8)__msa_insert_b((v16i8)C, 6, ptop[7]); 763 AC = __msa_ave_u_b(A, C); 764 B2 = __msa_ave_u_b(B, B); 765 R = __msa_aver_u_b(AC, B2); 766 val0 = __msa_copy_s_w((v4i32)R, 0); 767 R = SLDI_UB(R, R, 1); 768 val1 = __msa_copy_s_w((v4i32)R, 0); 769 R = SLDI_UB(R, R, 1); 770 val2 = __msa_copy_s_w((v4i32)R, 0); 771 R = SLDI_UB(R, R, 1); 772 val3 = __msa_copy_s_w((v4i32)R, 0); 773 SW4(val0, val1, val2, val3, dst, BPS); 774 } 775 776 // 16x16 777 778 static void DC16(uint8_t* dst) { // DC 779 uint32_t dc = 16; 780 int i; 781 const v16u8 rtop = LD_UB(dst - BPS); 782 const v8u16 dctop = __msa_hadd_u_h(rtop, rtop); 783 v16u8 out; 784 785 for (i = 0; i < 16; ++i) { 786 dc += dst[-1 + i * BPS]; 787 } 788 dc += HADD_UH_U32(dctop); 789 out = (v16u8)__msa_fill_b(dc >> 5); 790 ST_UB8(out, out, out, out, out, out, out, out, dst, BPS); 791 ST_UB8(out, out, out, out, out, out, out, out, dst + 8 * BPS, BPS); 792 } 793 794 static void TM16(uint8_t* dst) { 795 int j; 796 v8i16 d1, d2; 797 const v16i8 zero = { 0 }; 798 const v8i16 TL = (v8i16)__msa_fill_h(dst[-1 - BPS]); 799 const v16i8 T = LD_SB(dst - BPS); 800 801 ILVRL_B2_SH(zero, T, d1, d2); 802 SUB2(d1, TL, d2, TL, d1, d2); 803 for (j = 0; j < 16; j += 4) { 804 v16i8 t0, t1, t2, t3; 805 v8i16 r0, r1, r2, r3, r4, r5, r6, r7; 806 const v8i16 L0 = (v8i16)__msa_fill_h(dst[-1 + 0 * BPS]); 807 const v8i16 L1 = (v8i16)__msa_fill_h(dst[-1 + 1 * BPS]); 808 const v8i16 L2 = (v8i16)__msa_fill_h(dst[-1 + 2 * BPS]); 809 const v8i16 L3 = (v8i16)__msa_fill_h(dst[-1 + 3 * BPS]); 810 ADD4(d1, L0, d1, L1, d1, L2, d1, L3, r0, r1, r2, r3); 811 ADD4(d2, L0, d2, L1, d2, L2, d2, L3, r4, r5, r6, r7); 812 CLIP_SH4_0_255(r0, r1, r2, r3); 813 CLIP_SH4_0_255(r4, r5, r6, r7); 814 PCKEV_B4_SB(r4, r0, r5, r1, r6, r2, r7, r3, t0, t1, t2, t3); 815 ST_SB4(t0, t1, t2, t3, dst, BPS); 816 dst += 4 * BPS; 817 } 818 } 819 820 static void VE16(uint8_t* dst) { // vertical 821 const v16u8 rtop = LD_UB(dst - BPS); 822 ST_UB8(rtop, rtop, rtop, rtop, rtop, rtop, rtop, rtop, dst, BPS); 823 ST_UB8(rtop, rtop, rtop, rtop, rtop, rtop, rtop, rtop, dst + 8 * BPS, BPS); 824 } 825 826 static void HE16(uint8_t* dst) { // horizontal 827 int j; 828 for (j = 16; j > 0; j -= 4) { 829 const v16u8 L0 = (v16u8)__msa_fill_b(dst[-1 + 0 * BPS]); 830 const v16u8 L1 = (v16u8)__msa_fill_b(dst[-1 + 1 * BPS]); 831 const v16u8 L2 = (v16u8)__msa_fill_b(dst[-1 + 2 * BPS]); 832 const v16u8 L3 = (v16u8)__msa_fill_b(dst[-1 + 3 * BPS]); 833 ST_UB4(L0, L1, L2, L3, dst, BPS); 834 dst += 4 * BPS; 835 } 836 } 837 838 static void DC16NoTop(uint8_t* dst) { // DC with top samples not available 839 int j; 840 uint32_t dc = 8; 841 v16u8 out; 842 843 for (j = 0; j < 16; ++j) { 844 dc += dst[-1 + j * BPS]; 845 } 846 out = (v16u8)__msa_fill_b(dc >> 4); 847 ST_UB8(out, out, out, out, out, out, out, out, dst, BPS); 848 ST_UB8(out, out, out, out, out, out, out, out, dst + 8 * BPS, BPS); 849 } 850 851 static void DC16NoLeft(uint8_t* dst) { // DC with left samples not available 852 uint32_t dc = 8; 853 const v16u8 rtop = LD_UB(dst - BPS); 854 const v8u16 dctop = __msa_hadd_u_h(rtop, rtop); 855 v16u8 out; 856 857 dc += HADD_UH_U32(dctop); 858 out = (v16u8)__msa_fill_b(dc >> 4); 859 ST_UB8(out, out, out, out, out, out, out, out, dst, BPS); 860 ST_UB8(out, out, out, out, out, out, out, out, dst + 8 * BPS, BPS); 861 } 862 863 static void DC16NoTopLeft(uint8_t* dst) { // DC with nothing 864 const v16u8 out = (v16u8)__msa_fill_b(0x80); 865 ST_UB8(out, out, out, out, out, out, out, out, dst, BPS); 866 ST_UB8(out, out, out, out, out, out, out, out, dst + 8 * BPS, BPS); 867 } 868 869 // Chroma 870 871 #define STORE8x8(out, dst) do { \ 872 SD4(out, out, out, out, dst + 0 * BPS, BPS); \ 873 SD4(out, out, out, out, dst + 4 * BPS, BPS); \ 874 } while (0) 875 876 static void DC8uv(uint8_t* dst) { // DC 877 uint32_t dc = 8; 878 int i; 879 uint64_t out; 880 const v16u8 rtop = LD_UB(dst - BPS); 881 const v8u16 temp0 = __msa_hadd_u_h(rtop, rtop); 882 const v4u32 temp1 = __msa_hadd_u_w(temp0, temp0); 883 const v2u64 temp2 = __msa_hadd_u_d(temp1, temp1); 884 v16u8 dctemp; 885 886 for (i = 0; i < 8; ++i) { 887 dc += dst[-1 + i * BPS]; 888 } 889 dc += __msa_copy_s_w((v4i32)temp2, 0); 890 dctemp = (v16u8)__msa_fill_b(dc >> 4); 891 out = __msa_copy_s_d((v2i64)dctemp, 0); 892 STORE8x8(out, dst); 893 } 894 895 static void TM8uv(uint8_t* dst) { 896 int j; 897 const v16i8 T1 = LD_SB(dst - BPS); 898 const v16i8 zero = { 0 }; 899 const v8i16 T = (v8i16)__msa_ilvr_b(zero, T1); 900 const v8i16 TL = (v8i16)__msa_fill_h(dst[-1 - BPS]); 901 const v8i16 d = T - TL; 902 903 for (j = 0; j < 8; j += 4) { 904 v16i8 t0, t1; 905 v8i16 r0 = (v8i16)__msa_fill_h(dst[-1 + 0 * BPS]); 906 v8i16 r1 = (v8i16)__msa_fill_h(dst[-1 + 1 * BPS]); 907 v8i16 r2 = (v8i16)__msa_fill_h(dst[-1 + 2 * BPS]); 908 v8i16 r3 = (v8i16)__msa_fill_h(dst[-1 + 3 * BPS]); 909 ADD4(d, r0, d, r1, d, r2, d, r3, r0, r1, r2, r3); 910 CLIP_SH4_0_255(r0, r1, r2, r3); 911 PCKEV_B2_SB(r1, r0, r3, r2, t0, t1); 912 ST4x4_UB(t0, t1, 0, 2, 0, 2, dst, BPS); 913 ST4x4_UB(t0, t1, 1, 3, 1, 3, dst + 4, BPS); 914 dst += 4 * BPS; 915 } 916 } 917 918 static void VE8uv(uint8_t* dst) { // vertical 919 const v16u8 rtop = LD_UB(dst - BPS); 920 const uint64_t out = __msa_copy_s_d((v2i64)rtop, 0); 921 STORE8x8(out, dst); 922 } 923 924 static void HE8uv(uint8_t* dst) { // horizontal 925 int j; 926 for (j = 0; j < 8; j += 4) { 927 const v16u8 L0 = (v16u8)__msa_fill_b(dst[-1 + 0 * BPS]); 928 const v16u8 L1 = (v16u8)__msa_fill_b(dst[-1 + 1 * BPS]); 929 const v16u8 L2 = (v16u8)__msa_fill_b(dst[-1 + 2 * BPS]); 930 const v16u8 L3 = (v16u8)__msa_fill_b(dst[-1 + 3 * BPS]); 931 const uint64_t out0 = __msa_copy_s_d((v2i64)L0, 0); 932 const uint64_t out1 = __msa_copy_s_d((v2i64)L1, 0); 933 const uint64_t out2 = __msa_copy_s_d((v2i64)L2, 0); 934 const uint64_t out3 = __msa_copy_s_d((v2i64)L3, 0); 935 SD4(out0, out1, out2, out3, dst, BPS); 936 dst += 4 * BPS; 937 } 938 } 939 940 static void DC8uvNoLeft(uint8_t* dst) { // DC with no left samples 941 const uint32_t dc = 4; 942 const v16u8 rtop = LD_UB(dst - BPS); 943 const v8u16 temp0 = __msa_hadd_u_h(rtop, rtop); 944 const v4u32 temp1 = __msa_hadd_u_w(temp0, temp0); 945 const v2u64 temp2 = __msa_hadd_u_d(temp1, temp1); 946 const uint32_t sum_m = __msa_copy_s_w((v4i32)temp2, 0); 947 const v16u8 dcval = (v16u8)__msa_fill_b((dc + sum_m) >> 3); 948 const uint64_t out = __msa_copy_s_d((v2i64)dcval, 0); 949 STORE8x8(out, dst); 950 } 951 952 static void DC8uvNoTop(uint8_t* dst) { // DC with no top samples 953 uint32_t dc = 4; 954 int i; 955 uint64_t out; 956 v16u8 dctemp; 957 958 for (i = 0; i < 8; ++i) { 959 dc += dst[-1 + i * BPS]; 960 } 961 dctemp = (v16u8)__msa_fill_b(dc >> 3); 962 out = __msa_copy_s_d((v2i64)dctemp, 0); 963 STORE8x8(out, dst); 964 } 965 966 static void DC8uvNoTopLeft(uint8_t* dst) { // DC with nothing 967 const uint64_t out = 0x8080808080808080ULL; 968 STORE8x8(out, dst); 969 } 970 971 //------------------------------------------------------------------------------ 972 // Entry point 973 974 extern void VP8DspInitMSA(void); 975 976 WEBP_TSAN_IGNORE_FUNCTION void VP8DspInitMSA(void) { 977 VP8TransformWHT = TransformWHT; 978 VP8Transform = TransformTwo; 979 VP8TransformDC = TransformDC; 980 VP8TransformAC3 = TransformAC3; 981 982 VP8VFilter16 = VFilter16; 983 VP8HFilter16 = HFilter16; 984 VP8VFilter16i = VFilter16i; 985 VP8HFilter16i = HFilter16i; 986 VP8VFilter8 = VFilter8; 987 VP8HFilter8 = HFilter8; 988 VP8VFilter8i = VFilter8i; 989 VP8HFilter8i = HFilter8i; 990 VP8SimpleVFilter16 = SimpleVFilter16; 991 VP8SimpleHFilter16 = SimpleHFilter16; 992 VP8SimpleVFilter16i = SimpleVFilter16i; 993 VP8SimpleHFilter16i = SimpleHFilter16i; 994 995 VP8PredLuma4[0] = DC4; 996 VP8PredLuma4[1] = TM4; 997 VP8PredLuma4[2] = VE4; 998 VP8PredLuma4[4] = RD4; 999 VP8PredLuma4[6] = LD4; 1000 VP8PredLuma16[0] = DC16; 1001 VP8PredLuma16[1] = TM16; 1002 VP8PredLuma16[2] = VE16; 1003 VP8PredLuma16[3] = HE16; 1004 VP8PredLuma16[4] = DC16NoTop; 1005 VP8PredLuma16[5] = DC16NoLeft; 1006 VP8PredLuma16[6] = DC16NoTopLeft; 1007 VP8PredChroma8[0] = DC8uv; 1008 VP8PredChroma8[1] = TM8uv; 1009 VP8PredChroma8[2] = VE8uv; 1010 VP8PredChroma8[3] = HE8uv; 1011 VP8PredChroma8[4] = DC8uvNoTop; 1012 VP8PredChroma8[5] = DC8uvNoLeft; 1013 VP8PredChroma8[6] = DC8uvNoTopLeft; 1014 } 1015 1016 #else // !WEBP_USE_MSA 1017 1018 WEBP_DSP_INIT_STUB(VP8DspInitMSA) 1019 1020 #endif // WEBP_USE_MSA 1021
