Lines Matching refs:__m128i
30 const __m128i max_coeff_thresh = _mm_set1_epi16(MAX_COEFF_THRESH);
37 const __m128i out0 = _mm_loadu_si128((__m128i*)&out[0]);
38 const __m128i out1 = _mm_loadu_si128((__m128i*)&out[8]);
40 const __m128i sign0 = _mm_srai_epi16(out0, 15);
41 const __m128i sign1 = _mm_srai_epi16(out1, 15);
43 const __m128i xor0 = _mm_xor_si128(out0, sign0);
44 const __m128i xor1 = _mm_xor_si128(out1, sign1);
45 const __m128i abs0 = _mm_sub_epi16(xor0, sign0);
46 const __m128i abs1 = _mm_sub_epi16(xor1, sign1);
48 const __m128i v0 = _mm_srai_epi16(abs0, 2);
49 const __m128i v1 = _mm_srai_epi16(abs1, 2);
51 const __m128i bin0 = _mm_min_epi16(v0, max_coeff_thresh);
52 const __m128i bin1 = _mm_min_epi16(v1, max_coeff_thresh);
54 _mm_storeu_si128((__m128i*)&out[0], bin0);
55 _mm_storeu_si128((__m128i*)&out[8], bin1);
89 const __m128i k1 = _mm_set1_epi16(20091);
90 const __m128i k2 = _mm_set1_epi16(-30068);
91 __m128i T0, T1, T2, T3;
97 __m128i in0, in1, in2, in3;
99 in0 = _mm_loadl_epi64((__m128i*)&in[0]);
100 in1 = _mm_loadl_epi64((__m128i*)&in[4]);
101 in2 = _mm_loadl_epi64((__m128i*)&in[8]);
102 in3 = _mm_loadl_epi64((__m128i*)&in[12]);
108 const __m128i inB0 = _mm_loadl_epi64((__m128i*)&in[16]);
109 const __m128i inB1 = _mm_loadl_epi64((__m128i*)&in[20]);
110 const __m128i inB2 = _mm_loadl_epi64((__m128i*)&in[24]);
111 const __m128i inB3 = _mm_loadl_epi64((__m128i*)&in[28]);
127 const __m128i a = _mm_add_epi16(in0, in2);
128 const __m128i b = _mm_sub_epi16(in0, in2);
130 const __m128i c1 = _mm_mulhi_epi16(in1, k2);
131 const __m128i c2 = _mm_mulhi_epi16(in3, k1);
132 const __m128i c3 = _mm_sub_epi16(in1, in3);
133 const __m128i c4 = _mm_sub_epi16(c1, c2);
134 const __m128i c = _mm_add_epi16(c3, c4);
136 const __m128i d1 = _mm_mulhi_epi16(in1, k1);
137 const __m128i d2 = _mm_mulhi_epi16(in3, k2);
138 const __m128i d3 = _mm_add_epi16(in1, in3);
139 const __m128i d4 = _mm_add_epi16(d1, d2);
140 const __m128i d = _mm_add_epi16(d3, d4);
143 const __m128i tmp0 = _mm_add_epi16(a, d);
144 const __m128i tmp1 = _mm_add_epi16(b, c);
145 const __m128i tmp2 = _mm_sub_epi16(b, c);
146 const __m128i tmp3 = _mm_sub_epi16(a, d);
153 const __m128i transpose0_0 = _mm_unpacklo_epi16(tmp0, tmp1);
154 const __m128i transpose0_1 = _mm_unpacklo_epi16(tmp2, tmp3);
155 const __m128i transpose0_2 = _mm_unpackhi_epi16(tmp0, tmp1);
156 const __m128i transpose0_3 = _mm_unpackhi_epi16(tmp2, tmp3);
161 const __m128i transpose1_0 = _mm_unpacklo_epi32(transpose0_0, transpose0_1);
162 const __m128i transpose1_1 = _mm_unpacklo_epi32(transpose0_2, transpose0_3);
163 const __m128i transpose1_2 = _mm_unpackhi_epi32(transpose0_0, transpose0_1);
164 const __m128i transpose1_3 = _mm_unpackhi_epi32(transpose0_2, transpose0_3);
183 const __m128i four = _mm_set1_epi16(4);
184 const __m128i dc = _mm_add_epi16(T0, four);
185 const __m128i a = _mm_add_epi16(dc, T2);
186 const __m128i b = _mm_sub_epi16(dc, T2);
188 const __m128i c1 = _mm_mulhi_epi16(T1, k2);
189 const __m128i c2 = _mm_mulhi_epi16(T3, k1);
190 const __m128i c3 = _mm_sub_epi16(T1, T3);
191 const __m128i c4 = _mm_sub_epi16(c1, c2);
192 const __m128i c = _mm_add_epi16(c3, c4);
194 const __m128i d1 = _mm_mulhi_epi16(T1, k1);
195 const __m128i d2 = _mm_mulhi_epi16(T3, k2);
196 const __m128i d3 = _mm_add_epi16(T1, T3);
197 const __m128i d4 = _mm_add_epi16(d1, d2);
198 const __m128i d = _mm_add_epi16(d3, d4);
201 const __m128i tmp0 = _mm_add_epi16(a, d);
202 const __m128i tmp1 = _mm_add_epi16(b, c);
203 const __m128i tmp2 = _mm_sub_epi16(b, c);
204 const __m128i tmp3 = _mm_sub_epi16(a, d);
205 const __m128i shifted0 = _mm_srai_epi16(tmp0, 3);
206 const __m128i shifted1 = _mm_srai_epi16(tmp1, 3);
207 const __m128i shifted2 = _mm_srai_epi16(tmp2, 3);
208 const __m128i shifted3 = _mm_srai_epi16(tmp3, 3);
215 const __m128i transpose0_0 = _mm_unpacklo_epi16(shifted0, shifted1);
216 const __m128i transpose0_1 = _mm_unpacklo_epi16(shifted2, shifted3);
217 const __m128i transpose0_2 = _mm_unpackhi_epi16(shifted0, shifted1);
218 const __m128i transpose0_3 = _mm_unpackhi_epi16(shifted2, shifted3);
223 const __m128i transpose1_0 = _mm_unpacklo_epi32(transpose0_0, transpose0_1);
224 const __m128i transpose1_1 = _mm_unpacklo_epi32(transpose0_2, transpose0_3);
225 const __m128i transpose1_2 = _mm_unpackhi_epi32(transpose0_0, transpose0_1);
226 const __m128i transpose1_3 = _mm_unpackhi_epi32(transpose0_2, transpose0_3);
243 const __m128i zero = _mm_set1_epi16(0);
245 __m128i ref0, ref1, ref2, ref3;
248 ref0 = _mm_loadl_epi64((__m128i*)&ref[0 * BPS]);
249 ref1 = _mm_loadl_epi64((__m128i*)&ref[1 * BPS]);
250 ref2 = _mm_loadl_epi64((__m128i*)&ref[2 * BPS]);
251 ref3 = _mm_loadl_epi64((__m128i*)&ref[3 * BPS]);
277 _mm_storel_epi64((__m128i*)&dst[0 * BPS], ref0);
278 _mm_storel_epi64((__m128i*)&dst[1 * BPS], ref1);
279 _mm_storel_epi64((__m128i*)&dst[2 * BPS], ref2);
280 _mm_storel_epi64((__m128i*)&dst[3 * BPS], ref3);
293 const __m128i zero = _mm_setzero_si128();
294 const __m128i seven = _mm_set1_epi16(7);
295 const __m128i k7500 = _mm_set1_epi32(7500);
296 const __m128i k14500 = _mm_set1_epi32(14500);
297 const __m128i k51000 = _mm_set1_epi32(51000);
298 const __m128i k12000_plus_one = _mm_set1_epi32(12000 + (1 << 16));
299 const __m128i k5352_2217 = _mm_set_epi16(5352, 2217, 5352, 2217,
301 const __m128i k2217_5352 = _mm_set_epi16(2217, -5352, 2217, -5352,
304 __m128i v01, v32;
309 const __m128i src0 = _mm_loadl_epi64((__m128i*)&src[0 * BPS]);
310 const __m128i src1 = _mm_loadl_epi64((__m128i*)&src[1 * BPS]);
311 const __m128i src2 = _mm_loadl_epi64((__m128i*)&src[2 * BPS]);
312 const __m128i src3 = _mm_loadl_epi64((__m128i*)&src[3 * BPS]);
313 const __m128i src_0 = _mm_unpacklo_epi8(src0, zero);
314 const __m128i src_1 = _mm_unpacklo_epi8(src1, zero);
315 const __m128i src_2 = _mm_unpacklo_epi8(src2, zero);
316 const __m128i src_3 = _mm_unpacklo_epi8(src3, zero);
318 const __m128i ref0 = _mm_loadl_epi64((__m128i*)&ref[0 * BPS]);
319 const __m128i ref1 = _mm_loadl_epi64((__m128i*)&ref[1 * BPS]);
320 const __m128i ref2 = _mm_loadl_epi64((__m128i*)&ref[2 * BPS]);
321 const __m128i ref3 = _mm_loadl_epi64((__m128i*)&ref[3 * BPS]);
322 const __m128i ref_0 = _mm_unpacklo_epi8(ref0, zero);
323 const __m128i ref_1 = _mm_unpacklo_epi8(ref1, zero);
324 const __m128i ref_2 = _mm_unpacklo_epi8(ref2, zero);
325 const __m128i ref_3 = _mm_unpacklo_epi8(ref3, zero);
327 const __m128i diff0 = _mm_sub_epi16(src_0, ref_0);
328 const __m128i diff1 = _mm_sub_epi16(src_1, ref_1);
329 const __m128i diff2 = _mm_sub_epi16(src_2, ref_2);
330 const __m128i diff3 = _mm_sub_epi16(src_3, ref_3);
337 const __m128i transpose0_0 = _mm_unpacklo_epi16(diff0, diff1);
338 const __m128i transpose0_1 = _mm_unpacklo_epi16(diff2, diff3);
341 const __m128i v23 = _mm_unpackhi_epi32(transpose0_0, transpose0_1);
356 const __m128i a01 = _mm_add_epi16(v01, v32);
357 const __m128i a32 = _mm_sub_epi16(v01, v32);
358 const __m128i b01 = _mm_slli_epi16(a01, 3);
359 const __m128i b32 = _mm_slli_epi16(a32, 3);
360 const __m128i b11 = _mm_unpackhi_epi64(b01, b01);
361 const __m128i b22 = _mm_unpackhi_epi64(b32, b32);
365 const __m128i e0 = _mm_add_epi16(b01, b11);
366 const __m128i e2 = _mm_sub_epi16(b01, b11);
367 const __m128i e02 = _mm_unpacklo_epi64(e0, e2);
371 const __m128i b23 = _mm_unpacklo_epi16(b22, b32);
372 const __m128i c1 = _mm_madd_epi16(b23, k5352_2217);
373 const __m128i c3 = _mm_madd_epi16(b23, k2217_5352);
374 const __m128i d1 = _mm_add_epi32(c1, k14500);
375 const __m128i d3 = _mm_add_epi32(c3, k7500);
376 const __m128i e1 = _mm_srai_epi32(d1, 12);
377 const __m128i e3 = _mm_srai_epi32(d3, 12);
378 const __m128i e13 = _mm_packs_epi32(e1, e3);
383 const __m128i transpose0_0 = _mm_unpacklo_epi16(e02, e13);
384 const __m128i transpose0_1 = _mm_unpackhi_epi16(e02, e13);
387 __m128i v23 = _mm_unpackhi_epi32(transpose0_0, transpose0_1);
402 const __m128i a01 = _mm_add_epi16(v01, v32);
403 const __m128i a32 = _mm_sub_epi16(v01, v32);
404 const __m128i a11 = _mm_unpackhi_epi64(a01, a01);
405 const __m128i a22 = _mm_unpackhi_epi64(a32, a32);
409 const __m128i b0 = _mm_add_epi16(a01, a11);
410 const __m128i b2 = _mm_sub_epi16(a01, a11);
411 const __m128i c0 = _mm_add_epi16(b0, seven);
412 const __m128i c2 = _mm_add_epi16(b2, seven);
413 const __m128i d0 = _mm_srai_epi16(c0, 4);
414 const __m128i d2 = _mm_srai_epi16(c2, 4);
418 const __m128i b23 = _mm_unpacklo_epi16(a22, a32);
419 const __m128i c1 = _mm_madd_epi16(b23, k5352_2217);
420 const __m128i c3 = _mm_madd_epi16(b23, k2217_5352);
421 const __m128i d1 = _mm_add_epi32(c1, k12000_plus_one);
422 const __m128i d3 = _mm_add_epi32(c3, k51000);
423 const __m128i e1 = _mm_srai_epi32(d1, 16);
424 const __m128i e3 = _mm_srai_epi32(d3, 16);
425 const __m128i f1 = _mm_packs_epi32(e1, e1);
426 const __m128i f3 = _mm_packs_epi32(e3, e3);
430 const __m128i g1 = _mm_add_epi16(f1, _mm_cmpeq_epi16(a32, zero));
432 _mm_storel_epi64((__m128i*)&out[ 0], d0);
433 _mm_storel_epi64((__m128i*)&out[ 4], g1);
434 _mm_storel_epi64((__m128i*)&out[ 8], d2);
435 _mm_storel_epi64((__m128i*)&out[12], f3);
443 const __m128i zero = _mm_set1_epi16(0);
446 const __m128i a0 = _mm_loadl_epi64((__m128i*)&a[BPS * 0]);
447 const __m128i a1 = _mm_loadl_epi64((__m128i*)&a[BPS * 1]);
448 const __m128i a2 = _mm_loadl_epi64((__m128i*)&a[BPS * 2]);
449 const __m128i a3 = _mm_loadl_epi64((__m128i*)&a[BPS * 3]);
450 const __m128i b0 = _mm_loadl_epi64((__m128i*)&b[BPS * 0]);
451 const __m128i b1 = _mm_loadl_epi64((__m128i*)&b[BPS * 1]);
452 const __m128i b2 = _mm_loadl_epi64((__m128i*)&b[BPS * 2]);
453 const __m128i b3 = _mm_loadl_epi64((__m128i*)&b[BPS * 3]);
456 const __m128i a01 = _mm_unpacklo_epi32(a0, a1);
457 const __m128i a23 = _mm_unpacklo_epi32(a2, a3);
458 const __m128i b01 = _mm_unpacklo_epi32(b0, b1);
459 const __m128i b23 = _mm_unpacklo_epi32(b2, b3);
460 const __m128i a01s = _mm_unpacklo_epi8(a01, zero);
461 const __m128i a23s = _mm_unpacklo_epi8(a23, zero);
462 const __m128i b01s = _mm_unpacklo_epi8(b01, zero);
463 const __m128i b23s = _mm_unpacklo_epi8(b23, zero);
470 const __m128i d0 = _mm_subs_epu8(a01s, b01s);
471 const __m128i d1 = _mm_subs_epu8(b01s, a01s);
472 const __m128i d2 = _mm_subs_epu8(a23s, b23s);
473 const __m128i d3 = _mm_subs_epu8(b23s, a23s);
476 const __m128i madd0 = _mm_madd_epi16(d0, d0);
477 const __m128i madd1 = _mm_madd_epi16(d1, d1);
478 const __m128i madd2 = _mm_madd_epi16(d2, d2);
479 const __m128i madd3 = _mm_madd_epi16(d3, d3);
480 const __m128i sum0 = _mm_add_epi32(madd0, madd1);
481 const __m128i sum1 = _mm_add_epi32(madd2, madd3);
482 const __m128i sum2 = _mm_add_epi32(sum0, sum1);
484 _mm_storeu_si128((__m128i*)tmp, sum2);
500 __m128i tmp_0, tmp_1, tmp_2, tmp_3;
501 const __m128i zero = _mm_setzero_si128();
502 const __m128i one = _mm_set1_epi16(1);
503 const __m128i three = _mm_set1_epi16(3);
507 const __m128i inA_0 = _mm_loadl_epi64((__m128i*)&inA[BPS * 0]);
508 const __m128i inA_1 = _mm_loadl_epi64((__m128i*)&inA[BPS * 1]);
509 const __m128i inA_2 = _mm_loadl_epi64((__m128i*)&inA[BPS * 2]);
510 const __m128i inA_3 = _mm_loadl_epi64((__m128i*)&inA[BPS * 3]);
511 const __m128i inB_0 = _mm_loadl_epi64((__m128i*)&inB[BPS * 0]);
512 const __m128i inB_1 = _mm_loadl_epi64((__m128i*)&inB[BPS * 1]);
513 const __m128i inB_2 = _mm_loadl_epi64((__m128i*)&inB[BPS * 2]);
514 const __m128i inB_3 = _mm_loadl_epi64((__m128i*)&inB[BPS * 3]);
517 const __m128i inAB_0 = _mm_unpacklo_epi8(inA_0, inB_0);
518 const __m128i inAB_1 = _mm_unpacklo_epi8(inA_1, inB_1);
519 const __m128i inAB_2 = _mm_unpacklo_epi8(inA_2, inB_2);
520 const __m128i inAB_3 = _mm_unpacklo_epi8(inA_3, inB_3);
527 const __m128i transpose0_0 = _mm_unpacklo_epi8(inAB_0, inAB_2);
528 const __m128i transpose0_1 = _mm_unpacklo_epi8(inAB_1, inAB_3);
531 const __m128i transpose1_0 = _mm_unpacklo_epi8(transpose0_0, transpose0_1);
532 const __m128i transpose1_1 = _mm_unpackhi_epi8(transpose0_0, transpose0_1);
550 const __m128i a0 = _mm_slli_epi16(_mm_add_epi16(tmp_0, tmp_2), 2);
551 const __m128i a1 = _mm_slli_epi16(_mm_add_epi16(tmp_1, tmp_3), 2);
552 const __m128i a2 = _mm_slli_epi16(_mm_sub_epi16(tmp_1, tmp_3), 2);
553 const __m128i a3 = _mm_slli_epi16(_mm_sub_epi16(tmp_0, tmp_2), 2);
555 const __m128i b0_extra = _mm_andnot_si128(_mm_cmpeq_epi16 (a0, zero), one);
556 const __m128i b0_base = _mm_add_epi16(a0, a1);
557 const __m128i b1 = _mm_add_epi16(a3, a2);
558 const __m128i b2 = _mm_sub_epi16(a3, a2);
559 const __m128i b3 = _mm_sub_epi16(a0, a1);
560 const __m128i b0 = _mm_add_epi16(b0_base, b0_extra);
567 const __m128i transpose0_0 = _mm_unpacklo_epi16(b0, b1);
568 const __m128i transpose0_1 = _mm_unpacklo_epi16(b2, b3);
569 const __m128i transpose0_2 = _mm_unpackhi_epi16(b0, b1);
570 const __m128i transpose0_3 = _mm_unpackhi_epi16(b2, b3);
575 const __m128i transpose1_0 = _mm_unpacklo_epi32(transpose0_0, transpose0_1);
576 const __m128i transpose1_1 = _mm_unpacklo_epi32(transpose0_2, transpose0_3);
577 const __m128i transpose1_2 = _mm_unpackhi_epi32(transpose0_0, transpose0_1);
578 const __m128i transpose1_3 = _mm_unpackhi_epi32(transpose0_2, transpose0_3);
598 const __m128i w_0 = _mm_loadu_si128((__m128i*)&w[0]);
599 const __m128i w_8 = _mm_loadu_si128((__m128i*)&w[8]);
602 const __m128i a0 = _mm_add_epi16(tmp_0, tmp_2);
603 const __m128i a1 = _mm_add_epi16(tmp_1, tmp_3);
604 const __m128i a2 = _mm_sub_epi16(tmp_1, tmp_3);
605 const __m128i a3 = _mm_sub_epi16(tmp_0, tmp_2);
606 const __m128i b0 = _mm_add_epi16(a0, a1);
607 const __m128i b1 = _mm_add_epi16(a3, a2);
608 const __m128i b2 = _mm_sub_epi16(a3, a2);
609 const __m128i b3 = _mm_sub_epi16(a0, a1);
612 __m128i A_b0 = _mm_unpacklo_epi64(b0, b1);
613 __m128i A_b2 = _mm_unpacklo_epi64(b2, b3);
614 __m128i B_b0 = _mm_unpackhi_epi64(b0, b1);
615 __m128i B_b2 = _mm_unpackhi_epi64(b2, b3);
619 const __m128i sign_A_b0 = _mm_srai_epi16(A_b0, 15);
620 const __m128i sign_A_b2 = _mm_srai_epi16(A_b2, 15);
621 const __m128i sign_B_b0 = _mm_srai_epi16(B_b0, 15);
622 const __m128i sign_B_b2 = _mm_srai_epi16(B_b2, 15);
658 _mm_storeu_si128((__m128i*)&sum[0], A_b0);
689 const __m128i max_coeff_2047 = _mm_set1_epi16(2047);
690 const __m128i zero = _mm_set1_epi16(0);
691 __m128i sign0, sign8;
692 __m128i coeff0, coeff8;
693 __m128i out0, out8;
694 __m128i packed_out;
699 __m128i in0 = _mm_loadu_si128((__m128i*)&in[0]);
700 __m128i in8 = _mm_loadu_si128((__m128i*)&in[8]);
701 const __m128i sharpen0 = _mm_loadu_si128((__m128i*)&mtx->sharpen_[0]);
702 const __m128i sharpen8 = _mm_loadu_si128((__m128i*)&mtx->sharpen_[8]);
703 const __m128i iq0 = _mm_loadu_si128((__m128i*)&mtx->iq_[0]);
704 const __m128i iq8 = _mm_loadu_si128((__m128i*)&mtx->iq_[8]);
705 const __m128i bias0 = _mm_loadu_si128((__m128i*)&mtx->bias_[0]);
706 const __m128i bias8 = _mm_loadu_si128((__m128i*)&mtx->bias_[8]);
707 const __m128i q0 = _mm_loadu_si128((__m128i*)&mtx->q_[0]);
708 const __m128i q8 = _mm_loadu_si128((__m128i*)&mtx->q_[8]);
709 const __m128i zthresh0 = _mm_loadu_si128((__m128i*)&mtx->zthresh_[0]);
710 const __m128i zthresh8 = _mm_loadu_si128((__m128i*)&mtx->zthresh_[8]);
734 __m128i coeff_iQ0H = _mm_mulhi_epu16(coeff0, iq0);
735 __m128i coeff_iQ0L = _mm_mullo_epi16(coeff0, iq0);
736 __m128i coeff_iQ8H = _mm_mulhi_epu16(coeff8, iq8);
737 __m128i coeff_iQ8L = _mm_mullo_epi16(coeff8, iq8);
738 __m128i out_00 = _mm_unpacklo_epi16(coeff_iQ0L, coeff_iQ0H);
739 __m128i out_04 = _mm_unpackhi_epi16(coeff_iQ0L, coeff_iQ0H);
740 __m128i out_08 = _mm_unpacklo_epi16(coeff_iQ8L, coeff_iQ8H);
741 __m128i out_12 = _mm_unpackhi_epi16(coeff_iQ8L, coeff_iQ8H);
743 __m128i bias_00 = _mm_unpacklo_epi16(bias0, zero);
744 __m128i bias_04 = _mm_unpackhi_epi16(bias0, zero);
745 __m128i bias_08 = _mm_unpacklo_epi16(bias8, zero);
746 __m128i bias_12 = _mm_unpackhi_epi16(bias8, zero);
774 __m128i cmp0 = _mm_cmpgt_epi16(coeff0, zthresh0);
775 __m128i cmp8 = _mm_cmpgt_epi16(coeff8, zthresh8);
778 _mm_storeu_si128((__m128i*)&in[0], in0);
779 _mm_storeu_si128((__m128i*)&in[8], in8);
790 __m128i outZ0, outZ8;
797 _mm_storeu_si128((__m128i*)&out[0], outZ0);
798 _mm_storeu_si128((__m128i*)&out[8], outZ8);
811 _mm_storeu_si128((__m128i*)tmp, packed_out);
