1 /* 2 * Copyright (c) 2018, Alliance for Open Media. All rights reserved 3 * 4 * This source code is subject to the terms of the BSD 2 Clause License and 5 * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License 6 * was not distributed with this source code in the LICENSE file, you can 7 * obtain it at www.aomedia.org/license/software. If the Alliance for Open 8 * Media Patent License 1.0 was not distributed with this source code in the 9 * PATENTS file, you can obtain it at www.aomedia.org/license/patent. 10 */ 11 12 #include <emmintrin.h> // SSE2 13 #include <smmintrin.h> /* SSE4.1 */ 14 15 #include "aom/aom_integer.h" 16 #include "aom_dsp/blend.h" 17 #include "av1/common/blockd.h" 18 19 static INLINE __m128i calc_mask(const __m128i mask_base, const __m128i s0, 20 const __m128i s1) { 21 const __m128i diff = _mm_abs_epi16(_mm_sub_epi16(s0, s1)); 22 return _mm_abs_epi16(_mm_add_epi16(mask_base, _mm_srli_epi16(diff, 4))); 23 // clamp(diff, 0, 64) can be skiped for diff is always in the range ( 38, 54) 24 } 25 26 void av1_build_compound_diffwtd_mask_sse4_1(uint8_t *mask, 27 DIFFWTD_MASK_TYPE mask_type, 28 const uint8_t *src0, int stride0, 29 const uint8_t *src1, int stride1, 30 int h, int w) { 31 const int mb = (mask_type == DIFFWTD_38_INV) ? AOM_BLEND_A64_MAX_ALPHA : 0; 32 const __m128i mask_base = _mm_set1_epi16(38 - mb); 33 int i = 0; 34 if (4 == w) { 35 do { 36 const __m128i s0A = _mm_cvtsi32_si128(*(uint32_t *)src0); 37 const __m128i s0B = _mm_cvtsi32_si128(*(uint32_t *)(src0 + stride0)); 38 const __m128i s0AB = _mm_unpacklo_epi32(s0A, s0B); 39 const __m128i s0 = _mm_cvtepu8_epi16(s0AB); 40 41 const __m128i s1A = _mm_cvtsi32_si128(*(uint32_t *)src1); 42 const __m128i s1B = _mm_cvtsi32_si128(*(uint32_t *)(src1 + stride1)); 43 const __m128i s1AB = _mm_unpacklo_epi32(s1A, s1B); 44 const __m128i s1 = _mm_cvtepu8_epi16(s1AB); 45 46 const __m128i m16 = calc_mask(mask_base, s0, s1); 47 const __m128i m8 = _mm_packus_epi16(m16, m16); 48 49 *(uint32_t *)mask = _mm_cvtsi128_si32(m8); 50 *(uint32_t *)(mask + w) = _mm_extract_epi32(m8, 1); 51 src0 += (stride0 << 1); 52 src1 += (stride1 << 1); 53 mask += 8; 54 i += 2; 55 } while (i < h); 56 } else if (8 == w) { 57 do { 58 __m128i s0 = _mm_loadl_epi64((__m128i const *)src0); 59 __m128i s1 = _mm_loadl_epi64((__m128i const *)src1); 60 s0 = _mm_cvtepu8_epi16(s0); 61 s1 = _mm_cvtepu8_epi16(s1); 62 const __m128i m16 = calc_mask(mask_base, s0, s1); 63 const __m128i m8 = _mm_packus_epi16(m16, m16); 64 _mm_storel_epi64((__m128i *)mask, m8); 65 src0 += stride0; 66 src1 += stride1; 67 mask += 8; 68 i += 1; 69 } while (i < h); 70 } else { 71 const __m128i zero = _mm_setzero_si128(); 72 do { 73 int j = 0; 74 do { 75 const __m128i s0 = _mm_load_si128((__m128i const *)(src0 + j)); 76 const __m128i s1 = _mm_load_si128((__m128i const *)(src1 + j)); 77 const __m128i s0L = _mm_cvtepu8_epi16(s0); 78 const __m128i s1L = _mm_cvtepu8_epi16(s1); 79 const __m128i s0H = _mm_unpackhi_epi8(s0, zero); 80 const __m128i s1H = _mm_unpackhi_epi8(s1, zero); 81 82 const __m128i m16L = calc_mask(mask_base, s0L, s1L); 83 const __m128i m16H = calc_mask(mask_base, s0H, s1H); 84 85 const __m128i m8 = _mm_packus_epi16(m16L, m16H); 86 _mm_store_si128((__m128i *)(mask + j), m8); 87 j += 16; 88 } while (j < w); 89 src0 += stride0; 90 src1 += stride1; 91 mask += w; 92 i += 1; 93 } while (i < h); 94 } 95 } 96 97 void av1_build_compound_diffwtd_mask_d16_sse4_1( 98 uint8_t *mask, DIFFWTD_MASK_TYPE mask_type, const CONV_BUF_TYPE *src0, 99 int src0_stride, const CONV_BUF_TYPE *src1, int src1_stride, int h, int w, 100 ConvolveParams *conv_params, int bd) { 101 const int which_inverse = (mask_type == DIFFWTD_38) ? 0 : 1; 102 const int mask_base = 38; 103 int round = 104 2 * FILTER_BITS - conv_params->round_0 - conv_params->round_1 + (bd - 8); 105 const __m128i round_const = _mm_set1_epi16((1 << round) >> 1); 106 const __m128i mask_base_16 = _mm_set1_epi16(mask_base); 107 const __m128i clip_diff = _mm_set1_epi16(AOM_BLEND_A64_MAX_ALPHA); 108 const __m128i add_const = 109 _mm_set1_epi16((which_inverse ? AOM_BLEND_A64_MAX_ALPHA : 0)); 110 const __m128i add_sign = _mm_set1_epi16((which_inverse ? -1 : 1)); 111 112 int i, j; 113 // When rounding constant is added, there is a possibility of overflow. 114 // However that much precision is not required. Code should very well work for 115 // other values of DIFF_FACTOR_LOG2 and AOM_BLEND_A64_MAX_ALPHA as well. But 116 // there is a possibility of corner case bugs. 117 assert(DIFF_FACTOR_LOG2 == 4); 118 assert(AOM_BLEND_A64_MAX_ALPHA == 64); 119 for (i = 0; i < h; ++i) { 120 for (j = 0; j < w; j += 8) { 121 const __m128i data_src0 = 122 _mm_loadu_si128((__m128i *)&src0[(i * src0_stride) + j]); 123 const __m128i data_src1 = 124 _mm_loadu_si128((__m128i *)&src1[(i * src1_stride) + j]); 125 126 const __m128i diffa = _mm_subs_epu16(data_src0, data_src1); 127 const __m128i diffb = _mm_subs_epu16(data_src1, data_src0); 128 const __m128i diff = _mm_max_epu16(diffa, diffb); 129 const __m128i diff_round = 130 _mm_srli_epi16(_mm_adds_epu16(diff, round_const), round); 131 const __m128i diff_factor = _mm_srli_epi16(diff_round, DIFF_FACTOR_LOG2); 132 const __m128i diff_mask = _mm_adds_epi16(diff_factor, mask_base_16); 133 __m128i diff_clamp = _mm_min_epi16(diff_mask, clip_diff); 134 // clamp to 0 can be skipped since we are using add and saturate 135 // instruction 136 137 const __m128i diff_sign = _mm_sign_epi16(diff_clamp, add_sign); 138 const __m128i diff_const_16 = _mm_add_epi16(diff_sign, add_const); 139 140 // 8 bit conversion and saturation to uint8 141 const __m128i res_8 = _mm_packus_epi16(diff_const_16, diff_const_16); 142 143 // Store values into the destination buffer 144 __m128i *const dst = (__m128i *)&mask[i * w + j]; 145 146 if ((w - j) > 4) { 147 _mm_storel_epi64(dst, res_8); 148 } else { // w==4 149 *(uint32_t *)dst = _mm_cvtsi128_si32(res_8); 150 } 151 } 152 } 153 } 154
