1 // Copyright 2015 Google Inc. All Rights Reserved. 2 // 3 // Use of this source code is governed by a BSD-style license 4 // that can be found in the COPYING file in the root of the source 5 // tree. An additional intellectual property rights grant can be found 6 // in the file PATENTS. All contributing project authors may 7 // be found in the AUTHORS file in the root of the source tree. 8 // ----------------------------------------------------------------------------- 9 // 10 // SSE4 version of some encoding functions. 11 // 12 // Author: Skal (pascal.massimino (at) gmail.com) 13 14 #include "src/dsp/dsp.h" 15 16 #if defined(WEBP_USE_SSE41) 17 #include <smmintrin.h> 18 #include <stdlib.h> // for abs() 19 20 #include "src/dsp/common_sse2.h" 21 #include "src/enc/vp8i_enc.h" 22 23 //------------------------------------------------------------------------------ 24 // Compute susceptibility based on DCT-coeff histograms. 25 26 static void CollectHistogram_SSE41(const uint8_t* ref, const uint8_t* pred, 27 int start_block, int end_block, 28 VP8Histogram* const histo) { 29 const __m128i max_coeff_thresh = _mm_set1_epi16(MAX_COEFF_THRESH); 30 int j; 31 int distribution[MAX_COEFF_THRESH + 1] = { 0 }; 32 for (j = start_block; j < end_block; ++j) { 33 int16_t out[16]; 34 int k; 35 36 VP8FTransform(ref + VP8DspScan[j], pred + VP8DspScan[j], out); 37 38 // Convert coefficients to bin (within out[]). 39 { 40 // Load. 41 const __m128i out0 = _mm_loadu_si128((__m128i*)&out[0]); 42 const __m128i out1 = _mm_loadu_si128((__m128i*)&out[8]); 43 // v = abs(out) >> 3 44 const __m128i abs0 = _mm_abs_epi16(out0); 45 const __m128i abs1 = _mm_abs_epi16(out1); 46 const __m128i v0 = _mm_srai_epi16(abs0, 3); 47 const __m128i v1 = _mm_srai_epi16(abs1, 3); 48 // bin = min(v, MAX_COEFF_THRESH) 49 const __m128i bin0 = _mm_min_epi16(v0, max_coeff_thresh); 50 const __m128i bin1 = _mm_min_epi16(v1, max_coeff_thresh); 51 // Store. 52 _mm_storeu_si128((__m128i*)&out[0], bin0); 53 _mm_storeu_si128((__m128i*)&out[8], bin1); 54 } 55 56 // Convert coefficients to bin. 57 for (k = 0; k < 16; ++k) { 58 ++distribution[out[k]]; 59 } 60 } 61 VP8SetHistogramData(distribution, histo); 62 } 63 64 //------------------------------------------------------------------------------ 65 // Texture distortion 66 // 67 // We try to match the spectral content (weighted) between source and 68 // reconstructed samples. 69 70 // Hadamard transform 71 // Returns the weighted sum of the absolute value of transformed coefficients. 72 // w[] contains a row-major 4 by 4 symmetric matrix. 73 static int TTransform_SSE41(const uint8_t* inA, const uint8_t* inB, 74 const uint16_t* const w) { 75 int32_t sum[4]; 76 __m128i tmp_0, tmp_1, tmp_2, tmp_3; 77 78 // Load and combine inputs. 79 { 80 const __m128i inA_0 = _mm_loadu_si128((const __m128i*)&inA[BPS * 0]); 81 const __m128i inA_1 = _mm_loadu_si128((const __m128i*)&inA[BPS * 1]); 82 const __m128i inA_2 = _mm_loadu_si128((const __m128i*)&inA[BPS * 2]); 83 // In SSE4.1, with gcc 4.8 at least (maybe other versions), 84 // _mm_loadu_si128 is faster than _mm_loadl_epi64. But for the last lump 85 // of inA and inB, _mm_loadl_epi64 is still used not to have an out of 86 // bound read. 87 const __m128i inA_3 = _mm_loadl_epi64((const __m128i*)&inA[BPS * 3]); 88 const __m128i inB_0 = _mm_loadu_si128((const __m128i*)&inB[BPS * 0]); 89 const __m128i inB_1 = _mm_loadu_si128((const __m128i*)&inB[BPS * 1]); 90 const __m128i inB_2 = _mm_loadu_si128((const __m128i*)&inB[BPS * 2]); 91 const __m128i inB_3 = _mm_loadl_epi64((const __m128i*)&inB[BPS * 3]); 92 93 // Combine inA and inB (we'll do two transforms in parallel). 94 const __m128i inAB_0 = _mm_unpacklo_epi32(inA_0, inB_0); 95 const __m128i inAB_1 = _mm_unpacklo_epi32(inA_1, inB_1); 96 const __m128i inAB_2 = _mm_unpacklo_epi32(inA_2, inB_2); 97 const __m128i inAB_3 = _mm_unpacklo_epi32(inA_3, inB_3); 98 tmp_0 = _mm_cvtepu8_epi16(inAB_0); 99 tmp_1 = _mm_cvtepu8_epi16(inAB_1); 100 tmp_2 = _mm_cvtepu8_epi16(inAB_2); 101 tmp_3 = _mm_cvtepu8_epi16(inAB_3); 102 // a00 a01 a02 a03 b00 b01 b02 b03 103 // a10 a11 a12 a13 b10 b11 b12 b13 104 // a20 a21 a22 a23 b20 b21 b22 b23 105 // a30 a31 a32 a33 b30 b31 b32 b33 106 } 107 108 // Vertical pass first to avoid a transpose (vertical and horizontal passes 109 // are commutative because w/kWeightY is symmetric) and subsequent transpose. 110 { 111 // Calculate a and b (two 4x4 at once). 112 const __m128i a0 = _mm_add_epi16(tmp_0, tmp_2); 113 const __m128i a1 = _mm_add_epi16(tmp_1, tmp_3); 114 const __m128i a2 = _mm_sub_epi16(tmp_1, tmp_3); 115 const __m128i a3 = _mm_sub_epi16(tmp_0, tmp_2); 116 const __m128i b0 = _mm_add_epi16(a0, a1); 117 const __m128i b1 = _mm_add_epi16(a3, a2); 118 const __m128i b2 = _mm_sub_epi16(a3, a2); 119 const __m128i b3 = _mm_sub_epi16(a0, a1); 120 // a00 a01 a02 a03 b00 b01 b02 b03 121 // a10 a11 a12 a13 b10 b11 b12 b13 122 // a20 a21 a22 a23 b20 b21 b22 b23 123 // a30 a31 a32 a33 b30 b31 b32 b33 124 125 // Transpose the two 4x4. 126 VP8Transpose_2_4x4_16b(&b0, &b1, &b2, &b3, &tmp_0, &tmp_1, &tmp_2, &tmp_3); 127 } 128 129 // Horizontal pass and difference of weighted sums. 130 { 131 // Load all inputs. 132 const __m128i w_0 = _mm_loadu_si128((const __m128i*)&w[0]); 133 const __m128i w_8 = _mm_loadu_si128((const __m128i*)&w[8]); 134 135 // Calculate a and b (two 4x4 at once). 136 const __m128i a0 = _mm_add_epi16(tmp_0, tmp_2); 137 const __m128i a1 = _mm_add_epi16(tmp_1, tmp_3); 138 const __m128i a2 = _mm_sub_epi16(tmp_1, tmp_3); 139 const __m128i a3 = _mm_sub_epi16(tmp_0, tmp_2); 140 const __m128i b0 = _mm_add_epi16(a0, a1); 141 const __m128i b1 = _mm_add_epi16(a3, a2); 142 const __m128i b2 = _mm_sub_epi16(a3, a2); 143 const __m128i b3 = _mm_sub_epi16(a0, a1); 144 145 // Separate the transforms of inA and inB. 146 __m128i A_b0 = _mm_unpacklo_epi64(b0, b1); 147 __m128i A_b2 = _mm_unpacklo_epi64(b2, b3); 148 __m128i B_b0 = _mm_unpackhi_epi64(b0, b1); 149 __m128i B_b2 = _mm_unpackhi_epi64(b2, b3); 150 151 A_b0 = _mm_abs_epi16(A_b0); 152 A_b2 = _mm_abs_epi16(A_b2); 153 B_b0 = _mm_abs_epi16(B_b0); 154 B_b2 = _mm_abs_epi16(B_b2); 155 156 // weighted sums 157 A_b0 = _mm_madd_epi16(A_b0, w_0); 158 A_b2 = _mm_madd_epi16(A_b2, w_8); 159 B_b0 = _mm_madd_epi16(B_b0, w_0); 160 B_b2 = _mm_madd_epi16(B_b2, w_8); 161 A_b0 = _mm_add_epi32(A_b0, A_b2); 162 B_b0 = _mm_add_epi32(B_b0, B_b2); 163 164 // difference of weighted sums 165 A_b2 = _mm_sub_epi32(A_b0, B_b0); 166 _mm_storeu_si128((__m128i*)&sum[0], A_b2); 167 } 168 return sum[0] + sum[1] + sum[2] + sum[3]; 169 } 170 171 static int Disto4x4_SSE41(const uint8_t* const a, const uint8_t* const b, 172 const uint16_t* const w) { 173 const int diff_sum = TTransform_SSE41(a, b, w); 174 return abs(diff_sum) >> 5; 175 } 176 177 static int Disto16x16_SSE41(const uint8_t* const a, const uint8_t* const b, 178 const uint16_t* const w) { 179 int D = 0; 180 int x, y; 181 for (y = 0; y < 16 * BPS; y += 4 * BPS) { 182 for (x = 0; x < 16; x += 4) { 183 D += Disto4x4_SSE41(a + x + y, b + x + y, w); 184 } 185 } 186 return D; 187 } 188 189 //------------------------------------------------------------------------------ 190 // Quantization 191 // 192 193 // Generates a pshufb constant for shuffling 16b words. 194 #define PSHUFB_CST(A,B,C,D,E,F,G,H) \ 195 _mm_set_epi8(2 * (H) + 1, 2 * (H) + 0, 2 * (G) + 1, 2 * (G) + 0, \ 196 2 * (F) + 1, 2 * (F) + 0, 2 * (E) + 1, 2 * (E) + 0, \ 197 2 * (D) + 1, 2 * (D) + 0, 2 * (C) + 1, 2 * (C) + 0, \ 198 2 * (B) + 1, 2 * (B) + 0, 2 * (A) + 1, 2 * (A) + 0) 199 200 static WEBP_INLINE int DoQuantizeBlock_SSE41(int16_t in[16], int16_t out[16], 201 const uint16_t* const sharpen, 202 const VP8Matrix* const mtx) { 203 const __m128i max_coeff_2047 = _mm_set1_epi16(MAX_LEVEL); 204 const __m128i zero = _mm_setzero_si128(); 205 __m128i out0, out8; 206 __m128i packed_out; 207 208 // Load all inputs. 209 __m128i in0 = _mm_loadu_si128((__m128i*)&in[0]); 210 __m128i in8 = _mm_loadu_si128((__m128i*)&in[8]); 211 const __m128i iq0 = _mm_loadu_si128((const __m128i*)&mtx->iq_[0]); 212 const __m128i iq8 = _mm_loadu_si128((const __m128i*)&mtx->iq_[8]); 213 const __m128i q0 = _mm_loadu_si128((const __m128i*)&mtx->q_[0]); 214 const __m128i q8 = _mm_loadu_si128((const __m128i*)&mtx->q_[8]); 215 216 // coeff = abs(in) 217 __m128i coeff0 = _mm_abs_epi16(in0); 218 __m128i coeff8 = _mm_abs_epi16(in8); 219 220 // coeff = abs(in) + sharpen 221 if (sharpen != NULL) { 222 const __m128i sharpen0 = _mm_loadu_si128((const __m128i*)&sharpen[0]); 223 const __m128i sharpen8 = _mm_loadu_si128((const __m128i*)&sharpen[8]); 224 coeff0 = _mm_add_epi16(coeff0, sharpen0); 225 coeff8 = _mm_add_epi16(coeff8, sharpen8); 226 } 227 228 // out = (coeff * iQ + B) >> QFIX 229 { 230 // doing calculations with 32b precision (QFIX=17) 231 // out = (coeff * iQ) 232 const __m128i coeff_iQ0H = _mm_mulhi_epu16(coeff0, iq0); 233 const __m128i coeff_iQ0L = _mm_mullo_epi16(coeff0, iq0); 234 const __m128i coeff_iQ8H = _mm_mulhi_epu16(coeff8, iq8); 235 const __m128i coeff_iQ8L = _mm_mullo_epi16(coeff8, iq8); 236 __m128i out_00 = _mm_unpacklo_epi16(coeff_iQ0L, coeff_iQ0H); 237 __m128i out_04 = _mm_unpackhi_epi16(coeff_iQ0L, coeff_iQ0H); 238 __m128i out_08 = _mm_unpacklo_epi16(coeff_iQ8L, coeff_iQ8H); 239 __m128i out_12 = _mm_unpackhi_epi16(coeff_iQ8L, coeff_iQ8H); 240 // out = (coeff * iQ + B) 241 const __m128i bias_00 = _mm_loadu_si128((const __m128i*)&mtx->bias_[0]); 242 const __m128i bias_04 = _mm_loadu_si128((const __m128i*)&mtx->bias_[4]); 243 const __m128i bias_08 = _mm_loadu_si128((const __m128i*)&mtx->bias_[8]); 244 const __m128i bias_12 = _mm_loadu_si128((const __m128i*)&mtx->bias_[12]); 245 out_00 = _mm_add_epi32(out_00, bias_00); 246 out_04 = _mm_add_epi32(out_04, bias_04); 247 out_08 = _mm_add_epi32(out_08, bias_08); 248 out_12 = _mm_add_epi32(out_12, bias_12); 249 // out = QUANTDIV(coeff, iQ, B, QFIX) 250 out_00 = _mm_srai_epi32(out_00, QFIX); 251 out_04 = _mm_srai_epi32(out_04, QFIX); 252 out_08 = _mm_srai_epi32(out_08, QFIX); 253 out_12 = _mm_srai_epi32(out_12, QFIX); 254 255 // pack result as 16b 256 out0 = _mm_packs_epi32(out_00, out_04); 257 out8 = _mm_packs_epi32(out_08, out_12); 258 259 // if (coeff > 2047) coeff = 2047 260 out0 = _mm_min_epi16(out0, max_coeff_2047); 261 out8 = _mm_min_epi16(out8, max_coeff_2047); 262 } 263 264 // put sign back 265 out0 = _mm_sign_epi16(out0, in0); 266 out8 = _mm_sign_epi16(out8, in8); 267 268 // in = out * Q 269 in0 = _mm_mullo_epi16(out0, q0); 270 in8 = _mm_mullo_epi16(out8, q8); 271 272 _mm_storeu_si128((__m128i*)&in[0], in0); 273 _mm_storeu_si128((__m128i*)&in[8], in8); 274 275 // zigzag the output before storing it. The re-ordering is: 276 // 0 1 2 3 4 5 6 7 | 8 9 10 11 12 13 14 15 277 // -> 0 1 4[8]5 2 3 6 | 9 12 13 10 [7]11 14 15 278 // There's only two misplaced entries ([8] and [7]) that are crossing the 279 // reg's boundaries. 280 // We use pshufb instead of pshuflo/pshufhi. 281 { 282 const __m128i kCst_lo = PSHUFB_CST(0, 1, 4, -1, 5, 2, 3, 6); 283 const __m128i kCst_7 = PSHUFB_CST(-1, -1, -1, -1, 7, -1, -1, -1); 284 const __m128i tmp_lo = _mm_shuffle_epi8(out0, kCst_lo); 285 const __m128i tmp_7 = _mm_shuffle_epi8(out0, kCst_7); // extract #7 286 const __m128i kCst_hi = PSHUFB_CST(1, 4, 5, 2, -1, 3, 6, 7); 287 const __m128i kCst_8 = PSHUFB_CST(-1, -1, -1, 0, -1, -1, -1, -1); 288 const __m128i tmp_hi = _mm_shuffle_epi8(out8, kCst_hi); 289 const __m128i tmp_8 = _mm_shuffle_epi8(out8, kCst_8); // extract #8 290 const __m128i out_z0 = _mm_or_si128(tmp_lo, tmp_8); 291 const __m128i out_z8 = _mm_or_si128(tmp_hi, tmp_7); 292 _mm_storeu_si128((__m128i*)&out[0], out_z0); 293 _mm_storeu_si128((__m128i*)&out[8], out_z8); 294 packed_out = _mm_packs_epi16(out_z0, out_z8); 295 } 296 297 // detect if all 'out' values are zeroes or not 298 return (_mm_movemask_epi8(_mm_cmpeq_epi8(packed_out, zero)) != 0xffff); 299 } 300 301 #undef PSHUFB_CST 302 303 static int QuantizeBlock_SSE41(int16_t in[16], int16_t out[16], 304 const VP8Matrix* const mtx) { 305 return DoQuantizeBlock_SSE41(in, out, &mtx->sharpen_[0], mtx); 306 } 307 308 static int QuantizeBlockWHT_SSE41(int16_t in[16], int16_t out[16], 309 const VP8Matrix* const mtx) { 310 return DoQuantizeBlock_SSE41(in, out, NULL, mtx); 311 } 312 313 static int Quantize2Blocks_SSE41(int16_t in[32], int16_t out[32], 314 const VP8Matrix* const mtx) { 315 int nz; 316 const uint16_t* const sharpen = &mtx->sharpen_[0]; 317 nz = DoQuantizeBlock_SSE41(in + 0 * 16, out + 0 * 16, sharpen, mtx) << 0; 318 nz |= DoQuantizeBlock_SSE41(in + 1 * 16, out + 1 * 16, sharpen, mtx) << 1; 319 return nz; 320 } 321 322 //------------------------------------------------------------------------------ 323 // Entry point 324 325 extern void VP8EncDspInitSSE41(void); 326 WEBP_TSAN_IGNORE_FUNCTION void VP8EncDspInitSSE41(void) { 327 VP8CollectHistogram = CollectHistogram_SSE41; 328 VP8EncQuantizeBlock = QuantizeBlock_SSE41; 329 VP8EncQuantize2Blocks = Quantize2Blocks_SSE41; 330 VP8EncQuantizeBlockWHT = QuantizeBlockWHT_SSE41; 331 VP8TDisto4x4 = Disto4x4_SSE41; 332 VP8TDisto16x16 = Disto16x16_SSE41; 333 } 334 335 #else // !WEBP_USE_SSE41 336 337 WEBP_DSP_INIT_STUB(VP8EncDspInitSSE41) 338 339 #endif // WEBP_USE_SSE41 340
