1 /* 2 * Copyright (c) 2015 The WebM project authors. All Rights Reserved. 3 * 4 * Use of this source code is governed by a BSD-style license 5 * that can be found in the LICENSE file in the root of the source 6 * tree. An additional intellectual property rights grant can be found 7 * in the file PATENTS. All contributing project authors may 8 * be found in the AUTHORS file in the root of the source tree. 9 */ 10 11 #include "./vpx_dsp_rtcd.h" 12 #include "vpx_dsp/x86/highbd_inv_txfm_sse2.h" 13 #include "vpx_dsp/x86/inv_txfm_sse2.h" 14 #include "vpx_dsp/x86/transpose_sse2.h" 15 #include "vpx_dsp/x86/txfm_common_sse2.h" 16 17 static INLINE void highbd_idct32_4x32_quarter_2_stage_4_to_6( 18 __m128i *const step1 /*step1[16]*/, __m128i *const out /*out[16]*/) { 19 __m128i step2[32]; 20 21 // stage 4 22 step2[8] = step1[8]; 23 step2[15] = step1[15]; 24 highbd_butterfly_sse2(step1[14], step1[9], cospi_24_64, cospi_8_64, &step2[9], 25 &step2[14]); 26 highbd_butterfly_sse2(step1[10], step1[13], cospi_8_64, cospi_24_64, 27 &step2[13], &step2[10]); 28 step2[11] = step1[11]; 29 step2[12] = step1[12]; 30 31 // stage 5 32 step1[8] = _mm_add_epi32(step2[8], step2[11]); 33 step1[9] = _mm_add_epi32(step2[9], step2[10]); 34 step1[10] = _mm_sub_epi32(step2[9], step2[10]); 35 step1[11] = _mm_sub_epi32(step2[8], step2[11]); 36 step1[12] = _mm_sub_epi32(step2[15], step2[12]); 37 step1[13] = _mm_sub_epi32(step2[14], step2[13]); 38 step1[14] = _mm_add_epi32(step2[14], step2[13]); 39 step1[15] = _mm_add_epi32(step2[15], step2[12]); 40 41 // stage 6 42 out[8] = step1[8]; 43 out[9] = step1[9]; 44 highbd_butterfly_sse2(step1[13], step1[10], cospi_16_64, cospi_16_64, 45 &out[10], &out[13]); 46 highbd_butterfly_sse2(step1[12], step1[11], cospi_16_64, cospi_16_64, 47 &out[11], &out[12]); 48 out[14] = step1[14]; 49 out[15] = step1[15]; 50 } 51 52 static INLINE void highbd_idct32_4x32_quarter_3_4_stage_4_to_7( 53 __m128i *const step1 /*step1[32]*/, __m128i *const out /*out[32]*/) { 54 __m128i step2[32]; 55 56 // stage 4 57 step2[16] = _mm_add_epi32(step1[16], step1[19]); 58 step2[17] = _mm_add_epi32(step1[17], step1[18]); 59 step2[18] = _mm_sub_epi32(step1[17], step1[18]); 60 step2[19] = _mm_sub_epi32(step1[16], step1[19]); 61 step2[20] = _mm_sub_epi32(step1[20], step1[23]); // step2[20] = -step2[20] 62 step2[21] = _mm_sub_epi32(step1[21], step1[22]); // step2[21] = -step2[21] 63 step2[22] = _mm_add_epi32(step1[21], step1[22]); 64 step2[23] = _mm_add_epi32(step1[20], step1[23]); 65 66 step2[24] = _mm_add_epi32(step1[27], step1[24]); 67 step2[25] = _mm_add_epi32(step1[26], step1[25]); 68 step2[26] = _mm_sub_epi32(step1[26], step1[25]); // step2[26] = -step2[26] 69 step2[27] = _mm_sub_epi32(step1[27], step1[24]); // step2[27] = -step2[27] 70 step2[28] = _mm_sub_epi32(step1[31], step1[28]); 71 step2[29] = _mm_sub_epi32(step1[30], step1[29]); 72 step2[30] = _mm_add_epi32(step1[29], step1[30]); 73 step2[31] = _mm_add_epi32(step1[28], step1[31]); 74 75 // stage 5 76 step1[16] = step2[16]; 77 step1[17] = step2[17]; 78 highbd_butterfly_sse2(step2[29], step2[18], cospi_24_64, cospi_8_64, 79 &step1[18], &step1[29]); 80 highbd_butterfly_sse2(step2[28], step2[19], cospi_24_64, cospi_8_64, 81 &step1[19], &step1[28]); 82 highbd_butterfly_sse2(step2[20], step2[27], cospi_8_64, cospi_24_64, 83 &step1[27], &step1[20]); 84 highbd_butterfly_sse2(step2[21], step2[26], cospi_8_64, cospi_24_64, 85 &step1[26], &step1[21]); 86 step1[22] = step2[22]; 87 step1[23] = step2[23]; 88 step1[24] = step2[24]; 89 step1[25] = step2[25]; 90 step1[30] = step2[30]; 91 step1[31] = step2[31]; 92 93 // stage 6 94 step2[16] = _mm_add_epi32(step1[16], step1[23]); 95 step2[17] = _mm_add_epi32(step1[17], step1[22]); 96 step2[18] = _mm_add_epi32(step1[18], step1[21]); 97 step2[19] = _mm_add_epi32(step1[19], step1[20]); 98 step2[20] = _mm_sub_epi32(step1[19], step1[20]); 99 step2[21] = _mm_sub_epi32(step1[18], step1[21]); 100 step2[22] = _mm_sub_epi32(step1[17], step1[22]); 101 step2[23] = _mm_sub_epi32(step1[16], step1[23]); 102 103 step2[24] = _mm_sub_epi32(step1[31], step1[24]); 104 step2[25] = _mm_sub_epi32(step1[30], step1[25]); 105 step2[26] = _mm_sub_epi32(step1[29], step1[26]); 106 step2[27] = _mm_sub_epi32(step1[28], step1[27]); 107 step2[28] = _mm_add_epi32(step1[27], step1[28]); 108 step2[29] = _mm_add_epi32(step1[26], step1[29]); 109 step2[30] = _mm_add_epi32(step1[25], step1[30]); 110 step2[31] = _mm_add_epi32(step1[24], step1[31]); 111 112 // stage 7 113 out[16] = step2[16]; 114 out[17] = step2[17]; 115 out[18] = step2[18]; 116 out[19] = step2[19]; 117 highbd_butterfly_sse2(step2[27], step2[20], cospi_16_64, cospi_16_64, 118 &out[20], &out[27]); 119 highbd_butterfly_sse2(step2[26], step2[21], cospi_16_64, cospi_16_64, 120 &out[21], &out[26]); 121 highbd_butterfly_sse2(step2[25], step2[22], cospi_16_64, cospi_16_64, 122 &out[22], &out[25]); 123 highbd_butterfly_sse2(step2[24], step2[23], cospi_16_64, cospi_16_64, 124 &out[23], &out[24]); 125 out[28] = step2[28]; 126 out[29] = step2[29]; 127 out[30] = step2[30]; 128 out[31] = step2[31]; 129 } 130 131 // Group the coefficient calculation into smaller functions to prevent stack 132 // spillover in 32x32 idct optimizations: 133 // quarter_1: 0-7 134 // quarter_2: 8-15 135 // quarter_3_4: 16-23, 24-31 136 137 // For each 4x32 block __m128i in[32], 138 // Input with index, 0, 4, 8, 12, 16, 20, 24, 28 139 // output pixels: 0-7 in __m128i out[32] 140 static INLINE void highbd_idct32_1024_4x32_quarter_1( 141 const __m128i *const in /*in[32]*/, __m128i *const out /*out[8]*/) { 142 __m128i step1[8], step2[8]; 143 144 // stage 3 145 highbd_butterfly_sse2(in[4], in[28], cospi_28_64, cospi_4_64, &step1[4], 146 &step1[7]); 147 highbd_butterfly_sse2(in[20], in[12], cospi_12_64, cospi_20_64, &step1[5], 148 &step1[6]); 149 150 // stage 4 151 highbd_butterfly_sse2(in[0], in[16], cospi_16_64, cospi_16_64, &step2[1], 152 &step2[0]); 153 highbd_butterfly_sse2(in[8], in[24], cospi_24_64, cospi_8_64, &step2[2], 154 &step2[3]); 155 step2[4] = _mm_add_epi32(step1[4], step1[5]); 156 step2[5] = _mm_sub_epi32(step1[4], step1[5]); 157 step2[6] = _mm_sub_epi32(step1[7], step1[6]); 158 step2[7] = _mm_add_epi32(step1[7], step1[6]); 159 160 // stage 5 161 step1[0] = _mm_add_epi32(step2[0], step2[3]); 162 step1[1] = _mm_add_epi32(step2[1], step2[2]); 163 step1[2] = _mm_sub_epi32(step2[1], step2[2]); 164 step1[3] = _mm_sub_epi32(step2[0], step2[3]); 165 step1[4] = step2[4]; 166 highbd_butterfly_sse2(step2[6], step2[5], cospi_16_64, cospi_16_64, &step1[5], 167 &step1[6]); 168 step1[7] = step2[7]; 169 170 // stage 6 171 out[0] = _mm_add_epi32(step1[0], step1[7]); 172 out[1] = _mm_add_epi32(step1[1], step1[6]); 173 out[2] = _mm_add_epi32(step1[2], step1[5]); 174 out[3] = _mm_add_epi32(step1[3], step1[4]); 175 out[4] = _mm_sub_epi32(step1[3], step1[4]); 176 out[5] = _mm_sub_epi32(step1[2], step1[5]); 177 out[6] = _mm_sub_epi32(step1[1], step1[6]); 178 out[7] = _mm_sub_epi32(step1[0], step1[7]); 179 } 180 181 // For each 4x32 block __m128i in[32], 182 // Input with index, 2, 6, 10, 14, 18, 22, 26, 30 183 // output pixels: 8-15 in __m128i out[32] 184 static INLINE void highbd_idct32_1024_4x32_quarter_2( 185 const __m128i *in /*in[32]*/, __m128i *out /*out[16]*/) { 186 __m128i step1[32], step2[32]; 187 188 // stage 2 189 highbd_butterfly_sse2(in[2], in[30], cospi_30_64, cospi_2_64, &step2[8], 190 &step2[15]); 191 highbd_butterfly_sse2(in[18], in[14], cospi_14_64, cospi_18_64, &step2[9], 192 &step2[14]); 193 highbd_butterfly_sse2(in[10], in[22], cospi_22_64, cospi_10_64, &step2[10], 194 &step2[13]); 195 highbd_butterfly_sse2(in[26], in[6], cospi_6_64, cospi_26_64, &step2[11], 196 &step2[12]); 197 198 // stage 3 199 step1[8] = _mm_add_epi32(step2[8], step2[9]); 200 step1[9] = _mm_sub_epi32(step2[8], step2[9]); 201 step1[14] = _mm_sub_epi32(step2[15], step2[14]); 202 step1[15] = _mm_add_epi32(step2[15], step2[14]); 203 step1[10] = _mm_sub_epi32(step2[10], step2[11]); // step1[10] = -step1[10] 204 step1[11] = _mm_add_epi32(step2[10], step2[11]); 205 step1[12] = _mm_add_epi32(step2[13], step2[12]); 206 step1[13] = _mm_sub_epi32(step2[13], step2[12]); // step1[13] = -step1[13] 207 208 highbd_idct32_4x32_quarter_2_stage_4_to_6(step1, out); 209 } 210 211 static INLINE void highbd_idct32_1024_4x32_quarter_1_2( 212 const __m128i *const in /*in[32]*/, __m128i *const out /*out[32]*/) { 213 __m128i temp[16]; 214 highbd_idct32_1024_4x32_quarter_1(in, temp); 215 highbd_idct32_1024_4x32_quarter_2(in, temp); 216 // stage 7 217 highbd_add_sub_butterfly(temp, out, 16); 218 } 219 220 // For each 4x32 block __m128i in[32], 221 // Input with odd index, 222 // 1, 3, 5, 7, 9, 11, 13, 15, 17, 19, 21, 23, 25, 27, 29, 31 223 // output pixels: 16-23, 24-31 in __m128i out[32] 224 static INLINE void highbd_idct32_1024_4x32_quarter_3_4( 225 const __m128i *const in /*in[32]*/, __m128i *const out /*out[32]*/) { 226 __m128i step1[32], step2[32]; 227 228 // stage 1 229 highbd_butterfly_sse2(in[1], in[31], cospi_31_64, cospi_1_64, &step1[16], 230 &step1[31]); 231 highbd_butterfly_sse2(in[17], in[15], cospi_15_64, cospi_17_64, &step1[17], 232 &step1[30]); 233 highbd_butterfly_sse2(in[9], in[23], cospi_23_64, cospi_9_64, &step1[18], 234 &step1[29]); 235 highbd_butterfly_sse2(in[25], in[7], cospi_7_64, cospi_25_64, &step1[19], 236 &step1[28]); 237 238 highbd_butterfly_sse2(in[5], in[27], cospi_27_64, cospi_5_64, &step1[20], 239 &step1[27]); 240 highbd_butterfly_sse2(in[21], in[11], cospi_11_64, cospi_21_64, &step1[21], 241 &step1[26]); 242 243 highbd_butterfly_sse2(in[13], in[19], cospi_19_64, cospi_13_64, &step1[22], 244 &step1[25]); 245 highbd_butterfly_sse2(in[29], in[3], cospi_3_64, cospi_29_64, &step1[23], 246 &step1[24]); 247 248 // stage 2 249 step2[16] = _mm_add_epi32(step1[16], step1[17]); 250 step2[17] = _mm_sub_epi32(step1[16], step1[17]); 251 step2[18] = _mm_sub_epi32(step1[18], step1[19]); // step2[18] = -step2[18] 252 step2[19] = _mm_add_epi32(step1[18], step1[19]); 253 step2[20] = _mm_add_epi32(step1[20], step1[21]); 254 step2[21] = _mm_sub_epi32(step1[20], step1[21]); 255 step2[22] = _mm_sub_epi32(step1[22], step1[23]); // step2[22] = -step2[22] 256 step2[23] = _mm_add_epi32(step1[22], step1[23]); 257 258 step2[24] = _mm_add_epi32(step1[25], step1[24]); 259 step2[25] = _mm_sub_epi32(step1[25], step1[24]); // step2[25] = -step2[25] 260 step2[26] = _mm_sub_epi32(step1[27], step1[26]); 261 step2[27] = _mm_add_epi32(step1[27], step1[26]); 262 step2[28] = _mm_add_epi32(step1[29], step1[28]); 263 step2[29] = _mm_sub_epi32(step1[29], step1[28]); // step2[29] = -step2[29] 264 step2[30] = _mm_sub_epi32(step1[31], step1[30]); 265 step2[31] = _mm_add_epi32(step1[31], step1[30]); 266 267 // stage 3 268 step1[16] = step2[16]; 269 step1[31] = step2[31]; 270 highbd_butterfly_sse2(step2[30], step2[17], cospi_28_64, cospi_4_64, 271 &step1[17], &step1[30]); 272 highbd_butterfly_sse2(step2[18], step2[29], cospi_4_64, cospi_28_64, 273 &step1[29], &step1[18]); 274 step1[19] = step2[19]; 275 step1[20] = step2[20]; 276 highbd_butterfly_sse2(step2[26], step2[21], cospi_12_64, cospi_20_64, 277 &step1[21], &step1[26]); 278 highbd_butterfly_sse2(step2[22], step2[25], cospi_20_64, cospi_12_64, 279 &step1[25], &step1[22]); 280 step1[23] = step2[23]; 281 step1[24] = step2[24]; 282 step1[27] = step2[27]; 283 step1[28] = step2[28]; 284 285 highbd_idct32_4x32_quarter_3_4_stage_4_to_7(step1, out); 286 } 287 288 static void highbd_idct32_1024_4x32(__m128i *const io /*io[32]*/) { 289 __m128i temp[32]; 290 291 highbd_idct32_1024_4x32_quarter_1_2(io, temp); 292 highbd_idct32_1024_4x32_quarter_3_4(io, temp); 293 // final stage 294 highbd_add_sub_butterfly(temp, io, 32); 295 } 296 297 void vpx_highbd_idct32x32_1024_add_sse2(const tran_low_t *input, uint16_t *dest, 298 int stride, int bd) { 299 int i, j; 300 301 if (bd == 8) { 302 __m128i col[4][32], io[32]; 303 304 // rows 305 for (i = 0; i < 4; i++) { 306 highbd_load_pack_transpose_32bit_8x8(&input[0], 32, &io[0]); 307 highbd_load_pack_transpose_32bit_8x8(&input[8], 32, &io[8]); 308 highbd_load_pack_transpose_32bit_8x8(&input[16], 32, &io[16]); 309 highbd_load_pack_transpose_32bit_8x8(&input[24], 32, &io[24]); 310 idct32_1024_8x32(io, col[i]); 311 input += 32 << 3; 312 } 313 314 // columns 315 for (i = 0; i < 32; i += 8) { 316 // Transpose 32x8 block to 8x32 block 317 transpose_16bit_8x8(col[0] + i, io); 318 transpose_16bit_8x8(col[1] + i, io + 8); 319 transpose_16bit_8x8(col[2] + i, io + 16); 320 transpose_16bit_8x8(col[3] + i, io + 24); 321 idct32_1024_8x32(io, io); 322 for (j = 0; j < 32; ++j) { 323 highbd_write_buffer_8(dest + j * stride, io[j], bd); 324 } 325 dest += 8; 326 } 327 } else { 328 __m128i all[8][32], out[32], *in; 329 330 for (i = 0; i < 8; i++) { 331 in = all[i]; 332 highbd_load_transpose_32bit_8x4(&input[0], 32, &in[0]); 333 highbd_load_transpose_32bit_8x4(&input[8], 32, &in[8]); 334 highbd_load_transpose_32bit_8x4(&input[16], 32, &in[16]); 335 highbd_load_transpose_32bit_8x4(&input[24], 32, &in[24]); 336 highbd_idct32_1024_4x32(in); 337 input += 4 * 32; 338 } 339 340 for (i = 0; i < 32; i += 4) { 341 transpose_32bit_4x4(all[0] + i, out + 0); 342 transpose_32bit_4x4(all[1] + i, out + 4); 343 transpose_32bit_4x4(all[2] + i, out + 8); 344 transpose_32bit_4x4(all[3] + i, out + 12); 345 transpose_32bit_4x4(all[4] + i, out + 16); 346 transpose_32bit_4x4(all[5] + i, out + 20); 347 transpose_32bit_4x4(all[6] + i, out + 24); 348 transpose_32bit_4x4(all[7] + i, out + 28); 349 highbd_idct32_1024_4x32(out); 350 351 for (j = 0; j < 32; ++j) { 352 highbd_write_buffer_4(dest + j * stride, out[j], bd); 353 } 354 dest += 4; 355 } 356 } 357 } 358 359 // ----------------------------------------------------------------------------- 360 361 // For each 4x32 block __m128i in[32], 362 // Input with index, 0, 4, 8, 12 363 // output pixels: 0-7 in __m128i out[32] 364 static INLINE void highbd_idct32_135_4x32_quarter_1( 365 const __m128i *const in /*in[32]*/, __m128i *const out /*out[8]*/) { 366 __m128i step1[8], step2[8]; 367 368 // stage 3 369 highbd_partial_butterfly_sse2(in[4], cospi_28_64, cospi_4_64, &step1[4], 370 &step1[7]); 371 highbd_partial_butterfly_neg_sse2(in[12], cospi_12_64, cospi_20_64, &step1[5], 372 &step1[6]); 373 374 // stage 4 375 highbd_partial_butterfly_sse2(in[0], cospi_16_64, cospi_16_64, &step2[1], 376 &step2[0]); 377 highbd_partial_butterfly_sse2(in[8], cospi_24_64, cospi_8_64, &step2[2], 378 &step2[3]); 379 step2[4] = _mm_add_epi32(step1[4], step1[5]); 380 step2[5] = _mm_sub_epi32(step1[4], step1[5]); 381 step2[6] = _mm_sub_epi32(step1[7], step1[6]); 382 step2[7] = _mm_add_epi32(step1[7], step1[6]); 383 384 // stage 5 385 step1[0] = _mm_add_epi32(step2[0], step2[3]); 386 step1[1] = _mm_add_epi32(step2[1], step2[2]); 387 step1[2] = _mm_sub_epi32(step2[1], step2[2]); 388 step1[3] = _mm_sub_epi32(step2[0], step2[3]); 389 step1[4] = step2[4]; 390 highbd_butterfly_sse2(step2[6], step2[5], cospi_16_64, cospi_16_64, &step1[5], 391 &step1[6]); 392 step1[7] = step2[7]; 393 394 // stage 6 395 out[0] = _mm_add_epi32(step1[0], step1[7]); 396 out[1] = _mm_add_epi32(step1[1], step1[6]); 397 out[2] = _mm_add_epi32(step1[2], step1[5]); 398 out[3] = _mm_add_epi32(step1[3], step1[4]); 399 out[4] = _mm_sub_epi32(step1[3], step1[4]); 400 out[5] = _mm_sub_epi32(step1[2], step1[5]); 401 out[6] = _mm_sub_epi32(step1[1], step1[6]); 402 out[7] = _mm_sub_epi32(step1[0], step1[7]); 403 } 404 405 // For each 4x32 block __m128i in[32], 406 // Input with index, 2, 6, 10, 14 407 // output pixels: 8-15 in __m128i out[32] 408 static INLINE void highbd_idct32_135_4x32_quarter_2( 409 const __m128i *in /*in[32]*/, __m128i *out /*out[16]*/) { 410 __m128i step1[32], step2[32]; 411 412 // stage 2 413 highbd_partial_butterfly_sse2(in[2], cospi_30_64, cospi_2_64, &step2[8], 414 &step2[15]); 415 highbd_partial_butterfly_neg_sse2(in[14], cospi_14_64, cospi_18_64, &step2[9], 416 &step2[14]); 417 highbd_partial_butterfly_sse2(in[10], cospi_22_64, cospi_10_64, &step2[10], 418 &step2[13]); 419 highbd_partial_butterfly_neg_sse2(in[6], cospi_6_64, cospi_26_64, &step2[11], 420 &step2[12]); 421 422 // stage 3 423 step1[8] = _mm_add_epi32(step2[8], step2[9]); 424 step1[9] = _mm_sub_epi32(step2[8], step2[9]); 425 step1[14] = _mm_sub_epi32(step2[15], step2[14]); 426 step1[15] = _mm_add_epi32(step2[15], step2[14]); 427 step1[10] = _mm_sub_epi32(step2[10], step2[11]); // step1[10] = -step1[10] 428 step1[11] = _mm_add_epi32(step2[10], step2[11]); 429 step1[12] = _mm_add_epi32(step2[13], step2[12]); 430 step1[13] = _mm_sub_epi32(step2[13], step2[12]); // step1[13] = -step1[13] 431 432 highbd_idct32_4x32_quarter_2_stage_4_to_6(step1, out); 433 } 434 435 static INLINE void highbd_idct32_135_4x32_quarter_1_2( 436 const __m128i *const in /*in[32]*/, __m128i *const out /*out[32]*/) { 437 __m128i temp[16]; 438 highbd_idct32_135_4x32_quarter_1(in, temp); 439 highbd_idct32_135_4x32_quarter_2(in, temp); 440 // stage 7 441 highbd_add_sub_butterfly(temp, out, 16); 442 } 443 444 // For each 4x32 block __m128i in[32], 445 // Input with odd index, 446 // 1, 3, 5, 7, 9, 11, 13, 15 447 // output pixels: 16-23, 24-31 in __m128i out[32] 448 static INLINE void highbd_idct32_135_4x32_quarter_3_4( 449 const __m128i *const in /*in[32]*/, __m128i *const out /*out[32]*/) { 450 __m128i step1[32], step2[32]; 451 452 // stage 1 453 highbd_partial_butterfly_sse2(in[1], cospi_31_64, cospi_1_64, &step1[16], 454 &step1[31]); 455 highbd_partial_butterfly_neg_sse2(in[15], cospi_15_64, cospi_17_64, 456 &step1[17], &step1[30]); 457 highbd_partial_butterfly_sse2(in[9], cospi_23_64, cospi_9_64, &step1[18], 458 &step1[29]); 459 highbd_partial_butterfly_neg_sse2(in[7], cospi_7_64, cospi_25_64, &step1[19], 460 &step1[28]); 461 462 highbd_partial_butterfly_sse2(in[5], cospi_27_64, cospi_5_64, &step1[20], 463 &step1[27]); 464 highbd_partial_butterfly_neg_sse2(in[11], cospi_11_64, cospi_21_64, 465 &step1[21], &step1[26]); 466 467 highbd_partial_butterfly_sse2(in[13], cospi_19_64, cospi_13_64, &step1[22], 468 &step1[25]); 469 highbd_partial_butterfly_neg_sse2(in[3], cospi_3_64, cospi_29_64, &step1[23], 470 &step1[24]); 471 472 // stage 2 473 step2[16] = _mm_add_epi32(step1[16], step1[17]); 474 step2[17] = _mm_sub_epi32(step1[16], step1[17]); 475 step2[18] = _mm_sub_epi32(step1[18], step1[19]); // step2[18] = -step2[18] 476 step2[19] = _mm_add_epi32(step1[18], step1[19]); 477 step2[20] = _mm_add_epi32(step1[20], step1[21]); 478 step2[21] = _mm_sub_epi32(step1[20], step1[21]); 479 step2[22] = _mm_sub_epi32(step1[22], step1[23]); // step2[22] = -step2[22] 480 step2[23] = _mm_add_epi32(step1[22], step1[23]); 481 482 step2[24] = _mm_add_epi32(step1[25], step1[24]); 483 step2[25] = _mm_sub_epi32(step1[25], step1[24]); // step2[25] = -step2[25] 484 step2[26] = _mm_sub_epi32(step1[27], step1[26]); 485 step2[27] = _mm_add_epi32(step1[27], step1[26]); 486 step2[28] = _mm_add_epi32(step1[29], step1[28]); 487 step2[29] = _mm_sub_epi32(step1[29], step1[28]); // step2[29] = -step2[29] 488 step2[30] = _mm_sub_epi32(step1[31], step1[30]); 489 step2[31] = _mm_add_epi32(step1[31], step1[30]); 490 491 // stage 3 492 step1[16] = step2[16]; 493 step1[31] = step2[31]; 494 highbd_butterfly_sse2(step2[30], step2[17], cospi_28_64, cospi_4_64, 495 &step1[17], &step1[30]); 496 highbd_butterfly_sse2(step2[18], step2[29], cospi_4_64, cospi_28_64, 497 &step1[29], &step1[18]); 498 step1[19] = step2[19]; 499 step1[20] = step2[20]; 500 highbd_butterfly_sse2(step2[26], step2[21], cospi_12_64, cospi_20_64, 501 &step1[21], &step1[26]); 502 highbd_butterfly_sse2(step2[22], step2[25], cospi_20_64, cospi_12_64, 503 &step1[25], &step1[22]); 504 step1[23] = step2[23]; 505 step1[24] = step2[24]; 506 step1[27] = step2[27]; 507 step1[28] = step2[28]; 508 509 highbd_idct32_4x32_quarter_3_4_stage_4_to_7(step1, out); 510 } 511 512 static void highbd_idct32_135_4x32(__m128i *const io /*io[32]*/) { 513 __m128i temp[32]; 514 515 highbd_idct32_135_4x32_quarter_1_2(io, temp); 516 highbd_idct32_135_4x32_quarter_3_4(io, temp); 517 // final stage 518 highbd_add_sub_butterfly(temp, io, 32); 519 } 520 521 void vpx_highbd_idct32x32_135_add_sse2(const tran_low_t *input, uint16_t *dest, 522 int stride, int bd) { 523 int i, j; 524 525 if (bd == 8) { 526 __m128i col[2][32], in[32], out[32]; 527 528 for (i = 16; i < 32; i++) { 529 in[i] = _mm_setzero_si128(); 530 } 531 532 // rows 533 for (i = 0; i < 2; i++) { 534 highbd_load_pack_transpose_32bit_8x8(&input[0], 32, &in[0]); 535 highbd_load_pack_transpose_32bit_8x8(&input[8], 32, &in[8]); 536 idct32_1024_8x32(in, col[i]); 537 input += 32 << 3; 538 } 539 540 // columns 541 for (i = 0; i < 32; i += 8) { 542 transpose_16bit_8x8(col[0] + i, in); 543 transpose_16bit_8x8(col[1] + i, in + 8); 544 idct32_1024_8x32(in, out); 545 for (j = 0; j < 32; ++j) { 546 highbd_write_buffer_8(dest + j * stride, out[j], bd); 547 } 548 dest += 8; 549 } 550 } else { 551 __m128i all[8][32], out[32], *in; 552 553 for (i = 0; i < 4; i++) { 554 in = all[i]; 555 highbd_load_transpose_32bit_8x4(&input[0], 32, &in[0]); 556 highbd_load_transpose_32bit_8x4(&input[8], 32, &in[8]); 557 highbd_idct32_135_4x32(in); 558 input += 4 * 32; 559 } 560 561 for (i = 0; i < 32; i += 4) { 562 transpose_32bit_4x4(all[0] + i, out + 0); 563 transpose_32bit_4x4(all[1] + i, out + 4); 564 transpose_32bit_4x4(all[2] + i, out + 8); 565 transpose_32bit_4x4(all[3] + i, out + 12); 566 highbd_idct32_135_4x32(out); 567 568 for (j = 0; j < 32; ++j) { 569 highbd_write_buffer_4(dest + j * stride, out[j], bd); 570 } 571 dest += 4; 572 } 573 } 574 } 575 576 // ----------------------------------------------------------------------------- 577 578 // For each 4x32 block __m128i in[32], 579 // Input with index, 0, 4 580 // output pixels: 0-7 in __m128i out[32] 581 static INLINE void highbd_idct32_34_4x32_quarter_1( 582 const __m128i *const in /*in[32]*/, __m128i *const out /*out[8]*/) { 583 __m128i step1[8], step2[8]; 584 585 // stage 3 586 highbd_partial_butterfly_sse2(in[4], cospi_28_64, cospi_4_64, &step1[4], 587 &step1[7]); 588 589 // stage 4 590 highbd_partial_butterfly_sse2(in[0], cospi_16_64, cospi_16_64, &step2[1], 591 &step2[0]); 592 step2[4] = step1[4]; 593 step2[5] = step1[4]; 594 step2[6] = step1[7]; 595 step2[7] = step1[7]; 596 597 // stage 5 598 step1[0] = step2[0]; 599 step1[1] = step2[1]; 600 step1[2] = step2[1]; 601 step1[3] = step2[0]; 602 step1[4] = step2[4]; 603 highbd_butterfly_sse2(step2[6], step2[5], cospi_16_64, cospi_16_64, &step1[5], 604 &step1[6]); 605 step1[7] = step2[7]; 606 607 // stage 6 608 out[0] = _mm_add_epi32(step1[0], step1[7]); 609 out[1] = _mm_add_epi32(step1[1], step1[6]); 610 out[2] = _mm_add_epi32(step1[2], step1[5]); 611 out[3] = _mm_add_epi32(step1[3], step1[4]); 612 out[4] = _mm_sub_epi32(step1[3], step1[4]); 613 out[5] = _mm_sub_epi32(step1[2], step1[5]); 614 out[6] = _mm_sub_epi32(step1[1], step1[6]); 615 out[7] = _mm_sub_epi32(step1[0], step1[7]); 616 } 617 618 // For each 4x32 block __m128i in[32], 619 // Input with index, 2, 6 620 // output pixels: 8-15 in __m128i out[32] 621 static INLINE void highbd_idct32_34_4x32_quarter_2(const __m128i *in /*in[32]*/, 622 __m128i *out /*out[16]*/) { 623 __m128i step1[32], step2[32]; 624 625 // stage 2 626 highbd_partial_butterfly_sse2(in[2], cospi_30_64, cospi_2_64, &step2[8], 627 &step2[15]); 628 highbd_partial_butterfly_neg_sse2(in[6], cospi_6_64, cospi_26_64, &step2[11], 629 &step2[12]); 630 631 // stage 3 632 step1[8] = step2[8]; 633 step1[9] = step2[8]; 634 step1[14] = step2[15]; 635 step1[15] = step2[15]; 636 step1[10] = step2[11]; 637 step1[11] = step2[11]; 638 step1[12] = step2[12]; 639 step1[13] = step2[12]; 640 641 step1[10] = 642 _mm_sub_epi32(_mm_setzero_si128(), step1[10]); // step1[10] = -step1[10] 643 step1[13] = 644 _mm_sub_epi32(_mm_setzero_si128(), step1[13]); // step1[13] = -step1[13] 645 highbd_idct32_4x32_quarter_2_stage_4_to_6(step1, out); 646 } 647 648 static INLINE void highbd_idct32_34_4x32_quarter_1_2( 649 const __m128i *const in /*in[32]*/, __m128i *const out /*out[32]*/) { 650 __m128i temp[16]; 651 highbd_idct32_34_4x32_quarter_1(in, temp); 652 highbd_idct32_34_4x32_quarter_2(in, temp); 653 // stage 7 654 highbd_add_sub_butterfly(temp, out, 16); 655 } 656 657 // For each 4x32 block __m128i in[32], 658 // Input with odd index, 659 // 1, 3, 5, 7 660 // output pixels: 16-23, 24-31 in __m128i out[32] 661 static INLINE void highbd_idct32_34_4x32_quarter_3_4( 662 const __m128i *const in /*in[32]*/, __m128i *const out /*out[32]*/) { 663 __m128i step1[32], step2[32]; 664 665 // stage 1 666 highbd_partial_butterfly_sse2(in[1], cospi_31_64, cospi_1_64, &step1[16], 667 &step1[31]); 668 highbd_partial_butterfly_neg_sse2(in[7], cospi_7_64, cospi_25_64, &step1[19], 669 &step1[28]); 670 671 highbd_partial_butterfly_sse2(in[5], cospi_27_64, cospi_5_64, &step1[20], 672 &step1[27]); 673 highbd_partial_butterfly_neg_sse2(in[3], cospi_3_64, cospi_29_64, &step1[23], 674 &step1[24]); 675 676 // stage 2 677 step2[16] = step1[16]; 678 step2[17] = step1[16]; 679 step2[18] = step1[19]; 680 step2[19] = step1[19]; 681 step2[20] = step1[20]; 682 step2[21] = step1[20]; 683 step2[22] = step1[23]; 684 step2[23] = step1[23]; 685 686 step2[24] = step1[24]; 687 step2[25] = step1[24]; 688 step2[26] = step1[27]; 689 step2[27] = step1[27]; 690 step2[28] = step1[28]; 691 step2[29] = step1[28]; 692 step2[30] = step1[31]; 693 step2[31] = step1[31]; 694 695 // stage 3 696 step2[18] = 697 _mm_sub_epi32(_mm_setzero_si128(), step2[18]); // step2[18] = -step2[18] 698 step2[22] = 699 _mm_sub_epi32(_mm_setzero_si128(), step2[22]); // step2[22] = -step2[22] 700 step2[25] = 701 _mm_sub_epi32(_mm_setzero_si128(), step2[25]); // step2[25] = -step2[25] 702 step2[29] = 703 _mm_sub_epi32(_mm_setzero_si128(), step2[29]); // step2[29] = -step2[29] 704 step1[16] = step2[16]; 705 step1[31] = step2[31]; 706 highbd_butterfly_sse2(step2[30], step2[17], cospi_28_64, cospi_4_64, 707 &step1[17], &step1[30]); 708 highbd_butterfly_sse2(step2[18], step2[29], cospi_4_64, cospi_28_64, 709 &step1[29], &step1[18]); 710 step1[19] = step2[19]; 711 step1[20] = step2[20]; 712 highbd_butterfly_sse2(step2[26], step2[21], cospi_12_64, cospi_20_64, 713 &step1[21], &step1[26]); 714 highbd_butterfly_sse2(step2[22], step2[25], cospi_20_64, cospi_12_64, 715 &step1[25], &step1[22]); 716 step1[23] = step2[23]; 717 step1[24] = step2[24]; 718 step1[27] = step2[27]; 719 step1[28] = step2[28]; 720 721 highbd_idct32_4x32_quarter_3_4_stage_4_to_7(step1, out); 722 } 723 724 static void highbd_idct32_34_4x32(__m128i *const io /*io[32]*/) { 725 __m128i temp[32]; 726 727 highbd_idct32_34_4x32_quarter_1_2(io, temp); 728 highbd_idct32_34_4x32_quarter_3_4(io, temp); 729 // final stage 730 highbd_add_sub_butterfly(temp, io, 32); 731 } 732 733 void vpx_highbd_idct32x32_34_add_sse2(const tran_low_t *input, uint16_t *dest, 734 int stride, int bd) { 735 int i, j; 736 737 if (bd == 8) { 738 __m128i col[32], in[32], out[32]; 739 740 // rows 741 highbd_load_pack_transpose_32bit_8x8(&input[0], 32, &in[0]); 742 idct32_34_8x32_sse2(in, col); 743 744 // columns 745 for (i = 0; i < 32; i += 8) { 746 transpose_16bit_8x8(col + i, in); 747 idct32_34_8x32_sse2(in, out); 748 for (j = 0; j < 32; ++j) { 749 highbd_write_buffer_8(dest + j * stride, out[j], bd); 750 } 751 dest += 8; 752 } 753 } else { 754 __m128i all[8][32], out[32], *in; 755 756 for (i = 0; i < 4; i++) { 757 in = all[i]; 758 highbd_load_transpose_32bit_8x4(&input[0], 32, &in[0]); 759 highbd_load_transpose_32bit_8x4(&input[8], 32, &in[8]); 760 highbd_idct32_34_4x32(in); 761 input += 4 * 32; 762 } 763 764 for (i = 0; i < 32; i += 4) { 765 transpose_32bit_4x4(all[0] + i, out + 0); 766 transpose_32bit_4x4(all[1] + i, out + 4); 767 transpose_32bit_4x4(all[2] + i, out + 8); 768 transpose_32bit_4x4(all[3] + i, out + 12); 769 highbd_idct32_34_4x32(out); 770 771 for (j = 0; j < 32; ++j) { 772 highbd_write_buffer_4(dest + j * stride, out[j], bd); 773 } 774 dest += 4; 775 } 776 } 777 } 778 779 void vpx_highbd_idct32x32_1_add_sse2(const tran_low_t *input, uint16_t *dest, 780 int stride, int bd) { 781 highbd_idct_1_add_kernel(input, dest, stride, bd, 32); 782 } 783
