1 /**************************************************************************** 2 * Copyright (C) 2014-2015 Intel Corporation. All Rights Reserved. 3 * 4 * Permission is hereby granted, free of charge, to any person obtaining a 5 * copy of this software and associated documentation files (the "Software"), 6 * to deal in the Software without restriction, including without limitation 7 * the rights to use, copy, modify, merge, publish, distribute, sublicense, 8 * and/or sell copies of the Software, and to permit persons to whom the 9 * Software is furnished to do so, subject to the following conditions: 10 * 11 * The above copyright notice and this permission notice (including the next 12 * paragraph) shall be included in all copies or substantial portions of the 13 * Software. 14 * 15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS 21 * IN THE SOFTWARE. 22 * 23 * @file utils.h 24 * 25 * @brief Utilities used by SWR core. 26 * 27 ******************************************************************************/ 28 #pragma once 29 30 #include <string.h> 31 #include <type_traits> 32 #include <algorithm> 33 #include "common/os.h" 34 #include "common/simdintrin.h" 35 #include "common/swr_assert.h" 36 #include "core/api.h" 37 38 #if defined(_WIN64) || defined(__x86_64__) 39 #define _MM_INSERT_EPI64 _mm_insert_epi64 40 #define _MM_EXTRACT_EPI64 _mm_extract_epi64 41 #else 42 INLINE int64_t _MM_EXTRACT_EPI64(__m128i a, const int32_t ndx) 43 { 44 OSALIGNLINE(uint32_t) elems[4]; 45 _mm_store_si128((__m128i*)elems, a); 46 if (ndx == 0) 47 { 48 uint64_t foo = elems[0]; 49 foo |= (uint64_t)elems[1] << 32; 50 return foo; 51 } 52 else 53 { 54 uint64_t foo = elems[2]; 55 foo |= (uint64_t)elems[3] << 32; 56 return foo; 57 } 58 } 59 60 INLINE __m128i _MM_INSERT_EPI64(__m128i a, int64_t b, const int32_t ndx) 61 { 62 OSALIGNLINE(int64_t) elems[2]; 63 _mm_store_si128((__m128i*)elems, a); 64 if (ndx == 0) 65 { 66 elems[0] = b; 67 } 68 else 69 { 70 elems[1] = b; 71 } 72 __m128i out; 73 out = _mm_load_si128((const __m128i*)elems); 74 return out; 75 } 76 #endif 77 78 struct simdBBox 79 { 80 simdscalari ymin; 81 simdscalari ymax; 82 simdscalari xmin; 83 simdscalari xmax; 84 }; 85 86 INLINE 87 void vTranspose(__m128 &row0, __m128 &row1, __m128 &row2, __m128 &row3) 88 { 89 __m128i row0i = _mm_castps_si128(row0); 90 __m128i row1i = _mm_castps_si128(row1); 91 __m128i row2i = _mm_castps_si128(row2); 92 __m128i row3i = _mm_castps_si128(row3); 93 94 __m128i vTemp = row2i; 95 row2i = _mm_unpacklo_epi32(row2i, row3i); 96 vTemp = _mm_unpackhi_epi32(vTemp, row3i); 97 98 row3i = row0i; 99 row0i = _mm_unpacklo_epi32(row0i, row1i); 100 row3i = _mm_unpackhi_epi32(row3i, row1i); 101 102 row1i = row0i; 103 row0i = _mm_unpacklo_epi64(row0i, row2i); 104 row1i = _mm_unpackhi_epi64(row1i, row2i); 105 106 row2i = row3i; 107 row2i = _mm_unpacklo_epi64(row2i, vTemp); 108 row3i = _mm_unpackhi_epi64(row3i, vTemp); 109 110 row0 = _mm_castsi128_ps(row0i); 111 row1 = _mm_castsi128_ps(row1i); 112 row2 = _mm_castsi128_ps(row2i); 113 row3 = _mm_castsi128_ps(row3i); 114 } 115 116 INLINE 117 void vTranspose(__m128i &row0, __m128i &row1, __m128i &row2, __m128i &row3) 118 { 119 __m128i vTemp = row2; 120 row2 = _mm_unpacklo_epi32(row2, row3); 121 vTemp = _mm_unpackhi_epi32(vTemp, row3); 122 123 row3 = row0; 124 row0 = _mm_unpacklo_epi32(row0, row1); 125 row3 = _mm_unpackhi_epi32(row3, row1); 126 127 row1 = row0; 128 row0 = _mm_unpacklo_epi64(row0, row2); 129 row1 = _mm_unpackhi_epi64(row1, row2); 130 131 row2 = row3; 132 row2 = _mm_unpacklo_epi64(row2, vTemp); 133 row3 = _mm_unpackhi_epi64(row3, vTemp); 134 } 135 136 #define GCC_VERSION (__GNUC__ * 10000 \ 137 + __GNUC_MINOR__ * 100 \ 138 + __GNUC_PATCHLEVEL__) 139 140 #if defined(__clang__) || (defined(__GNUC__) && (GCC_VERSION < 40900)) 141 #define _mm_undefined_ps _mm_setzero_ps 142 #define _mm_undefined_si128 _mm_setzero_si128 143 #if KNOB_SIMD_WIDTH == 8 144 #define _mm256_undefined_ps _mm256_setzero_ps 145 #endif 146 #endif 147 148 #if KNOB_SIMD_WIDTH == 8 149 INLINE 150 void vTranspose3x8(__m128 (&vDst)[8], const __m256 &vSrc0, const __m256 &vSrc1, const __m256 &vSrc2) 151 { 152 __m256 r0r2 = _mm256_unpacklo_ps(vSrc0, vSrc2); //x0z0x1z1 x4z4x5z5 153 __m256 r1rx = _mm256_unpacklo_ps(vSrc1, _mm256_undefined_ps()); //y0w0y1w1 y4w4y5w5 154 __m256 r02r1xlolo = _mm256_unpacklo_ps(r0r2, r1rx); //x0y0z0w0 x4y4z4w4 155 __m256 r02r1xlohi = _mm256_unpackhi_ps(r0r2, r1rx); //x1y1z1w1 x5y5z5w5 156 157 r0r2 = _mm256_unpackhi_ps(vSrc0, vSrc2); //x2z2x3z3 x6z6x7z7 158 r1rx = _mm256_unpackhi_ps(vSrc1, _mm256_undefined_ps()); //y2w2y3w3 y6w6yw77 159 __m256 r02r1xhilo = _mm256_unpacklo_ps(r0r2, r1rx); //x2y2z2w2 x6y6z6w6 160 __m256 r02r1xhihi = _mm256_unpackhi_ps(r0r2, r1rx); //x3y3z3w3 x7y7z7w7 161 162 vDst[0] = _mm256_castps256_ps128(r02r1xlolo); 163 vDst[1] = _mm256_castps256_ps128(r02r1xlohi); 164 vDst[2] = _mm256_castps256_ps128(r02r1xhilo); 165 vDst[3] = _mm256_castps256_ps128(r02r1xhihi); 166 167 vDst[4] = _mm256_extractf128_ps(r02r1xlolo, 1); 168 vDst[5] = _mm256_extractf128_ps(r02r1xlohi, 1); 169 vDst[6] = _mm256_extractf128_ps(r02r1xhilo, 1); 170 vDst[7] = _mm256_extractf128_ps(r02r1xhihi, 1); 171 } 172 173 INLINE 174 void vTranspose4x8(__m128 (&vDst)[8], const __m256 &vSrc0, const __m256 &vSrc1, const __m256 &vSrc2, const __m256 &vSrc3) 175 { 176 __m256 r0r2 = _mm256_unpacklo_ps(vSrc0, vSrc2); //x0z0x1z1 x4z4x5z5 177 __m256 r1rx = _mm256_unpacklo_ps(vSrc1, vSrc3); //y0w0y1w1 y4w4y5w5 178 __m256 r02r1xlolo = _mm256_unpacklo_ps(r0r2, r1rx); //x0y0z0w0 x4y4z4w4 179 __m256 r02r1xlohi = _mm256_unpackhi_ps(r0r2, r1rx); //x1y1z1w1 x5y5z5w5 180 181 r0r2 = _mm256_unpackhi_ps(vSrc0, vSrc2); //x2z2x3z3 x6z6x7z7 182 r1rx = _mm256_unpackhi_ps(vSrc1, vSrc3) ; //y2w2y3w3 y6w6yw77 183 __m256 r02r1xhilo = _mm256_unpacklo_ps(r0r2, r1rx); //x2y2z2w2 x6y6z6w6 184 __m256 r02r1xhihi = _mm256_unpackhi_ps(r0r2, r1rx); //x3y3z3w3 x7y7z7w7 185 186 vDst[0] = _mm256_castps256_ps128(r02r1xlolo); 187 vDst[1] = _mm256_castps256_ps128(r02r1xlohi); 188 vDst[2] = _mm256_castps256_ps128(r02r1xhilo); 189 vDst[3] = _mm256_castps256_ps128(r02r1xhihi); 190 191 vDst[4] = _mm256_extractf128_ps(r02r1xlolo, 1); 192 vDst[5] = _mm256_extractf128_ps(r02r1xlohi, 1); 193 vDst[6] = _mm256_extractf128_ps(r02r1xhilo, 1); 194 vDst[7] = _mm256_extractf128_ps(r02r1xhihi, 1); 195 } 196 197 #if ENABLE_AVX512_SIMD16 198 INLINE 199 void vTranspose4x16(simd16scalar(&dst)[4], const simd16scalar &src0, const simd16scalar &src1, const simd16scalar &src2, const simd16scalar &src3) 200 { 201 const simd16scalari perm = _simd16_set_epi32(15, 11, 7, 3, 14, 10, 6, 2, 13, 9, 5, 1, 12, 8, 4, 0); // pre-permute input to setup the right order after all the unpacking 202 203 simd16scalar pre0 = _simd16_permute_ps(src0, perm); // r 204 simd16scalar pre1 = _simd16_permute_ps(src1, perm); // g 205 simd16scalar pre2 = _simd16_permute_ps(src2, perm); // b 206 simd16scalar pre3 = _simd16_permute_ps(src3, perm); // a 207 208 simd16scalar rblo = _simd16_unpacklo_ps(pre0, pre2); 209 simd16scalar galo = _simd16_unpacklo_ps(pre1, pre3); 210 simd16scalar rbhi = _simd16_unpackhi_ps(pre0, pre2); 211 simd16scalar gahi = _simd16_unpackhi_ps(pre1, pre3); 212 213 dst[0] = _simd16_unpacklo_ps(rblo, galo); 214 dst[1] = _simd16_unpackhi_ps(rblo, galo); 215 dst[2] = _simd16_unpacklo_ps(rbhi, gahi); 216 dst[3] = _simd16_unpackhi_ps(rbhi, gahi); 217 } 218 219 #endif 220 INLINE 221 void vTranspose8x8(__m256 (&vDst)[8], const __m256 &vMask0, const __m256 &vMask1, const __m256 &vMask2, const __m256 &vMask3, const __m256 &vMask4, const __m256 &vMask5, const __m256 &vMask6, const __m256 &vMask7) 222 { 223 __m256 __t0 = _mm256_unpacklo_ps(vMask0, vMask1); 224 __m256 __t1 = _mm256_unpackhi_ps(vMask0, vMask1); 225 __m256 __t2 = _mm256_unpacklo_ps(vMask2, vMask3); 226 __m256 __t3 = _mm256_unpackhi_ps(vMask2, vMask3); 227 __m256 __t4 = _mm256_unpacklo_ps(vMask4, vMask5); 228 __m256 __t5 = _mm256_unpackhi_ps(vMask4, vMask5); 229 __m256 __t6 = _mm256_unpacklo_ps(vMask6, vMask7); 230 __m256 __t7 = _mm256_unpackhi_ps(vMask6, vMask7); 231 __m256 __tt0 = _mm256_shuffle_ps(__t0,__t2,_MM_SHUFFLE(1,0,1,0)); 232 __m256 __tt1 = _mm256_shuffle_ps(__t0,__t2,_MM_SHUFFLE(3,2,3,2)); 233 __m256 __tt2 = _mm256_shuffle_ps(__t1,__t3,_MM_SHUFFLE(1,0,1,0)); 234 __m256 __tt3 = _mm256_shuffle_ps(__t1,__t3,_MM_SHUFFLE(3,2,3,2)); 235 __m256 __tt4 = _mm256_shuffle_ps(__t4,__t6,_MM_SHUFFLE(1,0,1,0)); 236 __m256 __tt5 = _mm256_shuffle_ps(__t4,__t6,_MM_SHUFFLE(3,2,3,2)); 237 __m256 __tt6 = _mm256_shuffle_ps(__t5,__t7,_MM_SHUFFLE(1,0,1,0)); 238 __m256 __tt7 = _mm256_shuffle_ps(__t5,__t7,_MM_SHUFFLE(3,2,3,2)); 239 vDst[0] = _mm256_permute2f128_ps(__tt0, __tt4, 0x20); 240 vDst[1] = _mm256_permute2f128_ps(__tt1, __tt5, 0x20); 241 vDst[2] = _mm256_permute2f128_ps(__tt2, __tt6, 0x20); 242 vDst[3] = _mm256_permute2f128_ps(__tt3, __tt7, 0x20); 243 vDst[4] = _mm256_permute2f128_ps(__tt0, __tt4, 0x31); 244 vDst[5] = _mm256_permute2f128_ps(__tt1, __tt5, 0x31); 245 vDst[6] = _mm256_permute2f128_ps(__tt2, __tt6, 0x31); 246 vDst[7] = _mm256_permute2f128_ps(__tt3, __tt7, 0x31); 247 } 248 249 INLINE 250 void vTranspose8x8(__m256 (&vDst)[8], const __m256i &vMask0, const __m256i &vMask1, const __m256i &vMask2, const __m256i &vMask3, const __m256i &vMask4, const __m256i &vMask5, const __m256i &vMask6, const __m256i &vMask7) 251 { 252 vTranspose8x8(vDst, _mm256_castsi256_ps(vMask0), _mm256_castsi256_ps(vMask1), _mm256_castsi256_ps(vMask2), _mm256_castsi256_ps(vMask3), 253 _mm256_castsi256_ps(vMask4), _mm256_castsi256_ps(vMask5), _mm256_castsi256_ps(vMask6), _mm256_castsi256_ps(vMask7)); 254 } 255 #endif 256 257 ////////////////////////////////////////////////////////////////////////// 258 /// TranposeSingleComponent 259 ////////////////////////////////////////////////////////////////////////// 260 template<uint32_t bpp> 261 struct TransposeSingleComponent 262 { 263 ////////////////////////////////////////////////////////////////////////// 264 /// @brief Pass-thru for single component. 265 /// @param pSrc - source data in SOA form 266 /// @param pDst - output data in AOS form 267 INLINE static void Transpose(const uint8_t* pSrc, uint8_t* pDst) 268 { 269 memcpy(pDst, pSrc, (bpp * KNOB_SIMD_WIDTH) / 8); 270 } 271 #if ENABLE_AVX512_SIMD16 272 273 INLINE static void Transpose_16(const uint8_t* pSrc, uint8_t* pDst) 274 { 275 memcpy(pDst, pSrc, (bpp * KNOB_SIMD16_WIDTH) / 8); 276 } 277 #endif 278 }; 279 280 ////////////////////////////////////////////////////////////////////////// 281 /// Transpose8_8_8_8 282 ////////////////////////////////////////////////////////////////////////// 283 struct Transpose8_8_8_8 284 { 285 ////////////////////////////////////////////////////////////////////////// 286 /// @brief Performs an SOA to AOS conversion for packed 8_8_8_8 data. 287 /// @param pSrc - source data in SOA form 288 /// @param pDst - output data in AOS form 289 INLINE static void Transpose(const uint8_t* pSrc, uint8_t* pDst) 290 { 291 simdscalari src = _simd_load_si((const simdscalari*)pSrc); 292 293 #if KNOB_SIMD_WIDTH == 8 294 #if KNOB_ARCH == KNOB_ARCH_AVX 295 __m128i c0c1 = _mm256_castsi256_si128(src); // rrrrrrrrgggggggg 296 __m128i c2c3 = _mm_castps_si128(_mm256_extractf128_ps(_mm256_castsi256_ps(src), 1)); // bbbbbbbbaaaaaaaa 297 __m128i c0c2 = _mm_unpacklo_epi64(c0c1, c2c3); // rrrrrrrrbbbbbbbb 298 __m128i c1c3 = _mm_unpackhi_epi64(c0c1, c2c3); // ggggggggaaaaaaaa 299 __m128i c01 = _mm_unpacklo_epi8(c0c2, c1c3); // rgrgrgrgrgrgrgrg 300 __m128i c23 = _mm_unpackhi_epi8(c0c2, c1c3); // babababababababa 301 __m128i c0123lo = _mm_unpacklo_epi16(c01, c23); // rgbargbargbargba 302 __m128i c0123hi = _mm_unpackhi_epi16(c01, c23); // rgbargbargbargba 303 _mm_store_si128((__m128i*)pDst, c0123lo); 304 _mm_store_si128((__m128i*)(pDst + 16), c0123hi); 305 #elif KNOB_ARCH == KNOB_ARCH_AVX2 306 simdscalari dst01 = _mm256_shuffle_epi8(src, 307 _mm256_set_epi32(0x0f078080, 0x0e068080, 0x0d058080, 0x0c048080, 0x80800b03, 0x80800a02, 0x80800901, 0x80800800)); 308 simdscalari dst23 = _mm256_permute2x128_si256(src, src, 0x01); 309 dst23 = _mm256_shuffle_epi8(dst23, 310 _mm256_set_epi32(0x80800f07, 0x80800e06, 0x80800d05, 0x80800c04, 0x0b038080, 0x0a028080, 0x09018080, 0x08008080)); 311 simdscalari dst = _mm256_or_si256(dst01, dst23); 312 _simd_store_si((simdscalari*)pDst, dst); 313 #endif 314 #else 315 #error Unsupported vector width 316 #endif 317 } 318 #if ENABLE_AVX512_SIMD16 319 320 INLINE static void Transpose_16(const uint8_t* pSrc, uint8_t* pDst) 321 { 322 __m128i src0 = _mm_load_si128(reinterpret_cast<const __m128i *>(pSrc)); // rrrrrrrrrrrrrrrr 323 __m128i src1 = _mm_load_si128(reinterpret_cast<const __m128i *>(pSrc) + 1); // gggggggggggggggg 324 __m128i src2 = _mm_load_si128(reinterpret_cast<const __m128i *>(pSrc) + 2); // bbbbbbbbbbbbbbbb 325 __m128i src3 = _mm_load_si128(reinterpret_cast<const __m128i *>(pSrc) + 3); // aaaaaaaaaaaaaaaa 326 327 simd16scalari cvt0 = _simd16_cvtepu8_epi32(src0); 328 simd16scalari cvt1 = _simd16_cvtepu8_epi32(src1); 329 simd16scalari cvt2 = _simd16_cvtepu8_epi32(src2); 330 simd16scalari cvt3 = _simd16_cvtepu8_epi32(src3); 331 332 simd16scalari shl1 = _simd16_slli_epi32(cvt1, 8); 333 simd16scalari shl2 = _simd16_slli_epi32(cvt2, 16); 334 simd16scalari shl3 = _simd16_slli_epi32(cvt3, 24); 335 336 simd16scalari dst = _simd16_or_si(_simd16_or_si(cvt0, shl1), _simd16_or_si(shl2, shl3)); 337 338 _simd16_store_si(reinterpret_cast<simd16scalari *>(pDst), dst); // rgbargbargbargbargbargbargbargbargbargbargbargbargbargbargbargba 339 } 340 #endif 341 }; 342 343 ////////////////////////////////////////////////////////////////////////// 344 /// Transpose8_8_8 345 ////////////////////////////////////////////////////////////////////////// 346 struct Transpose8_8_8 347 { 348 ////////////////////////////////////////////////////////////////////////// 349 /// @brief Performs an SOA to AOS conversion for packed 8_8_8 data. 350 /// @param pSrc - source data in SOA form 351 /// @param pDst - output data in AOS form 352 INLINE static void Transpose(const uint8_t* pSrc, uint8_t* pDst) = delete; 353 #if ENABLE_AVX512_SIMD16 354 355 INLINE static void Transpose_16(const uint8_t* pSrc, uint8_t* pDst) = delete; 356 #endif 357 }; 358 359 ////////////////////////////////////////////////////////////////////////// 360 /// Transpose8_8 361 ////////////////////////////////////////////////////////////////////////// 362 struct Transpose8_8 363 { 364 ////////////////////////////////////////////////////////////////////////// 365 /// @brief Performs an SOA to AOS conversion for packed 8_8 data. 366 /// @param pSrc - source data in SOA form 367 /// @param pDst - output data in AOS form 368 INLINE static void Transpose(const uint8_t* pSrc, uint8_t* pDst) 369 { 370 #if KNOB_SIMD_WIDTH == 8 371 simdscalari src = _simd_load_si((const simdscalari*)pSrc); 372 373 __m128i rg = _mm256_castsi256_si128(src); // rrrrrrrr gggggggg 374 __m128i g = _mm_unpackhi_epi64(rg, rg); // gggggggg gggggggg 375 rg = _mm_unpacklo_epi8(rg, g); 376 _mm_store_si128((__m128i*)pDst, rg); 377 #else 378 #error Unsupported vector width 379 #endif 380 } 381 #if ENABLE_AVX512_SIMD16 382 383 INLINE static void Transpose_16(const uint8_t* pSrc, uint8_t* pDst) 384 { 385 __m128i src0 = _mm_load_si128(reinterpret_cast<const __m128i *>(pSrc)); // rrrrrrrrrrrrrrrr 386 __m128i src1 = _mm_load_si128(reinterpret_cast<const __m128i *>(pSrc) + 1); // gggggggggggggggg 387 388 simdscalari cvt0 = _simd_cvtepu8_epi16(src0); 389 simdscalari cvt1 = _simd_cvtepu8_epi16(src1); 390 391 simdscalari shl1 = _simd_slli_epi32(cvt1, 8); 392 393 simdscalari dst = _simd_or_si(cvt0, shl1); 394 395 _simd_store_si(reinterpret_cast<simdscalari *>(pDst), dst); // rgrgrgrgrgrgrgrgrgrgrgrgrgrgrgrg 396 } 397 #endif 398 }; 399 400 ////////////////////////////////////////////////////////////////////////// 401 /// Transpose32_32_32_32 402 ////////////////////////////////////////////////////////////////////////// 403 struct Transpose32_32_32_32 404 { 405 ////////////////////////////////////////////////////////////////////////// 406 /// @brief Performs an SOA to AOS conversion for packed 32_32_32_32 data. 407 /// @param pSrc - source data in SOA form 408 /// @param pDst - output data in AOS form 409 INLINE static void Transpose(const uint8_t* pSrc, uint8_t* pDst) 410 { 411 #if KNOB_SIMD_WIDTH == 8 412 simdscalar src0 = _simd_load_ps((const float*)pSrc); 413 simdscalar src1 = _simd_load_ps((const float*)pSrc + 8); 414 simdscalar src2 = _simd_load_ps((const float*)pSrc + 16); 415 simdscalar src3 = _simd_load_ps((const float*)pSrc + 24); 416 417 __m128 vDst[8]; 418 vTranspose4x8(vDst, src0, src1, src2, src3); 419 _mm_store_ps((float*)pDst, vDst[0]); 420 _mm_store_ps((float*)pDst+4, vDst[1]); 421 _mm_store_ps((float*)pDst+8, vDst[2]); 422 _mm_store_ps((float*)pDst+12, vDst[3]); 423 _mm_store_ps((float*)pDst+16, vDst[4]); 424 _mm_store_ps((float*)pDst+20, vDst[5]); 425 _mm_store_ps((float*)pDst+24, vDst[6]); 426 _mm_store_ps((float*)pDst+28, vDst[7]); 427 #else 428 #error Unsupported vector width 429 #endif 430 } 431 #if ENABLE_AVX512_SIMD16 432 433 INLINE static void Transpose_16(const uint8_t* pSrc, uint8_t* pDst) 434 { 435 simd16scalar src0 = _simd16_load_ps(reinterpret_cast<const float *>(pSrc)); 436 simd16scalar src1 = _simd16_load_ps(reinterpret_cast<const float *>(pSrc) + 16); 437 simd16scalar src2 = _simd16_load_ps(reinterpret_cast<const float *>(pSrc) + 32); 438 simd16scalar src3 = _simd16_load_ps(reinterpret_cast<const float *>(pSrc) + 48); 439 440 simd16scalar dst[4]; 441 442 vTranspose4x16(dst, src0, src1, src2, src3); 443 444 _simd16_store_ps(reinterpret_cast<float *>(pDst) + 0, dst[0]); 445 _simd16_store_ps(reinterpret_cast<float *>(pDst) + 16, dst[1]); 446 _simd16_store_ps(reinterpret_cast<float *>(pDst) + 32, dst[2]); 447 _simd16_store_ps(reinterpret_cast<float *>(pDst) + 48, dst[3]); 448 } 449 #endif 450 }; 451 452 ////////////////////////////////////////////////////////////////////////// 453 /// Transpose32_32_32 454 ////////////////////////////////////////////////////////////////////////// 455 struct Transpose32_32_32 456 { 457 ////////////////////////////////////////////////////////////////////////// 458 /// @brief Performs an SOA to AOS conversion for packed 32_32_32 data. 459 /// @param pSrc - source data in SOA form 460 /// @param pDst - output data in AOS form 461 INLINE static void Transpose(const uint8_t* pSrc, uint8_t* pDst) 462 { 463 #if KNOB_SIMD_WIDTH == 8 464 simdscalar src0 = _simd_load_ps((const float*)pSrc); 465 simdscalar src1 = _simd_load_ps((const float*)pSrc + 8); 466 simdscalar src2 = _simd_load_ps((const float*)pSrc + 16); 467 468 __m128 vDst[8]; 469 vTranspose3x8(vDst, src0, src1, src2); 470 _mm_store_ps((float*)pDst, vDst[0]); 471 _mm_store_ps((float*)pDst + 4, vDst[1]); 472 _mm_store_ps((float*)pDst + 8, vDst[2]); 473 _mm_store_ps((float*)pDst + 12, vDst[3]); 474 _mm_store_ps((float*)pDst + 16, vDst[4]); 475 _mm_store_ps((float*)pDst + 20, vDst[5]); 476 _mm_store_ps((float*)pDst + 24, vDst[6]); 477 _mm_store_ps((float*)pDst + 28, vDst[7]); 478 #else 479 #error Unsupported vector width 480 #endif 481 } 482 #if ENABLE_AVX512_SIMD16 483 484 INLINE static void Transpose_16(const uint8_t* pSrc, uint8_t* pDst) 485 { 486 simd16scalar src0 = _simd16_load_ps(reinterpret_cast<const float *>(pSrc)); 487 simd16scalar src1 = _simd16_load_ps(reinterpret_cast<const float *>(pSrc) + 16); 488 simd16scalar src2 = _simd16_load_ps(reinterpret_cast<const float *>(pSrc) + 32); 489 simd16scalar src3 = _simd16_setzero_ps(); 490 491 simd16scalar dst[4]; 492 493 vTranspose4x16(dst, src0, src1, src2, src3); 494 495 _simd16_store_ps(reinterpret_cast<float *>(pDst) + 0, dst[0]); 496 _simd16_store_ps(reinterpret_cast<float *>(pDst) + 16, dst[1]); 497 _simd16_store_ps(reinterpret_cast<float *>(pDst) + 32, dst[2]); 498 _simd16_store_ps(reinterpret_cast<float *>(pDst) + 48, dst[3]); 499 } 500 #endif 501 }; 502 503 ////////////////////////////////////////////////////////////////////////// 504 /// Transpose32_32 505 ////////////////////////////////////////////////////////////////////////// 506 struct Transpose32_32 507 { 508 ////////////////////////////////////////////////////////////////////////// 509 /// @brief Performs an SOA to AOS conversion for packed 32_32 data. 510 /// @param pSrc - source data in SOA form 511 /// @param pDst - output data in AOS form 512 INLINE static void Transpose(const uint8_t* pSrc, uint8_t* pDst) 513 { 514 #if KNOB_SIMD_WIDTH == 8 515 const float* pfSrc = (const float*)pSrc; 516 __m128 src_r0 = _mm_load_ps(pfSrc + 0); 517 __m128 src_r1 = _mm_load_ps(pfSrc + 4); 518 __m128 src_g0 = _mm_load_ps(pfSrc + 8); 519 __m128 src_g1 = _mm_load_ps(pfSrc + 12); 520 521 __m128 dst0 = _mm_unpacklo_ps(src_r0, src_g0); 522 __m128 dst1 = _mm_unpackhi_ps(src_r0, src_g0); 523 __m128 dst2 = _mm_unpacklo_ps(src_r1, src_g1); 524 __m128 dst3 = _mm_unpackhi_ps(src_r1, src_g1); 525 526 float* pfDst = (float*)pDst; 527 _mm_store_ps(pfDst + 0, dst0); 528 _mm_store_ps(pfDst + 4, dst1); 529 _mm_store_ps(pfDst + 8, dst2); 530 _mm_store_ps(pfDst + 12, dst3); 531 #else 532 #error Unsupported vector width 533 #endif 534 } 535 #if ENABLE_AVX512_SIMD16 536 537 INLINE static void Transpose_16(const uint8_t* pSrc, uint8_t* pDst) 538 { 539 simd16scalar src0 = _simd16_load_ps(reinterpret_cast<const float *>(pSrc)); // rrrrrrrrrrrrrrrr 540 simd16scalar src1 = _simd16_load_ps(reinterpret_cast<const float *>(pSrc) + 16); // gggggggggggggggg 541 542 simd16scalar tmp0 = _simd16_unpacklo_ps(src0, src1); // r0 g0 r1 g1 r4 g4 r5 g5 r8 g8 r9 g9 rC gC rD gD 543 simd16scalar tmp1 = _simd16_unpackhi_ps(src0, src1); // r2 g2 r3 g3 r6 g6 r7 g7 rA gA rB gB rE gE rF gF 544 545 simd16scalar per0 = _simd16_permute2f128_ps(tmp0, tmp1, 0x44); // (1, 0, 1, 0) // r0 g0 r1 g1 r4 g4 r5 g5 r2 g2 r3 g3 r6 g6 r7 g7 546 simd16scalar per1 = _simd16_permute2f128_ps(tmp0, tmp1, 0xEE); // (3, 2, 3, 2) // r8 g8 r9 g9 rC gC rD gD rA gA rB gB rE gE rF gF 547 548 simd16scalar dst0 = _simd16_permute2f128_ps(per0, per0, 0xD8); // (3, 1, 2, 0) // r0 g0 r1 g1 r2 g2 r3 g3 r4 g4 r5 g5 r6 g6 r7 g7 549 simd16scalar dst1 = _simd16_permute2f128_ps(per1, per1, 0xD8); // (3, 1, 2, 0) // r8 g8 r9 g9 rA gA rB gB rC gC rD gD rE gE rF gF 550 551 _simd16_store_ps(reinterpret_cast<float *>(pDst) + 0, dst0); // rgrgrgrgrgrgrgrg 552 _simd16_store_ps(reinterpret_cast<float *>(pDst) + 16, dst1); // rgrgrgrgrgrgrgrg 553 } 554 #endif 555 }; 556 557 ////////////////////////////////////////////////////////////////////////// 558 /// Transpose16_16_16_16 559 ////////////////////////////////////////////////////////////////////////// 560 struct Transpose16_16_16_16 561 { 562 ////////////////////////////////////////////////////////////////////////// 563 /// @brief Performs an SOA to AOS conversion for packed 16_16_16_16 data. 564 /// @param pSrc - source data in SOA form 565 /// @param pDst - output data in AOS form 566 INLINE static void Transpose(const uint8_t* pSrc, uint8_t* pDst) 567 { 568 #if KNOB_SIMD_WIDTH == 8 569 simdscalari src_rg = _simd_load_si((const simdscalari*)pSrc); 570 simdscalari src_ba = _simd_load_si((const simdscalari*)(pSrc + sizeof(simdscalari))); 571 572 __m128i src_r = _mm256_extractf128_si256(src_rg, 0); 573 __m128i src_g = _mm256_extractf128_si256(src_rg, 1); 574 __m128i src_b = _mm256_extractf128_si256(src_ba, 0); 575 __m128i src_a = _mm256_extractf128_si256(src_ba, 1); 576 577 __m128i rg0 = _mm_unpacklo_epi16(src_r, src_g); 578 __m128i rg1 = _mm_unpackhi_epi16(src_r, src_g); 579 __m128i ba0 = _mm_unpacklo_epi16(src_b, src_a); 580 __m128i ba1 = _mm_unpackhi_epi16(src_b, src_a); 581 582 __m128i dst0 = _mm_unpacklo_epi32(rg0, ba0); 583 __m128i dst1 = _mm_unpackhi_epi32(rg0, ba0); 584 __m128i dst2 = _mm_unpacklo_epi32(rg1, ba1); 585 __m128i dst3 = _mm_unpackhi_epi32(rg1, ba1); 586 587 _mm_store_si128(((__m128i*)pDst) + 0, dst0); 588 _mm_store_si128(((__m128i*)pDst) + 1, dst1); 589 _mm_store_si128(((__m128i*)pDst) + 2, dst2); 590 _mm_store_si128(((__m128i*)pDst) + 3, dst3); 591 #else 592 #error Unsupported vector width 593 #endif 594 } 595 #if ENABLE_AVX512_SIMD16 596 597 INLINE static void Transpose_16(const uint8_t* pSrc, uint8_t* pDst) 598 { 599 simdscalari src0 = _simd_load_si(reinterpret_cast<const simdscalari *>(pSrc)); // rrrrrrrrrrrrrrrr 600 simdscalari src1 = _simd_load_si(reinterpret_cast<const simdscalari *>(pSrc) + 1); // gggggggggggggggg 601 simdscalari src2 = _simd_load_si(reinterpret_cast<const simdscalari *>(pSrc) + 2); // bbbbbbbbbbbbbbbb 602 simdscalari src3 = _simd_load_si(reinterpret_cast<const simdscalari *>(pSrc) + 3); // aaaaaaaaaaaaaaaa 603 604 simdscalari pre0 = _simd_unpacklo_epi16(src0, src1); // rg0 rg1 rg2 rg3 rg8 rg9 rgA rgB 605 simdscalari pre1 = _simd_unpackhi_epi16(src0, src1); // rg4 rg5 rg6 rg7 rgC rgD rgE rgF 606 simdscalari pre2 = _simd_unpacklo_epi16(src2, src3); // ba0 ba1 ba3 ba3 ba8 ba9 baA baB 607 simdscalari pre3 = _simd_unpackhi_epi16(src2, src3); // ba4 ba5 ba6 ba7 baC baD baE baF 608 609 simdscalari tmp0 = _simd_unpacklo_epi32(pre0, pre2); // rbga0 rbga1 rbga8 rbga9 610 simdscalari tmp1 = _simd_unpackhi_epi32(pre0, pre2); // rbga2 rbga3 rbgaA rbgaB 611 simdscalari tmp2 = _simd_unpacklo_epi32(pre1, pre3); // rbga4 rbga5 rgbaC rbgaD 612 simdscalari tmp3 = _simd_unpackhi_epi32(pre1, pre3); // rbga6 rbga7 rbgaE rbgaF 613 614 simdscalari dst0 = _simd_permute2f128_si(tmp0, tmp1, 0x20); // (2, 0) // rbga0 rbga1 rbga2 rbga3 615 simdscalari dst1 = _simd_permute2f128_si(tmp2, tmp3, 0x20); // (2, 0) // rbga4 rbga5 rbga6 rbga7 616 simdscalari dst2 = _simd_permute2f128_si(tmp0, tmp1, 0x31); // (3, 1) // rbga8 rbga9 rbgaA rbgaB 617 simdscalari dst3 = _simd_permute2f128_si(tmp2, tmp3, 0x31); // (3, 1) // rbgaC rbgaD rbgaE rbgaF 618 619 _simd_store_si(reinterpret_cast<simdscalari *>(pDst) + 0, dst0); // rgbargbargbargba 620 _simd_store_si(reinterpret_cast<simdscalari *>(pDst) + 1, dst1); // rgbargbargbargba 621 _simd_store_si(reinterpret_cast<simdscalari *>(pDst) + 2, dst2); // rgbargbargbargba 622 _simd_store_si(reinterpret_cast<simdscalari *>(pDst) + 3, dst3); // rgbargbargbargba 623 } 624 #endif 625 }; 626 627 ////////////////////////////////////////////////////////////////////////// 628 /// Transpose16_16_16 629 ////////////////////////////////////////////////////////////////////////// 630 struct Transpose16_16_16 631 { 632 ////////////////////////////////////////////////////////////////////////// 633 /// @brief Performs an SOA to AOS conversion for packed 16_16_16 data. 634 /// @param pSrc - source data in SOA form 635 /// @param pDst - output data in AOS form 636 INLINE static void Transpose(const uint8_t* pSrc, uint8_t* pDst) 637 { 638 #if KNOB_SIMD_WIDTH == 8 639 simdscalari src_rg = _simd_load_si((const simdscalari*)pSrc); 640 641 __m128i src_r = _mm256_extractf128_si256(src_rg, 0); 642 __m128i src_g = _mm256_extractf128_si256(src_rg, 1); 643 __m128i src_b = _mm_load_si128((const __m128i*)(pSrc + sizeof(simdscalari))); 644 __m128i src_a = _mm_undefined_si128(); 645 646 __m128i rg0 = _mm_unpacklo_epi16(src_r, src_g); 647 __m128i rg1 = _mm_unpackhi_epi16(src_r, src_g); 648 __m128i ba0 = _mm_unpacklo_epi16(src_b, src_a); 649 __m128i ba1 = _mm_unpackhi_epi16(src_b, src_a); 650 651 __m128i dst0 = _mm_unpacklo_epi32(rg0, ba0); 652 __m128i dst1 = _mm_unpackhi_epi32(rg0, ba0); 653 __m128i dst2 = _mm_unpacklo_epi32(rg1, ba1); 654 __m128i dst3 = _mm_unpackhi_epi32(rg1, ba1); 655 656 _mm_store_si128(((__m128i*)pDst) + 0, dst0); 657 _mm_store_si128(((__m128i*)pDst) + 1, dst1); 658 _mm_store_si128(((__m128i*)pDst) + 2, dst2); 659 _mm_store_si128(((__m128i*)pDst) + 3, dst3); 660 #else 661 #error Unsupported vector width 662 #endif 663 } 664 #if ENABLE_AVX512_SIMD16 665 666 INLINE static void Transpose_16(const uint8_t* pSrc, uint8_t* pDst) 667 { 668 simdscalari src0 = _simd_load_si(reinterpret_cast<const simdscalari *>(pSrc)); // rrrrrrrrrrrrrrrr 669 simdscalari src1 = _simd_load_si(reinterpret_cast<const simdscalari *>(pSrc) + 1); // gggggggggggggggg 670 simdscalari src2 = _simd_load_si(reinterpret_cast<const simdscalari *>(pSrc) + 2); // bbbbbbbbbbbbbbbb 671 simdscalari src3 = _simd_setzero_si(); // aaaaaaaaaaaaaaaa 672 673 simdscalari pre0 = _simd_unpacklo_epi16(src0, src1); // rg0 rg1 rg2 rg3 rg8 rg9 rgA rgB 674 simdscalari pre1 = _simd_unpackhi_epi16(src0, src1); // rg4 rg5 rg6 rg7 rgC rgD rgE rgF 675 simdscalari pre2 = _simd_unpacklo_epi16(src2, src3); // ba0 ba1 ba3 ba3 ba8 ba9 baA baB 676 simdscalari pre3 = _simd_unpackhi_epi16(src2, src3); // ba4 ba5 ba6 ba7 baC baD baE baF 677 678 simdscalari tmp0 = _simd_unpacklo_epi32(pre0, pre2); // rbga0 rbga1 rbga8 rbga9 679 simdscalari tmp1 = _simd_unpackhi_epi32(pre0, pre2); // rbga2 rbga3 rbgaA rbgaB 680 simdscalari tmp2 = _simd_unpacklo_epi32(pre1, pre3); // rbga4 rbga5 rgbaC rbgaD 681 simdscalari tmp3 = _simd_unpackhi_epi32(pre1, pre3); // rbga6 rbga7 rbgaE rbgaF 682 683 simdscalari dst0 = _simd_permute2f128_si(tmp0, tmp1, 0x20); // (2, 0) // rbga0 rbga1 rbga2 rbga3 684 simdscalari dst1 = _simd_permute2f128_si(tmp2, tmp3, 0x20); // (2, 0) // rbga4 rbga5 rbga6 rbga7 685 simdscalari dst2 = _simd_permute2f128_si(tmp0, tmp1, 0x31); // (3, 1) // rbga8 rbga9 rbgaA rbgaB 686 simdscalari dst3 = _simd_permute2f128_si(tmp2, tmp3, 0x31); // (3, 1) // rbgaC rbgaD rbgaE rbgaF 687 688 _simd_store_si(reinterpret_cast<simdscalari *>(pDst) + 0, dst0); // rgbargbargbargba 689 _simd_store_si(reinterpret_cast<simdscalari *>(pDst) + 1, dst1); // rgbargbargbargba 690 _simd_store_si(reinterpret_cast<simdscalari *>(pDst) + 2, dst2); // rgbargbargbargba 691 _simd_store_si(reinterpret_cast<simdscalari *>(pDst) + 3, dst3); // rgbargbargbargba 692 } 693 #endif 694 }; 695 696 ////////////////////////////////////////////////////////////////////////// 697 /// Transpose16_16 698 ////////////////////////////////////////////////////////////////////////// 699 struct Transpose16_16 700 { 701 ////////////////////////////////////////////////////////////////////////// 702 /// @brief Performs an SOA to AOS conversion for packed 16_16 data. 703 /// @param pSrc - source data in SOA form 704 /// @param pDst - output data in AOS form 705 INLINE static void Transpose(const uint8_t* pSrc, uint8_t* pDst) 706 { 707 #if KNOB_SIMD_WIDTH == 8 708 simdscalar src = _simd_load_ps((const float*)pSrc); 709 710 __m128 comp0 = _mm256_castps256_ps128(src); 711 __m128 comp1 = _mm256_extractf128_ps(src, 1); 712 713 __m128i comp0i = _mm_castps_si128(comp0); 714 __m128i comp1i = _mm_castps_si128(comp1); 715 716 __m128i resLo = _mm_unpacklo_epi16(comp0i, comp1i); 717 __m128i resHi = _mm_unpackhi_epi16(comp0i, comp1i); 718 719 _mm_store_si128((__m128i*)pDst, resLo); 720 _mm_store_si128((__m128i*)pDst + 1, resHi); 721 #else 722 #error Unsupported vector width 723 #endif 724 } 725 #if ENABLE_AVX512_SIMD16 726 727 INLINE static void Transpose_16(const uint8_t* pSrc, uint8_t* pDst) 728 { 729 simdscalari src0 = _simd_load_si(reinterpret_cast<const simdscalari *>(pSrc)); // rrrrrrrrrrrrrrrr 730 simdscalari src1 = _simd_load_si(reinterpret_cast<const simdscalari *>(pSrc) + 1); // gggggggggggggggg 731 732 simdscalari tmp0 = _simd_unpacklo_epi16(src0, src1); // rg0 rg1 rg2 rg3 rg8 rg9 rgA rgB 733 simdscalari tmp1 = _simd_unpackhi_epi16(src0, src1); // rg4 rg5 rg6 rg7 rgC rgD rgE rgF 734 735 simdscalari dst0 = _simd_permute2f128_si(tmp0, tmp1, 0x20); // (2, 0) // rg0 rg1 rg2 rg3 rg4 rg5 rg6 rg7 736 simdscalari dst1 = _simd_permute2f128_si(tmp0, tmp1, 0x31); // (3, 1) // rg8 rg9 rgA rgB rgC rgD rgE rgF 737 738 _simd_store_si(reinterpret_cast<simdscalari *>(pDst) + 0, dst0); // rgrgrgrgrgrgrgrg 739 _simd_store_si(reinterpret_cast<simdscalari *>(pDst) + 1, dst1); // rgrgrgrgrgrgrgrg 740 } 741 #endif 742 }; 743 744 ////////////////////////////////////////////////////////////////////////// 745 /// Transpose24_8 746 ////////////////////////////////////////////////////////////////////////// 747 struct Transpose24_8 748 { 749 ////////////////////////////////////////////////////////////////////////// 750 /// @brief Performs an SOA to AOS conversion for packed 24_8 data. 751 /// @param pSrc - source data in SOA form 752 /// @param pDst - output data in AOS form 753 static void Transpose(const uint8_t* pSrc, uint8_t* pDst) = delete; 754 #if ENABLE_AVX512_SIMD16 755 756 static void Transpose_16(const uint8_t* pSrc, uint8_t* pDst) = delete; 757 #endif 758 }; 759 760 ////////////////////////////////////////////////////////////////////////// 761 /// Transpose32_8_24 762 ////////////////////////////////////////////////////////////////////////// 763 struct Transpose32_8_24 764 { 765 ////////////////////////////////////////////////////////////////////////// 766 /// @brief Performs an SOA to AOS conversion for packed 32_8_24 data. 767 /// @param pSrc - source data in SOA form 768 /// @param pDst - output data in AOS form 769 static void Transpose(const uint8_t* pSrc, uint8_t* pDst) = delete; 770 #if ENABLE_AVX512_SIMD16 771 772 static void Transpose_16(const uint8_t* pSrc, uint8_t* pDst) = delete; 773 #endif 774 }; 775 776 ////////////////////////////////////////////////////////////////////////// 777 /// Transpose4_4_4_4 778 ////////////////////////////////////////////////////////////////////////// 779 struct Transpose4_4_4_4 780 { 781 ////////////////////////////////////////////////////////////////////////// 782 /// @brief Performs an SOA to AOS conversion for packed 4_4_4_4 data. 783 /// @param pSrc - source data in SOA form 784 /// @param pDst - output data in AOS form 785 static void Transpose(const uint8_t* pSrc, uint8_t* pDst) = delete; 786 #if ENABLE_AVX512_SIMD16 787 788 static void Transpose_16(const uint8_t* pSrc, uint8_t* pDst) = delete; 789 #endif 790 }; 791 792 ////////////////////////////////////////////////////////////////////////// 793 /// Transpose5_6_5 794 ////////////////////////////////////////////////////////////////////////// 795 struct Transpose5_6_5 796 { 797 ////////////////////////////////////////////////////////////////////////// 798 /// @brief Performs an SOA to AOS conversion for packed 5_6_5 data. 799 /// @param pSrc - source data in SOA form 800 /// @param pDst - output data in AOS form 801 static void Transpose(const uint8_t* pSrc, uint8_t* pDst) = delete; 802 #if ENABLE_AVX512_SIMD16 803 804 static void Transpose_16(const uint8_t* pSrc, uint8_t* pDst) = delete; 805 #endif 806 }; 807 808 ////////////////////////////////////////////////////////////////////////// 809 /// Transpose9_9_9_5 810 ////////////////////////////////////////////////////////////////////////// 811 struct Transpose9_9_9_5 812 { 813 ////////////////////////////////////////////////////////////////////////// 814 /// @brief Performs an SOA to AOS conversion for packed 9_9_9_5 data. 815 /// @param pSrc - source data in SOA form 816 /// @param pDst - output data in AOS form 817 static void Transpose(const uint8_t* pSrc, uint8_t* pDst) = delete; 818 #if ENABLE_AVX512_SIMD16 819 820 static void Transpose_16(const uint8_t* pSrc, uint8_t* pDst) = delete; 821 #endif 822 }; 823 824 ////////////////////////////////////////////////////////////////////////// 825 /// Transpose5_5_5_1 826 ////////////////////////////////////////////////////////////////////////// 827 struct Transpose5_5_5_1 828 { 829 ////////////////////////////////////////////////////////////////////////// 830 /// @brief Performs an SOA to AOS conversion for packed 5_5_5_1 data. 831 /// @param pSrc - source data in SOA form 832 /// @param pDst - output data in AOS form 833 static void Transpose(const uint8_t* pSrc, uint8_t* pDst) = delete; 834 #if ENABLE_AVX512_SIMD16 835 836 static void Transpose_16(const uint8_t* pSrc, uint8_t* pDst) = delete; 837 #endif 838 }; 839 840 ////////////////////////////////////////////////////////////////////////// 841 /// Transpose1_5_5_5 842 ////////////////////////////////////////////////////////////////////////// 843 struct Transpose1_5_5_5 844 { 845 ////////////////////////////////////////////////////////////////////////// 846 /// @brief Performs an SOA to AOS conversion for packed 5_5_5_1 data. 847 /// @param pSrc - source data in SOA form 848 /// @param pDst - output data in AOS form 849 static void Transpose(const uint8_t* pSrc, uint8_t* pDst) = delete; 850 }; 851 852 ////////////////////////////////////////////////////////////////////////// 853 /// Transpose10_10_10_2 854 ////////////////////////////////////////////////////////////////////////// 855 struct Transpose10_10_10_2 856 { 857 ////////////////////////////////////////////////////////////////////////// 858 /// @brief Performs an SOA to AOS conversion for packed 10_10_10_2 data. 859 /// @param pSrc - source data in SOA form 860 /// @param pDst - output data in AOS form 861 static void Transpose(const uint8_t* pSrc, uint8_t* pDst) = delete; 862 #if ENABLE_AVX512_SIMD16 863 864 static void Transpose_16(const uint8_t* pSrc, uint8_t* pDst) = delete; 865 #endif 866 }; 867 868 ////////////////////////////////////////////////////////////////////////// 869 /// Transpose11_11_10 870 ////////////////////////////////////////////////////////////////////////// 871 struct Transpose11_11_10 872 { 873 ////////////////////////////////////////////////////////////////////////// 874 /// @brief Performs an SOA to AOS conversion for packed 11_11_10 data. 875 /// @param pSrc - source data in SOA form 876 /// @param pDst - output data in AOS form 877 static void Transpose(const uint8_t* pSrc, uint8_t* pDst) = delete; 878 #if ENABLE_AVX512_SIMD16 879 880 static void Transpose_16(const uint8_t* pSrc, uint8_t* pDst) = delete; 881 #endif 882 }; 883 884 ////////////////////////////////////////////////////////////////////////// 885 /// Transpose64 886 ////////////////////////////////////////////////////////////////////////// 887 struct Transpose64 888 { 889 ////////////////////////////////////////////////////////////////////////// 890 /// @brief Performs an SOA to AOS conversion 891 /// @param pSrc - source data in SOA form 892 /// @param pDst - output data in AOS form 893 static void Transpose(const uint8_t* pSrc, uint8_t* pDst) = delete; 894 #if ENABLE_AVX512_SIMD16 895 896 static void Transpose_16(const uint8_t* pSrc, uint8_t* pDst) = delete; 897 #endif 898 }; 899 900 ////////////////////////////////////////////////////////////////////////// 901 /// Transpose64_64 902 ////////////////////////////////////////////////////////////////////////// 903 struct Transpose64_64 904 { 905 ////////////////////////////////////////////////////////////////////////// 906 /// @brief Performs an SOA to AOS conversion 907 /// @param pSrc - source data in SOA form 908 /// @param pDst - output data in AOS form 909 static void Transpose(const uint8_t* pSrc, uint8_t* pDst) = delete; 910 #if ENABLE_AVX512_SIMD16 911 912 static void Transpose_16(const uint8_t* pSrc, uint8_t* pDst) = delete; 913 #endif 914 }; 915 916 ////////////////////////////////////////////////////////////////////////// 917 /// Transpose64_64_64 918 ////////////////////////////////////////////////////////////////////////// 919 struct Transpose64_64_64 920 { 921 ////////////////////////////////////////////////////////////////////////// 922 /// @brief Performs an SOA to AOS conversion 923 /// @param pSrc - source data in SOA form 924 /// @param pDst - output data in AOS form 925 static void Transpose(const uint8_t* pSrc, uint8_t* pDst) = delete; 926 #if ENABLE_AVX512_SIMD16 927 928 static void Transpose_16(const uint8_t* pSrc, uint8_t* pDst) = delete; 929 #endif 930 }; 931 932 ////////////////////////////////////////////////////////////////////////// 933 /// Transpose64_64_64_64 934 ////////////////////////////////////////////////////////////////////////// 935 struct Transpose64_64_64_64 936 { 937 ////////////////////////////////////////////////////////////////////////// 938 /// @brief Performs an SOA to AOS conversion 939 /// @param pSrc - source data in SOA form 940 /// @param pDst - output data in AOS form 941 static void Transpose(const uint8_t* pSrc, uint8_t* pDst) = delete; 942 #if ENABLE_AVX512_SIMD16 943 944 static void Transpose_16(const uint8_t* pSrc, uint8_t* pDst) = delete; 945 #endif 946 }; 947 948 // helper function to unroll loops 949 template<int Begin, int End, int Step = 1> 950 struct UnrollerL { 951 template<typename Lambda> 952 INLINE static void step(Lambda& func) { 953 func(Begin); 954 UnrollerL<Begin + Step, End, Step>::step(func); 955 } 956 }; 957 958 template<int End, int Step> 959 struct UnrollerL<End, End, Step> { 960 template<typename Lambda> 961 static void step(Lambda& func) { 962 } 963 }; 964 965 // helper function to unroll loops, with mask to skip specific iterations 966 template<int Begin, int End, int Step = 1, int Mask = 0x7f> 967 struct UnrollerLMask { 968 template<typename Lambda> 969 INLINE static void step(Lambda& func) { 970 if(Mask & (1 << Begin)) 971 { 972 func(Begin); 973 } 974 UnrollerL<Begin + Step, End, Step>::step(func); 975 } 976 }; 977 978 template<int End, int Step, int Mask> 979 struct UnrollerLMask<End, End, Step, Mask> { 980 template<typename Lambda> 981 static void step(Lambda& func) { 982 } 983 }; 984 985 // general CRC compute 986 INLINE 987 uint32_t ComputeCRC(uint32_t crc, const void *pData, uint32_t size) 988 { 989 #if defined(_WIN64) || defined(__x86_64__) 990 uint32_t sizeInQwords = size / sizeof(uint64_t); 991 uint32_t sizeRemainderBytes = size % sizeof(uint64_t); 992 uint64_t* pDataWords = (uint64_t*)pData; 993 for (uint32_t i = 0; i < sizeInQwords; ++i) 994 { 995 crc = (uint32_t)_mm_crc32_u64(crc, *pDataWords++); 996 } 997 #else 998 uint32_t sizeInDwords = size / sizeof(uint32_t); 999 uint32_t sizeRemainderBytes = size % sizeof(uint32_t); 1000 uint32_t* pDataWords = (uint32_t*)pData; 1001 for (uint32_t i = 0; i < sizeInDwords; ++i) 1002 { 1003 crc = _mm_crc32_u32(crc, *pDataWords++); 1004 } 1005 #endif 1006 1007 uint8_t* pRemainderBytes = (uint8_t*)pDataWords; 1008 for (uint32_t i = 0; i < sizeRemainderBytes; ++i) 1009 { 1010 crc = _mm_crc32_u8(crc, *pRemainderBytes++); 1011 } 1012 1013 return crc; 1014 } 1015 1016 ////////////////////////////////////////////////////////////////////////// 1017 /// Add byte offset to any-type pointer 1018 ////////////////////////////////////////////////////////////////////////// 1019 template <typename T> 1020 INLINE 1021 static T* PtrAdd(T* p, intptr_t offset) 1022 { 1023 intptr_t intp = reinterpret_cast<intptr_t>(p); 1024 return reinterpret_cast<T*>(intp + offset); 1025 } 1026 1027 ////////////////////////////////////////////////////////////////////////// 1028 /// Is a power-of-2? 1029 ////////////////////////////////////////////////////////////////////////// 1030 template <typename T> 1031 INLINE 1032 static bool IsPow2(T value) 1033 { 1034 return value == (value & (0 - value)); 1035 } 1036 1037 ////////////////////////////////////////////////////////////////////////// 1038 /// Align down to specified alignment 1039 /// Note: IsPow2(alignment) MUST be true 1040 ////////////////////////////////////////////////////////////////////////// 1041 template <typename T1, typename T2> 1042 INLINE 1043 static T1 AlignDownPow2(T1 value, T2 alignment) 1044 { 1045 SWR_ASSERT(IsPow2(alignment)); 1046 return value & ~T1(alignment - 1); 1047 } 1048 1049 ////////////////////////////////////////////////////////////////////////// 1050 /// Align up to specified alignment 1051 /// Note: IsPow2(alignment) MUST be true 1052 ////////////////////////////////////////////////////////////////////////// 1053 template <typename T1, typename T2> 1054 INLINE 1055 static T1 AlignUpPow2(T1 value, T2 alignment) 1056 { 1057 return AlignDownPow2(value + T1(alignment - 1), alignment); 1058 } 1059 1060 ////////////////////////////////////////////////////////////////////////// 1061 /// Align up ptr to specified alignment 1062 /// Note: IsPow2(alignment) MUST be true 1063 ////////////////////////////////////////////////////////////////////////// 1064 template <typename T1, typename T2> 1065 INLINE 1066 static T1* AlignUpPow2(T1* value, T2 alignment) 1067 { 1068 return reinterpret_cast<T1*>( 1069 AlignDownPow2(reinterpret_cast<uintptr_t>(value) + uintptr_t(alignment - 1), alignment)); 1070 } 1071 1072 ////////////////////////////////////////////////////////////////////////// 1073 /// Align down to specified alignment 1074 ////////////////////////////////////////////////////////////////////////// 1075 template <typename T1, typename T2> 1076 INLINE 1077 static T1 AlignDown(T1 value, T2 alignment) 1078 { 1079 if (IsPow2(alignment)) { return AlignDownPow2(value, alignment); } 1080 return value - T1(value % alignment); 1081 } 1082 1083 ////////////////////////////////////////////////////////////////////////// 1084 /// Align down to specified alignment 1085 ////////////////////////////////////////////////////////////////////////// 1086 template <typename T1, typename T2> 1087 INLINE 1088 static T1* AlignDown(T1* value, T2 alignment) 1089 { 1090 return (T1*)AlignDown(uintptr_t(value), alignment); 1091 } 1092 1093 ////////////////////////////////////////////////////////////////////////// 1094 /// Align up to specified alignment 1095 /// Note: IsPow2(alignment) MUST be true 1096 ////////////////////////////////////////////////////////////////////////// 1097 template <typename T1, typename T2> 1098 INLINE 1099 static T1 AlignUp(T1 value, T2 alignment) 1100 { 1101 return AlignDown(value + T1(alignment - 1), alignment); 1102 } 1103 1104 ////////////////////////////////////////////////////////////////////////// 1105 /// Align up to specified alignment 1106 /// Note: IsPow2(alignment) MUST be true 1107 ////////////////////////////////////////////////////////////////////////// 1108 template <typename T1, typename T2> 1109 INLINE 1110 static T1* AlignUp(T1* value, T2 alignment) 1111 { 1112 return AlignDown(PtrAdd(value, alignment - 1), alignment); 1113 } 1114 1115 ////////////////////////////////////////////////////////////////////////// 1116 /// Helper structure used to access an array of elements that don't 1117 /// correspond to a typical word size. 1118 ////////////////////////////////////////////////////////////////////////// 1119 template<typename T, size_t BitsPerElementT, size_t ArrayLenT> 1120 class BitsArray 1121 { 1122 private: 1123 static const size_t BITS_PER_WORD = sizeof(size_t) * 8; 1124 static const size_t ELEMENTS_PER_WORD = BITS_PER_WORD / BitsPerElementT; 1125 static const size_t NUM_WORDS = (ArrayLenT + ELEMENTS_PER_WORD - 1) / ELEMENTS_PER_WORD; 1126 static const size_t ELEMENT_MASK = (size_t(1) << BitsPerElementT) - 1; 1127 1128 static_assert(ELEMENTS_PER_WORD * BitsPerElementT == BITS_PER_WORD, 1129 "Element size must an integral fraction of pointer size"); 1130 1131 size_t m_words[NUM_WORDS] = {}; 1132 1133 public: 1134 1135 T operator[] (size_t elementIndex) const 1136 { 1137 size_t word = m_words[elementIndex / ELEMENTS_PER_WORD]; 1138 word >>= ((elementIndex % ELEMENTS_PER_WORD) * BitsPerElementT); 1139 return T(word & ELEMENT_MASK); 1140 } 1141 }; 1142 1143 // Ranged integer argument for TemplateArgUnroller 1144 template <uint32_t TMin, uint32_t TMax> 1145 struct IntArg 1146 { 1147 uint32_t val; 1148 }; 1149 1150 // Recursive template used to auto-nest conditionals. Converts dynamic boolean function 1151 // arguments to static template arguments. 1152 template <typename TermT, typename... ArgsB> 1153 struct TemplateArgUnroller 1154 { 1155 //----------------------------------------- 1156 // Boolean value 1157 //----------------------------------------- 1158 1159 // Last Arg Terminator 1160 static typename TermT::FuncType GetFunc(bool bArg) 1161 { 1162 if (bArg) 1163 { 1164 return TermT::template GetFunc<ArgsB..., std::true_type>(); 1165 } 1166 1167 return TermT::template GetFunc<ArgsB..., std::false_type>(); 1168 } 1169 1170 // Recursively parse args 1171 template <typename... TArgsT> 1172 static typename TermT::FuncType GetFunc(bool bArg, TArgsT... remainingArgs) 1173 { 1174 if (bArg) 1175 { 1176 return TemplateArgUnroller<TermT, ArgsB..., std::true_type>::GetFunc(remainingArgs...); 1177 } 1178 1179 return TemplateArgUnroller<TermT, ArgsB..., std::false_type>::GetFunc(remainingArgs...); 1180 } 1181 1182 //----------------------------------------- 1183 // Integer value (within specified range) 1184 //----------------------------------------- 1185 1186 // Last Arg Terminator 1187 template <uint32_t TMin, uint32_t TMax> 1188 static typename TermT::FuncType GetFunc(IntArg<TMin, TMax> iArg) 1189 { 1190 if (iArg.val == TMax) 1191 { 1192 return TermT::template GetFunc<ArgsB..., std::integral_constant<uint32_t, TMax>>(); 1193 } 1194 if (TMax > TMin) 1195 { 1196 return TemplateArgUnroller<TermT, ArgsB...>::GetFunc(IntArg<TMin, TMax-1>{iArg.val}); 1197 } 1198 SWR_ASSUME(false); return nullptr; 1199 } 1200 template <uint32_t TVal> 1201 static typename TermT::FuncType GetFunc(IntArg<TVal, TVal> iArg) 1202 { 1203 SWR_ASSERT(iArg.val == TVal); 1204 return TermT::template GetFunc<ArgsB..., std::integral_constant<uint32_t, TVal>>(); 1205 } 1206 1207 // Recursively parse args 1208 template <uint32_t TMin, uint32_t TMax, typename... TArgsT> 1209 static typename TermT::FuncType GetFunc(IntArg<TMin, TMax> iArg, TArgsT... remainingArgs) 1210 { 1211 if (iArg.val == TMax) 1212 { 1213 return TemplateArgUnroller<TermT, ArgsB..., std::integral_constant<uint32_t, TMax>>::GetFunc(remainingArgs...); 1214 } 1215 if (TMax > TMin) 1216 { 1217 return TemplateArgUnroller<TermT, ArgsB...>::GetFunc(IntArg<TMin, TMax - 1>{iArg.val}, remainingArgs...); 1218 } 1219 SWR_ASSUME(false); return nullptr; 1220 } 1221 template <uint32_t TVal, typename... TArgsT> 1222 static typename TermT::FuncType GetFunc(IntArg<TVal, TVal> iArg, TArgsT... remainingArgs) 1223 { 1224 SWR_ASSERT(iArg.val == TVal); 1225 return TemplateArgUnroller<TermT, ArgsB..., std::integral_constant<uint32_t, TVal>>::GetFunc(remainingArgs...); 1226 } 1227 }; 1228 1229 1230
