1 /*===---- avxintrin.h - AVX intrinsics -------------------------------------=== 2 * 3 * Permission is hereby granted, free of charge, to any person obtaining a copy 4 * of this software and associated documentation files (the "Software"), to deal 5 * in the Software without restriction, including without limitation the rights 6 * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 7 * copies of the Software, and to permit persons to whom the Software is 8 * furnished to do so, subject to the following conditions: 9 * 10 * The above copyright notice and this permission notice shall be included in 11 * all copies or substantial portions of the Software. 12 * 13 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 14 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 15 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 16 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 17 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 18 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 19 * THE SOFTWARE. 20 * 21 *===-----------------------------------------------------------------------=== 22 */ 23 24 #ifndef __IMMINTRIN_H 25 #error "Never use <avxintrin.h> directly; include <immintrin.h> instead." 26 #endif 27 28 typedef double __v4df __attribute__ ((__vector_size__ (32))); 29 typedef float __v8sf __attribute__ ((__vector_size__ (32))); 30 typedef long long __v4di __attribute__ ((__vector_size__ (32))); 31 typedef int __v8si __attribute__ ((__vector_size__ (32))); 32 typedef short __v16hi __attribute__ ((__vector_size__ (32))); 33 typedef char __v32qi __attribute__ ((__vector_size__ (32))); 34 35 typedef float __m256 __attribute__ ((__vector_size__ (32))); 36 typedef double __m256d __attribute__((__vector_size__(32))); 37 typedef long long __m256i __attribute__((__vector_size__(32))); 38 39 /* Arithmetic */ 40 static __inline __m256d __attribute__((__always_inline__, __nodebug__)) 41 _mm256_add_pd(__m256d a, __m256d b) 42 { 43 return a+b; 44 } 45 46 static __inline __m256 __attribute__((__always_inline__, __nodebug__)) 47 _mm256_add_ps(__m256 a, __m256 b) 48 { 49 return a+b; 50 } 51 52 static __inline __m256d __attribute__((__always_inline__, __nodebug__)) 53 _mm256_sub_pd(__m256d a, __m256d b) 54 { 55 return a-b; 56 } 57 58 static __inline __m256 __attribute__((__always_inline__, __nodebug__)) 59 _mm256_sub_ps(__m256 a, __m256 b) 60 { 61 return a-b; 62 } 63 64 static __inline __m256d __attribute__((__always_inline__, __nodebug__)) 65 _mm256_addsub_pd(__m256d a, __m256d b) 66 { 67 return (__m256d)__builtin_ia32_addsubpd256((__v4df)a, (__v4df)b); 68 } 69 70 static __inline __m256 __attribute__((__always_inline__, __nodebug__)) 71 _mm256_addsub_ps(__m256 a, __m256 b) 72 { 73 return (__m256)__builtin_ia32_addsubps256((__v8sf)a, (__v8sf)b); 74 } 75 76 static __inline __m256d __attribute__((__always_inline__, __nodebug__)) 77 _mm256_div_pd(__m256d a, __m256d b) 78 { 79 return a / b; 80 } 81 82 static __inline __m256 __attribute__((__always_inline__, __nodebug__)) 83 _mm256_div_ps(__m256 a, __m256 b) 84 { 85 return a / b; 86 } 87 88 static __inline __m256d __attribute__((__always_inline__, __nodebug__)) 89 _mm256_max_pd(__m256d a, __m256d b) 90 { 91 return (__m256d)__builtin_ia32_maxpd256((__v4df)a, (__v4df)b); 92 } 93 94 static __inline __m256 __attribute__((__always_inline__, __nodebug__)) 95 _mm256_max_ps(__m256 a, __m256 b) 96 { 97 return (__m256)__builtin_ia32_maxps256((__v8sf)a, (__v8sf)b); 98 } 99 100 static __inline __m256d __attribute__((__always_inline__, __nodebug__)) 101 _mm256_min_pd(__m256d a, __m256d b) 102 { 103 return (__m256d)__builtin_ia32_minpd256((__v4df)a, (__v4df)b); 104 } 105 106 static __inline __m256 __attribute__((__always_inline__, __nodebug__)) 107 _mm256_min_ps(__m256 a, __m256 b) 108 { 109 return (__m256)__builtin_ia32_minps256((__v8sf)a, (__v8sf)b); 110 } 111 112 static __inline __m256d __attribute__((__always_inline__, __nodebug__)) 113 _mm256_mul_pd(__m256d a, __m256d b) 114 { 115 return a * b; 116 } 117 118 static __inline __m256 __attribute__((__always_inline__, __nodebug__)) 119 _mm256_mul_ps(__m256 a, __m256 b) 120 { 121 return a * b; 122 } 123 124 static __inline __m256d __attribute__((__always_inline__, __nodebug__)) 125 _mm256_sqrt_pd(__m256d a) 126 { 127 return (__m256d)__builtin_ia32_sqrtpd256((__v4df)a); 128 } 129 130 static __inline __m256 __attribute__((__always_inline__, __nodebug__)) 131 _mm256_sqrt_ps(__m256 a) 132 { 133 return (__m256)__builtin_ia32_sqrtps256((__v8sf)a); 134 } 135 136 static __inline __m256 __attribute__((__always_inline__, __nodebug__)) 137 _mm256_rsqrt_ps(__m256 a) 138 { 139 return (__m256)__builtin_ia32_rsqrtps256((__v8sf)a); 140 } 141 142 static __inline __m256 __attribute__((__always_inline__, __nodebug__)) 143 _mm256_rcp_ps(__m256 a) 144 { 145 return (__m256)__builtin_ia32_rcpps256((__v8sf)a); 146 } 147 148 static __inline __m256d __attribute__((__always_inline__, __nodebug__)) 149 _mm256_round_pd(__m256d v, const int m) 150 { 151 return (__m256d)__builtin_ia32_roundpd256((__v4df)v, m); 152 } 153 154 static __inline __m256 __attribute__((__always_inline__, __nodebug__)) 155 _mm256_round_ps(__m256 v, const int m) 156 { 157 return (__m256)__builtin_ia32_roundps256((__v8sf)v, m); 158 } 159 160 #define _mm256_ceil_pd(V) _mm256_round_pd((V), _MM_FROUND_CEIL) 161 #define _mm256_floor_pd(V) _mm256_round_pd((V), _MM_FROUND_FLOOR) 162 #define _mm256_ceil_ps(V) _mm256_round_ps((V), _MM_FROUND_CEIL) 163 #define _mm256_floor_ps(V) _mm256_round_ps((V), _MM_FROUND_FLOOR) 164 165 /* Logical */ 166 static __inline __m256d __attribute__((__always_inline__, __nodebug__)) 167 _mm256_and_pd(__m256d a, __m256d b) 168 { 169 return (__m256d)((__v4di)a & (__v4di)b); 170 } 171 172 static __inline __m256 __attribute__((__always_inline__, __nodebug__)) 173 _mm256_and_ps(__m256 a, __m256 b) 174 { 175 return (__m256)((__v8si)a & (__v8si)b); 176 } 177 178 static __inline __m256d __attribute__((__always_inline__, __nodebug__)) 179 _mm256_andnot_pd(__m256d a, __m256d b) 180 { 181 return (__m256d)(~(__v4di)a & (__v4di)b); 182 } 183 184 static __inline __m256 __attribute__((__always_inline__, __nodebug__)) 185 _mm256_andnot_ps(__m256 a, __m256 b) 186 { 187 return (__m256)(~(__v8si)a & (__v8si)b); 188 } 189 190 static __inline __m256d __attribute__((__always_inline__, __nodebug__)) 191 _mm256_or_pd(__m256d a, __m256d b) 192 { 193 return (__m256d)((__v4di)a | (__v4di)b); 194 } 195 196 static __inline __m256 __attribute__((__always_inline__, __nodebug__)) 197 _mm256_or_ps(__m256 a, __m256 b) 198 { 199 return (__m256)((__v8si)a | (__v8si)b); 200 } 201 202 static __inline __m256d __attribute__((__always_inline__, __nodebug__)) 203 _mm256_xor_pd(__m256d a, __m256d b) 204 { 205 return (__m256d)((__v4di)a ^ (__v4di)b); 206 } 207 208 static __inline __m256 __attribute__((__always_inline__, __nodebug__)) 209 _mm256_xor_ps(__m256 a, __m256 b) 210 { 211 return (__m256)((__v8si)a ^ (__v8si)b); 212 } 213 214 /* Horizontal arithmetic */ 215 static __inline __m256d __attribute__((__always_inline__, __nodebug__)) 216 _mm256_hadd_pd(__m256d a, __m256d b) 217 { 218 return (__m256d)__builtin_ia32_haddpd256((__v4df)a, (__v4df)b); 219 } 220 221 static __inline __m256 __attribute__((__always_inline__, __nodebug__)) 222 _mm256_hadd_ps(__m256 a, __m256 b) 223 { 224 return (__m256)__builtin_ia32_haddps256((__v8sf)a, (__v8sf)b); 225 } 226 227 static __inline __m256d __attribute__((__always_inline__, __nodebug__)) 228 _mm256_hsub_pd(__m256d a, __m256d b) 229 { 230 return (__m256d)__builtin_ia32_hsubpd256((__v4df)a, (__v4df)b); 231 } 232 233 static __inline __m256 __attribute__((__always_inline__, __nodebug__)) 234 _mm256_hsub_ps(__m256 a, __m256 b) 235 { 236 return (__m256)__builtin_ia32_hsubps256((__v8sf)a, (__v8sf)b); 237 } 238 239 /* Vector permutations */ 240 static __inline __m128d __attribute__((__always_inline__, __nodebug__)) 241 _mm_permutevar_pd(__m128d a, __m128i c) 242 { 243 return (__m128d)__builtin_ia32_vpermilvarpd((__v2df)a, (__v2di)c); 244 } 245 246 static __inline __m256d __attribute__((__always_inline__, __nodebug__)) 247 _mm256_permutevar_pd(__m256d a, __m256i c) 248 { 249 return (__m256d)__builtin_ia32_vpermilvarpd256((__v4df)a, (__v4di)c); 250 } 251 252 static __inline __m128 __attribute__((__always_inline__, __nodebug__)) 253 _mm_permutevar_ps(__m128 a, __m128i c) 254 { 255 return (__m128)__builtin_ia32_vpermilvarps((__v4sf)a, (__v4si)c); 256 } 257 258 static __inline __m256 __attribute__((__always_inline__, __nodebug__)) 259 _mm256_permutevar_ps(__m256 a, __m256i c) 260 { 261 return (__m256)__builtin_ia32_vpermilvarps256((__v8sf)a, 262 (__v8si)c); 263 } 264 265 static __inline __m128d __attribute__((__always_inline__, __nodebug__)) 266 _mm_permute_pd(__m128d a, const int c) 267 { 268 return (__m128d)__builtin_ia32_vpermilpd((__v2df)a, c); 269 } 270 271 static __inline __m256d __attribute__((__always_inline__, __nodebug__)) 272 _mm256_permute_pd(__m256d a, const int c) 273 { 274 return (__m256d)__builtin_ia32_vpermilpd256((__v4df)a, c); 275 } 276 277 static __inline __m128 __attribute__((__always_inline__, __nodebug__)) 278 _mm_permute_ps(__m128 a, const int c) 279 { 280 return (__m128)__builtin_ia32_vpermilps((__v4sf)a, c); 281 } 282 283 static __inline __m256 __attribute__((__always_inline__, __nodebug__)) 284 _mm256_permute_ps(__m256 a, const int c) 285 { 286 return (__m256)__builtin_ia32_vpermilps256((__v8sf)a, c); 287 } 288 289 static __inline __m256d __attribute__((__always_inline__, __nodebug__)) 290 _mm256_permute2f128_pd(__m256d a, __m256d b, const int c) 291 { 292 return (__m256d)__builtin_ia32_vperm2f128_pd256((__v4df)a, (__v4df)b, c); 293 } 294 295 static __inline __m256 __attribute__((__always_inline__, __nodebug__)) 296 _mm256_permute2f128_ps(__m256 a, __m256 b, const int c) 297 { 298 return (__m256)__builtin_ia32_vperm2f128_ps256((__v8sf)a, (__v8sf)b, c); 299 } 300 301 static __inline __m256i __attribute__((__always_inline__, __nodebug__)) 302 _mm256_permute2f128_si256(__m256i a, __m256i b, const int c) 303 { 304 return (__m256i)__builtin_ia32_vperm2f128_si256((__v8si)a, (__v8si)b, c); 305 } 306 307 /* Vector Blend */ 308 static __inline __m256d __attribute__((__always_inline__, __nodebug__)) 309 _mm256_blend_pd(__m256d a, __m256d b, const int c) 310 { 311 return (__m256d)__builtin_ia32_blendpd256((__v4df)a, (__v4df)b, c); 312 } 313 314 static __inline __m256 __attribute__((__always_inline__, __nodebug__)) 315 _mm256_blend_ps(__m256 a, __m256 b, const int c) 316 { 317 return (__m256)__builtin_ia32_blendps256((__v8sf)a, (__v8sf)b, c); 318 } 319 320 static __inline __m256d __attribute__((__always_inline__, __nodebug__)) 321 _mm256_blendv_pd(__m256d a, __m256d b, __m256d c) 322 { 323 return (__m256d)__builtin_ia32_blendvpd256((__v4df)a, (__v4df)b, (__v4df)c); 324 } 325 326 static __inline __m256 __attribute__((__always_inline__, __nodebug__)) 327 _mm256_blendv_ps(__m256 a, __m256 b, __m256 c) 328 { 329 return (__m256)__builtin_ia32_blendvps256((__v8sf)a, (__v8sf)b, (__v8sf)c); 330 } 331 332 /* Vector Dot Product */ 333 static __inline __m256 __attribute__((__always_inline__, __nodebug__)) 334 _mm256_dp_ps(__m256 a, __m256 b, const int c) 335 { 336 return (__m256)__builtin_ia32_dpps256((__v8sf)a, (__v8sf)b, c); 337 } 338 339 /* Vector shuffle */ 340 #define _mm256_shuffle_ps(a, b, mask) \ 341 (__builtin_shufflevector((__v8sf)(a), (__v8sf)(b), \ 342 (mask) & 0x3, ((mask) & 0xc) >> 2, \ 343 (((mask) & 0x30) >> 4) + 8, (((mask) & 0xc0) >> 6) + 8, \ 344 (mask) & 0x3 + 4, (((mask) & 0xc) >> 2) + 4, \ 345 (((mask) & 0x30) >> 4) + 12, (((mask) & 0xc0) >> 6) + 12)) 346 347 #define _mm256_shuffle_pd(a, b, mask) \ 348 (__builtin_shufflevector((__v4df)(a), (__v4df)(b), \ 349 (mask) & 0x1, \ 350 (((mask) & 0x2) >> 1) + 4, \ 351 (((mask) & 0x4) >> 2) + 2, \ 352 (((mask) & 0x8) >> 3) + 6)) 353 354 /* Compare */ 355 #define _CMP_EQ_OQ 0x00 /* Equal (ordered, non-signaling) */ 356 #define _CMP_LT_OS 0x01 /* Less-than (ordered, signaling) */ 357 #define _CMP_LE_OS 0x02 /* Less-than-or-equal (ordered, signaling) */ 358 #define _CMP_UNORD_Q 0x03 /* Unordered (non-signaling) */ 359 #define _CMP_NEQ_UQ 0x04 /* Not-equal (unordered, non-signaling) */ 360 #define _CMP_NLT_US 0x05 /* Not-less-than (unordered, signaling) */ 361 #define _CMP_NLE_US 0x06 /* Not-less-than-or-equal (unordered, signaling) */ 362 #define _CMP_ORD_Q 0x07 /* Ordered (nonsignaling) */ 363 #define _CMP_EQ_UQ 0x08 /* Equal (unordered, non-signaling) */ 364 #define _CMP_NGE_US 0x09 /* Not-greater-than-or-equal (unord, signaling) */ 365 #define _CMP_NGT_US 0x0a /* Not-greater-than (unordered, signaling) */ 366 #define _CMP_FALSE_OQ 0x0b /* False (ordered, non-signaling) */ 367 #define _CMP_NEQ_OQ 0x0c /* Not-equal (ordered, non-signaling) */ 368 #define _CMP_GE_OS 0x0d /* Greater-than-or-equal (ordered, signaling) */ 369 #define _CMP_GT_OS 0x0e /* Greater-than (ordered, signaling) */ 370 #define _CMP_TRUE_UQ 0x0f /* True (unordered, non-signaling) */ 371 #define _CMP_EQ_OS 0x10 /* Equal (ordered, signaling) */ 372 #define _CMP_LT_OQ 0x11 /* Less-than (ordered, non-signaling) */ 373 #define _CMP_LE_OQ 0x12 /* Less-than-or-equal (ordered, non-signaling) */ 374 #define _CMP_UNORD_S 0x13 /* Unordered (signaling) */ 375 #define _CMP_NEQ_US 0x14 /* Not-equal (unordered, signaling) */ 376 #define _CMP_NLT_UQ 0x15 /* Not-less-than (unordered, non-signaling) */ 377 #define _CMP_NLE_UQ 0x16 /* Not-less-than-or-equal (unord, non-signaling) */ 378 #define _CMP_ORD_S 0x17 /* Ordered (signaling) */ 379 #define _CMP_EQ_US 0x18 /* Equal (unordered, signaling) */ 380 #define _CMP_NGE_UQ 0x19 /* Not-greater-than-or-equal (unord, non-sign) */ 381 #define _CMP_NGT_UQ 0x1a /* Not-greater-than (unordered, non-signaling) */ 382 #define _CMP_FALSE_OS 0x1b /* False (ordered, signaling) */ 383 #define _CMP_NEQ_OS 0x1c /* Not-equal (ordered, signaling) */ 384 #define _CMP_GE_OQ 0x1d /* Greater-than-or-equal (ordered, non-signaling) */ 385 #define _CMP_GT_OQ 0x1e /* Greater-than (ordered, non-signaling) */ 386 #define _CMP_TRUE_US 0x1f /* True (unordered, signaling) */ 387 388 static __inline __m128d __attribute__((__always_inline__, __nodebug__)) 389 _mm_cmp_pd(__m128d a, __m128d b, const int c) 390 { 391 return (__m128d)__builtin_ia32_cmppd((__v2df)a, (__v2df)b, c); 392 } 393 394 static __inline __m128 __attribute__((__always_inline__, __nodebug__)) 395 _mm_cmp_ps(__m128 a, __m128 b, const int c) 396 { 397 return (__m128)__builtin_ia32_cmpps((__v4sf)a, (__v4sf)b, c); 398 } 399 400 static __inline __m256d __attribute__((__always_inline__, __nodebug__)) 401 _mm256_cmp_pd(__m256d a, __m256d b, const int c) 402 { 403 return (__m256d)__builtin_ia32_cmppd256((__v4df)a, (__v4df)b, c); 404 } 405 406 static __inline __m256 __attribute__((__always_inline__, __nodebug__)) 407 _mm256_cmp_ps(__m256 a, __m256 b, const int c) 408 { 409 return (__m256)__builtin_ia32_cmpps256((__v8sf)a, (__v8sf)b, c); 410 } 411 412 static __inline __m128d __attribute__((__always_inline__, __nodebug__)) 413 _mm_cmp_sd(__m128d a, __m128d b, const int c) 414 { 415 return (__m128d)__builtin_ia32_cmpsd((__v2df)a, (__v2df)b, c); 416 } 417 418 static __inline __m128 __attribute__((__always_inline__, __nodebug__)) 419 _mm_cmp_ss(__m128 a, __m128 b, const int c) 420 { 421 return (__m128)__builtin_ia32_cmpss((__v4sf)a, (__v4sf)b, c); 422 } 423 424 /* Vector extract */ 425 static __inline __m128d __attribute__((__always_inline__, __nodebug__)) 426 _mm256_extractf128_pd(__m256d a, const int o) 427 { 428 return (__m128d)__builtin_ia32_vextractf128_pd256((__v4df)a, o); 429 } 430 431 static __inline __m128 __attribute__((__always_inline__, __nodebug__)) 432 _mm256_extractf128_ps(__m256 a, const int o) 433 { 434 return (__m128)__builtin_ia32_vextractf128_ps256((__v8sf)a, o); 435 } 436 437 static __inline __m128i __attribute__((__always_inline__, __nodebug__)) 438 _mm256_extractf128_si256(__m256i a, const int o) 439 { 440 return (__m128i)__builtin_ia32_vextractf128_si256((__v8si)a, o); 441 } 442 443 static __inline int __attribute__((__always_inline__, __nodebug__)) 444 _mm256_extract_epi32(__m256i a, int const imm) 445 { 446 __v8si b = (__v8si)a; 447 return b[imm]; 448 } 449 450 static __inline int __attribute__((__always_inline__, __nodebug__)) 451 _mm256_extract_epi16(__m256i a, int const imm) 452 { 453 __v16hi b = (__v16hi)a; 454 return b[imm]; 455 } 456 457 static __inline int __attribute__((__always_inline__, __nodebug__)) 458 _mm256_extract_epi8(__m256i a, int const imm) 459 { 460 __v32qi b = (__v32qi)a; 461 return b[imm]; 462 } 463 464 #ifdef __x86_64__ 465 static __inline long long __attribute__((__always_inline__, __nodebug__)) 466 _mm256_extract_epi64(__m256i a, const int imm) 467 { 468 __v4di b = (__v4di)a; 469 return b[imm]; 470 } 471 #endif 472 473 /* Vector insert */ 474 static __inline __m256d __attribute__((__always_inline__, __nodebug__)) 475 _mm256_insertf128_pd(__m256d a, __m128d b, const int o) 476 { 477 return (__m256d)__builtin_ia32_vinsertf128_pd256((__v4df)a, (__v2df)b, o); 478 } 479 480 static __inline __m256 __attribute__((__always_inline__, __nodebug__)) 481 _mm256_insertf128_ps(__m256 a, __m128 b, const int o) 482 { 483 return (__m256)__builtin_ia32_vinsertf128_ps256((__v8sf)a, (__v4sf)b, o); 484 } 485 486 static __inline __m256i __attribute__((__always_inline__, __nodebug__)) 487 _mm256_insertf128_si256(__m256i a, __m128i b, const int o) 488 { 489 return (__m256i)__builtin_ia32_vinsertf128_si256((__v8si)a, (__v4si)b, o); 490 } 491 492 static __inline __m256i __attribute__((__always_inline__, __nodebug__)) 493 _mm256_insert_epi32(__m256i a, int b, int const imm) 494 { 495 __v8si c = (__v8si)a; 496 c[imm & 7] = b; 497 return (__m256i)c; 498 } 499 500 static __inline __m256i __attribute__((__always_inline__, __nodebug__)) 501 _mm256_insert_epi16(__m256i a, int b, int const imm) 502 { 503 __v16hi c = (__v16hi)a; 504 c[imm & 15] = b; 505 return (__m256i)c; 506 } 507 508 static __inline __m256i __attribute__((__always_inline__, __nodebug__)) 509 _mm256_insert_epi8(__m256i a, int b, int const imm) 510 { 511 __v32qi c = (__v32qi)a; 512 c[imm & 31] = b; 513 return (__m256i)c; 514 } 515 516 #ifdef __x86_64__ 517 static __inline __m256i __attribute__((__always_inline__, __nodebug__)) 518 _mm256_insert_epi64(__m256i a, int b, int const imm) 519 { 520 __v4di c = (__v4di)a; 521 c[imm & 3] = b; 522 return (__m256i)c; 523 } 524 #endif 525 526 /* Conversion */ 527 static __inline __m256d __attribute__((__always_inline__, __nodebug__)) 528 _mm256_cvtepi32_pd(__m128i a) 529 { 530 return (__m256d)__builtin_ia32_cvtdq2pd256((__v4si) a); 531 } 532 533 static __inline __m256 __attribute__((__always_inline__, __nodebug__)) 534 _mm256_cvtepi32_ps(__m256i a) 535 { 536 return (__m256)__builtin_ia32_cvtdq2ps256((__v8si) a); 537 } 538 539 static __inline __m128 __attribute__((__always_inline__, __nodebug__)) 540 _mm256_cvtpd_ps(__m256d a) 541 { 542 return (__m128)__builtin_ia32_cvtpd2ps256((__v4df) a); 543 } 544 545 static __inline __m256i __attribute__((__always_inline__, __nodebug__)) 546 _mm256_cvtps_epi32(__m256 a) 547 { 548 return (__m256i)__builtin_ia32_cvtps2dq256((__v8sf) a); 549 } 550 551 static __inline __m256d __attribute__((__always_inline__, __nodebug__)) 552 _mm256_cvtps_pd(__m128 a) 553 { 554 return (__m256d)__builtin_ia32_cvtps2pd256((__v4sf) a); 555 } 556 557 static __inline __m128i __attribute__((__always_inline__, __nodebug__)) 558 _mm256_cvttpd_epi32(__m256d a) 559 { 560 return (__m128i)__builtin_ia32_cvttpd2dq256((__v4df) a); 561 } 562 563 static __inline __m128i __attribute__((__always_inline__, __nodebug__)) 564 _mm256_cvtpd_epi32(__m256d a) 565 { 566 return (__m128i)__builtin_ia32_cvtpd2dq256((__v4df) a); 567 } 568 569 static __inline __m256i __attribute__((__always_inline__, __nodebug__)) 570 _mm256_cvttps_epi32(__m256 a) 571 { 572 return (__m256i)__builtin_ia32_cvttps2dq256((__v8sf) a); 573 } 574 575 /* Vector replicate */ 576 static __inline __m256 __attribute__((__always_inline__, __nodebug__)) 577 _mm256_movehdup_ps(__m256 a) 578 { 579 return __builtin_shufflevector(a, a, 1, 1, 3, 3, 5, 5, 7, 7); 580 } 581 582 static __inline __m256 __attribute__((__always_inline__, __nodebug__)) 583 _mm256_moveldup_ps(__m256 a) 584 { 585 return __builtin_shufflevector(a, a, 0, 0, 2, 2, 4, 4, 6, 6); 586 } 587 588 static __inline __m256d __attribute__((__always_inline__, __nodebug__)) 589 _mm256_movedup_pd(__m256d a) 590 { 591 return __builtin_shufflevector(a, a, 0, 0, 2, 2); 592 } 593 594 /* Unpack and Interleave */ 595 static __inline __m256d __attribute__((__always_inline__, __nodebug__)) 596 _mm256_unpackhi_pd(__m256d a, __m256d b) 597 { 598 return __builtin_shufflevector(a, b, 1, 5, 1+2, 5+2); 599 } 600 601 static __inline __m256d __attribute__((__always_inline__, __nodebug__)) 602 _mm256_unpacklo_pd(__m256d a, __m256d b) 603 { 604 return __builtin_shufflevector(a, b, 0, 4, 0+2, 4+2); 605 } 606 607 static __inline __m256 __attribute__((__always_inline__, __nodebug__)) 608 _mm256_unpackhi_ps(__m256 a, __m256 b) 609 { 610 return __builtin_shufflevector(a, b, 2, 10, 2+1, 10+1, 6, 14, 6+1, 14+1); 611 } 612 613 static __inline __m256 __attribute__((__always_inline__, __nodebug__)) 614 _mm256_unpacklo_ps(__m256 a, __m256 b) 615 { 616 return __builtin_shufflevector(a, b, 0, 8, 0+1, 8+1, 4, 12, 4+1, 12+1); 617 } 618 619 /* Bit Test */ 620 static __inline int __attribute__((__always_inline__, __nodebug__)) 621 _mm_testz_pd(__m128d a, __m128d b) 622 { 623 return __builtin_ia32_vtestzpd((__v2df)a, (__v2df)b); 624 } 625 626 static __inline int __attribute__((__always_inline__, __nodebug__)) 627 _mm_testc_pd(__m128d a, __m128d b) 628 { 629 return __builtin_ia32_vtestcpd((__v2df)a, (__v2df)b); 630 } 631 632 static __inline int __attribute__((__always_inline__, __nodebug__)) 633 _mm_testnzc_pd(__m128d a, __m128d b) 634 { 635 return __builtin_ia32_vtestnzcpd((__v2df)a, (__v2df)b); 636 } 637 638 static __inline int __attribute__((__always_inline__, __nodebug__)) 639 _mm_testz_ps(__m128 a, __m128 b) 640 { 641 return __builtin_ia32_vtestzps((__v4sf)a, (__v4sf)b); 642 } 643 644 static __inline int __attribute__((__always_inline__, __nodebug__)) 645 _mm_testc_ps(__m128 a, __m128 b) 646 { 647 return __builtin_ia32_vtestcps((__v4sf)a, (__v4sf)b); 648 } 649 650 static __inline int __attribute__((__always_inline__, __nodebug__)) 651 _mm_testnzc_ps(__m128 a, __m128 b) 652 { 653 return __builtin_ia32_vtestnzcps((__v4sf)a, (__v4sf)b); 654 } 655 656 static __inline int __attribute__((__always_inline__, __nodebug__)) 657 _mm256_testz_pd(__m256d a, __m256d b) 658 { 659 return __builtin_ia32_vtestzpd256((__v4df)a, (__v4df)b); 660 } 661 662 static __inline int __attribute__((__always_inline__, __nodebug__)) 663 _mm256_testc_pd(__m256d a, __m256d b) 664 { 665 return __builtin_ia32_vtestcpd256((__v4df)a, (__v4df)b); 666 } 667 668 static __inline int __attribute__((__always_inline__, __nodebug__)) 669 _mm256_testnzc_pd(__m256d a, __m256d b) 670 { 671 return __builtin_ia32_vtestnzcpd256((__v4df)a, (__v4df)b); 672 } 673 674 static __inline int __attribute__((__always_inline__, __nodebug__)) 675 _mm256_testz_ps(__m256 a, __m256 b) 676 { 677 return __builtin_ia32_vtestzps256((__v8sf)a, (__v8sf)b); 678 } 679 680 static __inline int __attribute__((__always_inline__, __nodebug__)) 681 _mm256_testc_ps(__m256 a, __m256 b) 682 { 683 return __builtin_ia32_vtestcps256((__v8sf)a, (__v8sf)b); 684 } 685 686 static __inline int __attribute__((__always_inline__, __nodebug__)) 687 _mm256_testnzc_ps(__m256 a, __m256 b) 688 { 689 return __builtin_ia32_vtestnzcps256((__v8sf)a, (__v8sf)b); 690 } 691 692 static __inline int __attribute__((__always_inline__, __nodebug__)) 693 _mm256_testz_si256(__m256i a, __m256i b) 694 { 695 return __builtin_ia32_ptestz256((__v4di)a, (__v4di)b); 696 } 697 698 static __inline int __attribute__((__always_inline__, __nodebug__)) 699 _mm256_testc_si256(__m256i a, __m256i b) 700 { 701 return __builtin_ia32_ptestc256((__v4di)a, (__v4di)b); 702 } 703 704 static __inline int __attribute__((__always_inline__, __nodebug__)) 705 _mm256_testnzc_si256(__m256i a, __m256i b) 706 { 707 return __builtin_ia32_ptestnzc256((__v4di)a, (__v4di)b); 708 } 709 710 /* Vector extract sign mask */ 711 static __inline int __attribute__((__always_inline__, __nodebug__)) 712 _mm256_movemask_pd(__m256d a) 713 { 714 return __builtin_ia32_movmskpd256((__v4df)a); 715 } 716 717 static __inline int __attribute__((__always_inline__, __nodebug__)) 718 _mm256_movemask_ps(__m256 a) 719 { 720 return __builtin_ia32_movmskps256((__v8sf)a); 721 } 722 723 /* Vector zero */ 724 static __inline void __attribute__((__always_inline__, __nodebug__)) 725 _mm256_zeroall(void) 726 { 727 __builtin_ia32_vzeroall(); 728 } 729 730 static __inline void __attribute__((__always_inline__, __nodebug__)) 731 _mm256_zeroupper(void) 732 { 733 __builtin_ia32_vzeroupper(); 734 } 735 736 /* Vector load with broadcast */ 737 static __inline __m128 __attribute__((__always_inline__, __nodebug__)) 738 _mm_broadcast_ss(float const *a) 739 { 740 return (__m128)__builtin_ia32_vbroadcastss(a); 741 } 742 743 static __inline __m256d __attribute__((__always_inline__, __nodebug__)) 744 _mm256_broadcast_sd(double const *a) 745 { 746 return (__m256d)__builtin_ia32_vbroadcastsd256(a); 747 } 748 749 static __inline __m256 __attribute__((__always_inline__, __nodebug__)) 750 _mm256_broadcast_ss(float const *a) 751 { 752 return (__m256)__builtin_ia32_vbroadcastss256(a); 753 } 754 755 static __inline __m256d __attribute__((__always_inline__, __nodebug__)) 756 _mm256_broadcast_pd(__m128d const *a) 757 { 758 return (__m256d)__builtin_ia32_vbroadcastf128_pd256(a); 759 } 760 761 static __inline __m256 __attribute__((__always_inline__, __nodebug__)) 762 _mm256_broadcast_ps(__m128 const *a) 763 { 764 return (__m256)__builtin_ia32_vbroadcastf128_ps256(a); 765 } 766 767 /* SIMD load ops */ 768 static __inline __m256d __attribute__((__always_inline__, __nodebug__)) 769 _mm256_load_pd(double const *p) 770 { 771 return *(__m256d *)p; 772 } 773 774 static __inline __m256 __attribute__((__always_inline__, __nodebug__)) 775 _mm256_load_ps(float const *p) 776 { 777 return *(__m256 *)p; 778 } 779 780 static __inline __m256d __attribute__((__always_inline__, __nodebug__)) 781 _mm256_loadu_pd(double const *p) 782 { 783 return (__m256d)__builtin_ia32_loadupd256(p); 784 } 785 786 static __inline __m256 __attribute__((__always_inline__, __nodebug__)) 787 _mm256_loadu_ps(float const *p) 788 { 789 return (__m256)__builtin_ia32_loadups256(p); 790 } 791 792 static __inline __m256i __attribute__((__always_inline__, __nodebug__)) 793 _mm256_load_si256(__m256i const *p) 794 { 795 return *p; 796 } 797 798 static __inline __m256i __attribute__((__always_inline__, __nodebug__)) 799 _mm256_loadu_si256(__m256i const *p) 800 { 801 return (__m256i)__builtin_ia32_loaddqu256((char const *)p); 802 } 803 804 static __inline __m256i __attribute__((__always_inline__, __nodebug__)) 805 _mm256_lddqu_si256(__m256i const *p) 806 { 807 return (__m256i)__builtin_ia32_lddqu256((char const *)p); 808 } 809 810 /* SIMD store ops */ 811 static __inline void __attribute__((__always_inline__, __nodebug__)) 812 _mm256_store_pd(double *p, __m256d a) 813 { 814 *(__m256d *)p = a; 815 } 816 817 static __inline void __attribute__((__always_inline__, __nodebug__)) 818 _mm256_store_ps(float *p, __m256 a) 819 { 820 *(__m256 *)p = a; 821 } 822 823 static __inline void __attribute__((__always_inline__, __nodebug__)) 824 _mm256_storeu_pd(double *p, __m256d a) 825 { 826 __builtin_ia32_storeupd256(p, (__v4df)a); 827 } 828 829 static __inline void __attribute__((__always_inline__, __nodebug__)) 830 _mm256_storeu_ps(float *p, __m256 a) 831 { 832 __builtin_ia32_storeups256(p, (__v8sf)a); 833 } 834 835 static __inline void __attribute__((__always_inline__, __nodebug__)) 836 _mm256_store_si256(__m256i *p, __m256i a) 837 { 838 *p = a; 839 } 840 841 static __inline void __attribute__((__always_inline__, __nodebug__)) 842 _mm256_storeu_si256(__m256i *p, __m256i a) 843 { 844 __builtin_ia32_storedqu256((char *)p, (__v32qi)a); 845 } 846 847 /* Conditional load ops */ 848 static __inline __m128d __attribute__((__always_inline__, __nodebug__)) 849 _mm_maskload_pd(double const *p, __m128d m) 850 { 851 return (__m128d)__builtin_ia32_maskloadpd((const __v2df *)p, (__v2df)m); 852 } 853 854 static __inline __m256d __attribute__((__always_inline__, __nodebug__)) 855 _mm256_maskload_pd(double const *p, __m256d m) 856 { 857 return (__m256d)__builtin_ia32_maskloadpd256((const __v4df *)p, (__v4df)m); 858 } 859 860 static __inline __m128 __attribute__((__always_inline__, __nodebug__)) 861 _mm_maskload_ps(float const *p, __m128 m) 862 { 863 return (__m128)__builtin_ia32_maskloadps((const __v4sf *)p, (__v4sf)m); 864 } 865 866 static __inline __m256 __attribute__((__always_inline__, __nodebug__)) 867 _mm256_maskload_ps(float const *p, __m256 m) 868 { 869 return (__m256)__builtin_ia32_maskloadps256((const __v8sf *)p, (__v8sf)m); 870 } 871 872 /* Conditional store ops */ 873 static __inline void __attribute__((__always_inline__, __nodebug__)) 874 _mm256_maskstore_ps(float *p, __m256 m, __m256 a) 875 { 876 __builtin_ia32_maskstoreps256((__v8sf *)p, (__v8sf)m, (__v8sf)a); 877 } 878 879 static __inline void __attribute__((__always_inline__, __nodebug__)) 880 _mm_maskstore_pd(double *p, __m128d m, __m128d a) 881 { 882 __builtin_ia32_maskstorepd((__v2df *)p, (__v2df)m, (__v2df)a); 883 } 884 885 static __inline void __attribute__((__always_inline__, __nodebug__)) 886 _mm256_maskstore_pd(double *p, __m256d m, __m256d a) 887 { 888 __builtin_ia32_maskstorepd256((__v4df *)p, (__v4df)m, (__v4df)a); 889 } 890 891 static __inline void __attribute__((__always_inline__, __nodebug__)) 892 _mm_maskstore_ps(float *p, __m128 m, __m128 a) 893 { 894 __builtin_ia32_maskstoreps((__v4sf *)p, (__v4sf)m, (__v4sf)a); 895 } 896 897 /* Cacheability support ops */ 898 static __inline void __attribute__((__always_inline__, __nodebug__)) 899 _mm256_stream_si256(__m256i *a, __m256i b) 900 { 901 __builtin_ia32_movntdq256((__v4di *)a, (__v4di)b); 902 } 903 904 static __inline void __attribute__((__always_inline__, __nodebug__)) 905 _mm256_stream_pd(double *a, __m256d b) 906 { 907 __builtin_ia32_movntpd256(a, (__v4df)b); 908 } 909 910 static __inline void __attribute__((__always_inline__, __nodebug__)) 911 _mm256_stream_ps(float *p, __m256 a) 912 { 913 __builtin_ia32_movntps256(p, (__v8sf)a); 914 } 915 916 /* Create vectors */ 917 static __inline __m256d __attribute__((__always_inline__, __nodebug__)) 918 _mm256_set_pd(double a, double b, double c, double d) 919 { 920 return (__m256d){ d, c, b, a }; 921 } 922 923 static __inline __m256 __attribute__((__always_inline__, __nodebug__)) 924 _mm256_set_ps(float a, float b, float c, float d, 925 float e, float f, float g, float h) 926 { 927 return (__m256){ h, g, f, e, d, c, b, a }; 928 } 929 930 static __inline __m256i __attribute__((__always_inline__, __nodebug__)) 931 _mm256_set_epi32(int i0, int i1, int i2, int i3, 932 int i4, int i5, int i6, int i7) 933 { 934 return (__m256i)(__v8si){ i7, i6, i5, i4, i3, i2, i1, i0 }; 935 } 936 937 static __inline __m256i __attribute__((__always_inline__, __nodebug__)) 938 _mm256_set_epi16(short w15, short w14, short w13, short w12, 939 short w11, short w10, short w09, short w08, 940 short w07, short w06, short w05, short w04, 941 short w03, short w02, short w01, short w00) 942 { 943 return (__m256i)(__v16hi){ w00, w01, w02, w03, w04, w05, w06, w07, 944 w08, w09, w10, w11, w12, w13, w14, w15 }; 945 } 946 947 static __inline __m256i __attribute__((__always_inline__, __nodebug__)) 948 _mm256_set_epi8(char b31, char b30, char b29, char b28, 949 char b27, char b26, char b25, char b24, 950 char b23, char b22, char b21, char b20, 951 char b19, char b18, char b17, char b16, 952 char b15, char b14, char b13, char b12, 953 char b11, char b10, char b09, char b08, 954 char b07, char b06, char b05, char b04, 955 char b03, char b02, char b01, char b00) 956 { 957 return (__m256i)(__v32qi){ 958 b00, b01, b02, b03, b04, b05, b06, b07, 959 b08, b09, b10, b11, b12, b13, b14, b15, 960 b16, b17, b18, b19, b20, b21, b22, b23, 961 b24, b25, b26, b27, b28, b29, b30, b31 962 }; 963 } 964 965 static __inline __m256i __attribute__((__always_inline__, __nodebug__)) 966 _mm256_set_epi64x(long long a, long long b, long long c, long long d) 967 { 968 return (__m256i)(__v4di){ d, c, b, a }; 969 } 970 971 /* Create vectors with elements in reverse order */ 972 static __inline __m256d __attribute__((__always_inline__, __nodebug__)) 973 _mm256_setr_pd(double a, double b, double c, double d) 974 { 975 return (__m256d){ a, b, c, d }; 976 } 977 978 static __inline __m256 __attribute__((__always_inline__, __nodebug__)) 979 _mm256_setr_ps(float a, float b, float c, float d, 980 float e, float f, float g, float h) 981 { 982 return (__m256){ a, b, c, d, e, f, g, h }; 983 } 984 985 static __inline __m256i __attribute__((__always_inline__, __nodebug__)) 986 _mm256_setr_epi32(int i0, int i1, int i2, int i3, 987 int i4, int i5, int i6, int i7) 988 { 989 return (__m256i)(__v8si){ i0, i1, i2, i3, i4, i5, i6, i7 }; 990 } 991 992 static __inline __m256i __attribute__((__always_inline__, __nodebug__)) 993 _mm256_setr_epi16(short w15, short w14, short w13, short w12, 994 short w11, short w10, short w09, short w08, 995 short w07, short w06, short w05, short w04, 996 short w03, short w02, short w01, short w00) 997 { 998 return (__m256i)(__v16hi){ w15, w14, w13, w12, w11, w10, w09, w08, 999 w07, w06, w05, w04, w03, w02, w01, w00 }; 1000 } 1001 1002 static __inline __m256i __attribute__((__always_inline__, __nodebug__)) 1003 _mm256_setr_epi8(char b31, char b30, char b29, char b28, 1004 char b27, char b26, char b25, char b24, 1005 char b23, char b22, char b21, char b20, 1006 char b19, char b18, char b17, char b16, 1007 char b15, char b14, char b13, char b12, 1008 char b11, char b10, char b09, char b08, 1009 char b07, char b06, char b05, char b04, 1010 char b03, char b02, char b01, char b00) 1011 { 1012 return (__m256i)(__v32qi){ 1013 b31, b30, b29, b28, b27, b26, b25, b24, 1014 b23, b22, b21, b20, b19, b18, b17, b16, 1015 b15, b14, b13, b12, b11, b10, b09, b08, 1016 b07, b06, b05, b04, b03, b02, b01, b00 }; 1017 } 1018 1019 static __inline __m256i __attribute__((__always_inline__, __nodebug__)) 1020 _mm256_setr_epi64x(long long a, long long b, long long c, long long d) 1021 { 1022 return (__m256i)(__v4di){ a, b, c, d }; 1023 } 1024 1025 /* Create vectors with repeated elements */ 1026 static __inline __m256d __attribute__((__always_inline__, __nodebug__)) 1027 _mm256_set1_pd(double w) 1028 { 1029 return (__m256d){ w, w, w, w }; 1030 } 1031 1032 static __inline __m256 __attribute__((__always_inline__, __nodebug__)) 1033 _mm256_set1_ps(float w) 1034 { 1035 return (__m256){ w, w, w, w, w, w, w, w }; 1036 } 1037 1038 static __inline __m256i __attribute__((__always_inline__, __nodebug__)) 1039 _mm256_set1_epi32(int i) 1040 { 1041 return (__m256i)(__v8si){ i, i, i, i, i, i, i, i }; 1042 } 1043 1044 static __inline __m256i __attribute__((__always_inline__, __nodebug__)) 1045 _mm256_set1_epi16(short w) 1046 { 1047 return (__m256i)(__v16hi){ w, w, w, w, w, w, w, w, w, w, w, w, w, w, w, w }; 1048 } 1049 1050 static __inline __m256i __attribute__((__always_inline__, __nodebug__)) 1051 _mm256_set1_epi8(char b) 1052 { 1053 return (__m256i)(__v32qi){ b, b, b, b, b, b, b, b, b, b, b, b, b, b, b, b, 1054 b, b, b, b, b, b, b, b, b, b, b, b, b, b, b, b }; 1055 } 1056 1057 static __inline __m256i __attribute__((__always_inline__, __nodebug__)) 1058 _mm256_set1_epi64x(long long q) 1059 { 1060 return (__m256i)(__v4di){ q, q, q, q }; 1061 } 1062 1063 /* Create zeroed vectors */ 1064 static __inline __m256d __attribute__((__always_inline__, __nodebug__)) 1065 _mm256_setzero_pd(void) 1066 { 1067 return (__m256d){ 0, 0, 0, 0 }; 1068 } 1069 1070 static __inline __m256 __attribute__((__always_inline__, __nodebug__)) 1071 _mm256_setzero_ps(void) 1072 { 1073 return (__m256){ 0, 0, 0, 0, 0, 0, 0, 0 }; 1074 } 1075 1076 static __inline __m256i __attribute__((__always_inline__, __nodebug__)) 1077 _mm256_setzero_si256(void) 1078 { 1079 return (__m256i){ 0LL, 0LL, 0LL, 0LL }; 1080 } 1081 1082 /* Cast between vector types */ 1083 static __inline __m256 __attribute__((__always_inline__, __nodebug__)) 1084 _mm256_castpd_ps(__m256d in) 1085 { 1086 return (__m256)in; 1087 } 1088 1089 static __inline __m256i __attribute__((__always_inline__, __nodebug__)) 1090 _mm256_castpd_si256(__m256d in) 1091 { 1092 return (__m256i)in; 1093 } 1094 1095 static __inline __m256d __attribute__((__always_inline__, __nodebug__)) 1096 _mm256_castps_pd(__m256 in) 1097 { 1098 return (__m256d)in; 1099 } 1100 1101 static __inline __m256i __attribute__((__always_inline__, __nodebug__)) 1102 _mm256_castps_si256(__m256 in) 1103 { 1104 return (__m256i)in; 1105 } 1106 1107 static __inline __m256 __attribute__((__always_inline__, __nodebug__)) 1108 _mm256_castsi256_ps(__m256i in) 1109 { 1110 return (__m256)in; 1111 } 1112 1113 static __inline __m256d __attribute__((__always_inline__, __nodebug__)) 1114 _mm256_castsi256_pd(__m256i in) 1115 { 1116 return (__m256d)in; 1117 } 1118 1119 static __inline __m128d __attribute__((__always_inline__, __nodebug__)) 1120 _mm256_castpd256_pd128(__m256d in) 1121 { 1122 return __builtin_shufflevector(in, in, 0, 1); 1123 } 1124 1125 static __inline __m128 __attribute__((__always_inline__, __nodebug__)) 1126 _mm256_castps256_ps128(__m256 in) 1127 { 1128 return __builtin_shufflevector(in, in, 0, 1, 2, 3); 1129 } 1130 1131 static __inline __m128i __attribute__((__always_inline__, __nodebug__)) 1132 _mm256_castsi256_si128(__m256i in) 1133 { 1134 return __builtin_shufflevector(in, in, 0, 1); 1135 } 1136 1137 static __inline __m256d __attribute__((__always_inline__, __nodebug__)) 1138 _mm256_castpd128_pd256(__m128d in) 1139 { 1140 __m128d zero = _mm_setzero_pd(); 1141 return __builtin_shufflevector(in, zero, 0, 1, 2, 2); 1142 } 1143 1144 static __inline __m256 __attribute__((__always_inline__, __nodebug__)) 1145 _mm256_castps128_ps256(__m128 in) 1146 { 1147 __m128 zero = _mm_setzero_ps(); 1148 return __builtin_shufflevector(in, zero, 0, 1, 2, 3, 4, 4, 4, 4); 1149 } 1150 1151 static __inline __m256i __attribute__((__always_inline__, __nodebug__)) 1152 _mm256_castsi128_si256(__m128i in) 1153 { 1154 __m128i zero = _mm_setzero_si128(); 1155 return __builtin_shufflevector(in, zero, 0, 1, 2, 2); 1156 } 1157
