1 // This file is part of Eigen, a lightweight C++ template library 2 // for linear algebra. 3 // 4 // Copyright (C) 2014 Benoit Steiner (benoit.steiner.goog (at) gmail.com) 5 // 6 // This Source Code Form is subject to the terms of the Mozilla 7 // Public License v. 2.0. If a copy of the MPL was not distributed 8 // with this file, You can obtain one at http://mozilla.org/MPL/2.0/. 9 10 #ifndef EIGEN_COMPLEX_AVX_H 11 #define EIGEN_COMPLEX_AVX_H 12 13 namespace Eigen { 14 15 namespace internal { 16 17 //---------- float ---------- 18 struct Packet4cf 19 { 20 EIGEN_STRONG_INLINE Packet4cf() {} 21 EIGEN_STRONG_INLINE explicit Packet4cf(const __m256& a) : v(a) {} 22 __m256 v; 23 }; 24 25 template<> struct packet_traits<std::complex<float> > : default_packet_traits 26 { 27 typedef Packet4cf type; 28 typedef Packet2cf half; 29 enum { 30 Vectorizable = 1, 31 AlignedOnScalar = 1, 32 size = 4, 33 HasHalfPacket = 1, 34 35 HasAdd = 1, 36 HasSub = 1, 37 HasMul = 1, 38 HasDiv = 1, 39 HasNegate = 1, 40 HasAbs = 0, 41 HasAbs2 = 0, 42 HasMin = 0, 43 HasMax = 0, 44 HasSetLinear = 0 45 }; 46 }; 47 48 template<> struct unpacket_traits<Packet4cf> { typedef std::complex<float> type; enum {size=4, alignment=Aligned32}; typedef Packet2cf half; }; 49 50 template<> EIGEN_STRONG_INLINE Packet4cf padd<Packet4cf>(const Packet4cf& a, const Packet4cf& b) { return Packet4cf(_mm256_add_ps(a.v,b.v)); } 51 template<> EIGEN_STRONG_INLINE Packet4cf psub<Packet4cf>(const Packet4cf& a, const Packet4cf& b) { return Packet4cf(_mm256_sub_ps(a.v,b.v)); } 52 template<> EIGEN_STRONG_INLINE Packet4cf pnegate(const Packet4cf& a) 53 { 54 return Packet4cf(pnegate(a.v)); 55 } 56 template<> EIGEN_STRONG_INLINE Packet4cf pconj(const Packet4cf& a) 57 { 58 const __m256 mask = _mm256_castsi256_ps(_mm256_setr_epi32(0x00000000,0x80000000,0x00000000,0x80000000,0x00000000,0x80000000,0x00000000,0x80000000)); 59 return Packet4cf(_mm256_xor_ps(a.v,mask)); 60 } 61 62 template<> EIGEN_STRONG_INLINE Packet4cf pmul<Packet4cf>(const Packet4cf& a, const Packet4cf& b) 63 { 64 __m256 tmp1 = _mm256_mul_ps(_mm256_moveldup_ps(a.v), b.v); 65 __m256 tmp2 = _mm256_mul_ps(_mm256_movehdup_ps(a.v), _mm256_permute_ps(b.v, _MM_SHUFFLE(2,3,0,1))); 66 __m256 result = _mm256_addsub_ps(tmp1, tmp2); 67 return Packet4cf(result); 68 } 69 70 template<> EIGEN_STRONG_INLINE Packet4cf pand <Packet4cf>(const Packet4cf& a, const Packet4cf& b) { return Packet4cf(_mm256_and_ps(a.v,b.v)); } 71 template<> EIGEN_STRONG_INLINE Packet4cf por <Packet4cf>(const Packet4cf& a, const Packet4cf& b) { return Packet4cf(_mm256_or_ps(a.v,b.v)); } 72 template<> EIGEN_STRONG_INLINE Packet4cf pxor <Packet4cf>(const Packet4cf& a, const Packet4cf& b) { return Packet4cf(_mm256_xor_ps(a.v,b.v)); } 73 template<> EIGEN_STRONG_INLINE Packet4cf pandnot<Packet4cf>(const Packet4cf& a, const Packet4cf& b) { return Packet4cf(_mm256_andnot_ps(a.v,b.v)); } 74 75 template<> EIGEN_STRONG_INLINE Packet4cf pload <Packet4cf>(const std::complex<float>* from) { EIGEN_DEBUG_ALIGNED_LOAD return Packet4cf(pload<Packet8f>(&numext::real_ref(*from))); } 76 template<> EIGEN_STRONG_INLINE Packet4cf ploadu<Packet4cf>(const std::complex<float>* from) { EIGEN_DEBUG_UNALIGNED_LOAD return Packet4cf(ploadu<Packet8f>(&numext::real_ref(*from))); } 77 78 79 template<> EIGEN_STRONG_INLINE Packet4cf pset1<Packet4cf>(const std::complex<float>& from) 80 { 81 return Packet4cf(_mm256_castpd_ps(_mm256_broadcast_sd((const double*)(const void*)&from))); 82 } 83 84 template<> EIGEN_STRONG_INLINE Packet4cf ploaddup<Packet4cf>(const std::complex<float>* from) 85 { 86 // FIXME The following might be optimized using _mm256_movedup_pd 87 Packet2cf a = ploaddup<Packet2cf>(from); 88 Packet2cf b = ploaddup<Packet2cf>(from+1); 89 return Packet4cf(_mm256_insertf128_ps(_mm256_castps128_ps256(a.v), b.v, 1)); 90 } 91 92 template<> EIGEN_STRONG_INLINE void pstore <std::complex<float> >(std::complex<float>* to, const Packet4cf& from) { EIGEN_DEBUG_ALIGNED_STORE pstore(&numext::real_ref(*to), from.v); } 93 template<> EIGEN_STRONG_INLINE void pstoreu<std::complex<float> >(std::complex<float>* to, const Packet4cf& from) { EIGEN_DEBUG_UNALIGNED_STORE pstoreu(&numext::real_ref(*to), from.v); } 94 95 template<> EIGEN_DEVICE_FUNC inline Packet4cf pgather<std::complex<float>, Packet4cf>(const std::complex<float>* from, Index stride) 96 { 97 return Packet4cf(_mm256_set_ps(std::imag(from[3*stride]), std::real(from[3*stride]), 98 std::imag(from[2*stride]), std::real(from[2*stride]), 99 std::imag(from[1*stride]), std::real(from[1*stride]), 100 std::imag(from[0*stride]), std::real(from[0*stride]))); 101 } 102 103 template<> EIGEN_DEVICE_FUNC inline void pscatter<std::complex<float>, Packet4cf>(std::complex<float>* to, const Packet4cf& from, Index stride) 104 { 105 __m128 low = _mm256_extractf128_ps(from.v, 0); 106 to[stride*0] = std::complex<float>(_mm_cvtss_f32(_mm_shuffle_ps(low, low, 0)), 107 _mm_cvtss_f32(_mm_shuffle_ps(low, low, 1))); 108 to[stride*1] = std::complex<float>(_mm_cvtss_f32(_mm_shuffle_ps(low, low, 2)), 109 _mm_cvtss_f32(_mm_shuffle_ps(low, low, 3))); 110 111 __m128 high = _mm256_extractf128_ps(from.v, 1); 112 to[stride*2] = std::complex<float>(_mm_cvtss_f32(_mm_shuffle_ps(high, high, 0)), 113 _mm_cvtss_f32(_mm_shuffle_ps(high, high, 1))); 114 to[stride*3] = std::complex<float>(_mm_cvtss_f32(_mm_shuffle_ps(high, high, 2)), 115 _mm_cvtss_f32(_mm_shuffle_ps(high, high, 3))); 116 117 } 118 119 template<> EIGEN_STRONG_INLINE std::complex<float> pfirst<Packet4cf>(const Packet4cf& a) 120 { 121 return pfirst(Packet2cf(_mm256_castps256_ps128(a.v))); 122 } 123 124 template<> EIGEN_STRONG_INLINE Packet4cf preverse(const Packet4cf& a) { 125 __m128 low = _mm256_extractf128_ps(a.v, 0); 126 __m128 high = _mm256_extractf128_ps(a.v, 1); 127 __m128d lowd = _mm_castps_pd(low); 128 __m128d highd = _mm_castps_pd(high); 129 low = _mm_castpd_ps(_mm_shuffle_pd(lowd,lowd,0x1)); 130 high = _mm_castpd_ps(_mm_shuffle_pd(highd,highd,0x1)); 131 __m256 result = _mm256_setzero_ps(); 132 result = _mm256_insertf128_ps(result, low, 1); 133 result = _mm256_insertf128_ps(result, high, 0); 134 return Packet4cf(result); 135 } 136 137 template<> EIGEN_STRONG_INLINE std::complex<float> predux<Packet4cf>(const Packet4cf& a) 138 { 139 return predux(padd(Packet2cf(_mm256_extractf128_ps(a.v,0)), 140 Packet2cf(_mm256_extractf128_ps(a.v,1)))); 141 } 142 143 template<> EIGEN_STRONG_INLINE Packet4cf preduxp<Packet4cf>(const Packet4cf* vecs) 144 { 145 Packet8f t0 = _mm256_shuffle_ps(vecs[0].v, vecs[0].v, _MM_SHUFFLE(3, 1, 2 ,0)); 146 Packet8f t1 = _mm256_shuffle_ps(vecs[1].v, vecs[1].v, _MM_SHUFFLE(3, 1, 2 ,0)); 147 t0 = _mm256_hadd_ps(t0,t1); 148 Packet8f t2 = _mm256_shuffle_ps(vecs[2].v, vecs[2].v, _MM_SHUFFLE(3, 1, 2 ,0)); 149 Packet8f t3 = _mm256_shuffle_ps(vecs[3].v, vecs[3].v, _MM_SHUFFLE(3, 1, 2 ,0)); 150 t2 = _mm256_hadd_ps(t2,t3); 151 152 t1 = _mm256_permute2f128_ps(t0,t2, 0 + (2<<4)); 153 t3 = _mm256_permute2f128_ps(t0,t2, 1 + (3<<4)); 154 155 return Packet4cf(_mm256_add_ps(t1,t3)); 156 } 157 158 template<> EIGEN_STRONG_INLINE std::complex<float> predux_mul<Packet4cf>(const Packet4cf& a) 159 { 160 return predux_mul(pmul(Packet2cf(_mm256_extractf128_ps(a.v, 0)), 161 Packet2cf(_mm256_extractf128_ps(a.v, 1)))); 162 } 163 164 template<int Offset> 165 struct palign_impl<Offset,Packet4cf> 166 { 167 static EIGEN_STRONG_INLINE void run(Packet4cf& first, const Packet4cf& second) 168 { 169 if (Offset==0) return; 170 palign_impl<Offset*2,Packet8f>::run(first.v, second.v); 171 } 172 }; 173 174 template<> struct conj_helper<Packet4cf, Packet4cf, false,true> 175 { 176 EIGEN_STRONG_INLINE Packet4cf pmadd(const Packet4cf& x, const Packet4cf& y, const Packet4cf& c) const 177 { return padd(pmul(x,y),c); } 178 179 EIGEN_STRONG_INLINE Packet4cf pmul(const Packet4cf& a, const Packet4cf& b) const 180 { 181 return internal::pmul(a, pconj(b)); 182 } 183 }; 184 185 template<> struct conj_helper<Packet4cf, Packet4cf, true,false> 186 { 187 EIGEN_STRONG_INLINE Packet4cf pmadd(const Packet4cf& x, const Packet4cf& y, const Packet4cf& c) const 188 { return padd(pmul(x,y),c); } 189 190 EIGEN_STRONG_INLINE Packet4cf pmul(const Packet4cf& a, const Packet4cf& b) const 191 { 192 return internal::pmul(pconj(a), b); 193 } 194 }; 195 196 template<> struct conj_helper<Packet4cf, Packet4cf, true,true> 197 { 198 EIGEN_STRONG_INLINE Packet4cf pmadd(const Packet4cf& x, const Packet4cf& y, const Packet4cf& c) const 199 { return padd(pmul(x,y),c); } 200 201 EIGEN_STRONG_INLINE Packet4cf pmul(const Packet4cf& a, const Packet4cf& b) const 202 { 203 return pconj(internal::pmul(a, b)); 204 } 205 }; 206 207 template<> struct conj_helper<Packet8f, Packet4cf, false,false> 208 { 209 EIGEN_STRONG_INLINE Packet4cf pmadd(const Packet8f& x, const Packet4cf& y, const Packet4cf& c) const 210 { return padd(c, pmul(x,y)); } 211 212 EIGEN_STRONG_INLINE Packet4cf pmul(const Packet8f& x, const Packet4cf& y) const 213 { return Packet4cf(Eigen::internal::pmul(x, y.v)); } 214 }; 215 216 template<> struct conj_helper<Packet4cf, Packet8f, false,false> 217 { 218 EIGEN_STRONG_INLINE Packet4cf pmadd(const Packet4cf& x, const Packet8f& y, const Packet4cf& c) const 219 { return padd(c, pmul(x,y)); } 220 221 EIGEN_STRONG_INLINE Packet4cf pmul(const Packet4cf& x, const Packet8f& y) const 222 { return Packet4cf(Eigen::internal::pmul(x.v, y)); } 223 }; 224 225 template<> EIGEN_STRONG_INLINE Packet4cf pdiv<Packet4cf>(const Packet4cf& a, const Packet4cf& b) 226 { 227 Packet4cf num = pmul(a, pconj(b)); 228 __m256 tmp = _mm256_mul_ps(b.v, b.v); 229 __m256 tmp2 = _mm256_shuffle_ps(tmp,tmp,0xB1); 230 __m256 denom = _mm256_add_ps(tmp, tmp2); 231 return Packet4cf(_mm256_div_ps(num.v, denom)); 232 } 233 234 template<> EIGEN_STRONG_INLINE Packet4cf pcplxflip<Packet4cf>(const Packet4cf& x) 235 { 236 return Packet4cf(_mm256_shuffle_ps(x.v, x.v, _MM_SHUFFLE(2, 3, 0 ,1))); 237 } 238 239 //---------- double ---------- 240 struct Packet2cd 241 { 242 EIGEN_STRONG_INLINE Packet2cd() {} 243 EIGEN_STRONG_INLINE explicit Packet2cd(const __m256d& a) : v(a) {} 244 __m256d v; 245 }; 246 247 template<> struct packet_traits<std::complex<double> > : default_packet_traits 248 { 249 typedef Packet2cd type; 250 typedef Packet1cd half; 251 enum { 252 Vectorizable = 1, 253 AlignedOnScalar = 0, 254 size = 2, 255 HasHalfPacket = 1, 256 257 HasAdd = 1, 258 HasSub = 1, 259 HasMul = 1, 260 HasDiv = 1, 261 HasNegate = 1, 262 HasAbs = 0, 263 HasAbs2 = 0, 264 HasMin = 0, 265 HasMax = 0, 266 HasSetLinear = 0 267 }; 268 }; 269 270 template<> struct unpacket_traits<Packet2cd> { typedef std::complex<double> type; enum {size=2, alignment=Aligned32}; typedef Packet1cd half; }; 271 272 template<> EIGEN_STRONG_INLINE Packet2cd padd<Packet2cd>(const Packet2cd& a, const Packet2cd& b) { return Packet2cd(_mm256_add_pd(a.v,b.v)); } 273 template<> EIGEN_STRONG_INLINE Packet2cd psub<Packet2cd>(const Packet2cd& a, const Packet2cd& b) { return Packet2cd(_mm256_sub_pd(a.v,b.v)); } 274 template<> EIGEN_STRONG_INLINE Packet2cd pnegate(const Packet2cd& a) { return Packet2cd(pnegate(a.v)); } 275 template<> EIGEN_STRONG_INLINE Packet2cd pconj(const Packet2cd& a) 276 { 277 const __m256d mask = _mm256_castsi256_pd(_mm256_set_epi32(0x80000000,0x0,0x0,0x0,0x80000000,0x0,0x0,0x0)); 278 return Packet2cd(_mm256_xor_pd(a.v,mask)); 279 } 280 281 template<> EIGEN_STRONG_INLINE Packet2cd pmul<Packet2cd>(const Packet2cd& a, const Packet2cd& b) 282 { 283 __m256d tmp1 = _mm256_shuffle_pd(a.v,a.v,0x0); 284 __m256d even = _mm256_mul_pd(tmp1, b.v); 285 __m256d tmp2 = _mm256_shuffle_pd(a.v,a.v,0xF); 286 __m256d tmp3 = _mm256_shuffle_pd(b.v,b.v,0x5); 287 __m256d odd = _mm256_mul_pd(tmp2, tmp3); 288 return Packet2cd(_mm256_addsub_pd(even, odd)); 289 } 290 291 template<> EIGEN_STRONG_INLINE Packet2cd pand <Packet2cd>(const Packet2cd& a, const Packet2cd& b) { return Packet2cd(_mm256_and_pd(a.v,b.v)); } 292 template<> EIGEN_STRONG_INLINE Packet2cd por <Packet2cd>(const Packet2cd& a, const Packet2cd& b) { return Packet2cd(_mm256_or_pd(a.v,b.v)); } 293 template<> EIGEN_STRONG_INLINE Packet2cd pxor <Packet2cd>(const Packet2cd& a, const Packet2cd& b) { return Packet2cd(_mm256_xor_pd(a.v,b.v)); } 294 template<> EIGEN_STRONG_INLINE Packet2cd pandnot<Packet2cd>(const Packet2cd& a, const Packet2cd& b) { return Packet2cd(_mm256_andnot_pd(a.v,b.v)); } 295 296 template<> EIGEN_STRONG_INLINE Packet2cd pload <Packet2cd>(const std::complex<double>* from) 297 { EIGEN_DEBUG_ALIGNED_LOAD return Packet2cd(pload<Packet4d>((const double*)from)); } 298 template<> EIGEN_STRONG_INLINE Packet2cd ploadu<Packet2cd>(const std::complex<double>* from) 299 { EIGEN_DEBUG_UNALIGNED_LOAD return Packet2cd(ploadu<Packet4d>((const double*)from)); } 300 301 template<> EIGEN_STRONG_INLINE Packet2cd pset1<Packet2cd>(const std::complex<double>& from) 302 { 303 // in case casting to a __m128d* is really not safe, then we can still fallback to this version: (much slower though) 304 // return Packet2cd(_mm256_loadu2_m128d((const double*)&from,(const double*)&from)); 305 return Packet2cd(_mm256_broadcast_pd((const __m128d*)(const void*)&from)); 306 } 307 308 template<> EIGEN_STRONG_INLINE Packet2cd ploaddup<Packet2cd>(const std::complex<double>* from) { return pset1<Packet2cd>(*from); } 309 310 template<> EIGEN_STRONG_INLINE void pstore <std::complex<double> >(std::complex<double> * to, const Packet2cd& from) { EIGEN_DEBUG_ALIGNED_STORE pstore((double*)to, from.v); } 311 template<> EIGEN_STRONG_INLINE void pstoreu<std::complex<double> >(std::complex<double> * to, const Packet2cd& from) { EIGEN_DEBUG_UNALIGNED_STORE pstoreu((double*)to, from.v); } 312 313 template<> EIGEN_DEVICE_FUNC inline Packet2cd pgather<std::complex<double>, Packet2cd>(const std::complex<double>* from, Index stride) 314 { 315 return Packet2cd(_mm256_set_pd(std::imag(from[1*stride]), std::real(from[1*stride]), 316 std::imag(from[0*stride]), std::real(from[0*stride]))); 317 } 318 319 template<> EIGEN_DEVICE_FUNC inline void pscatter<std::complex<double>, Packet2cd>(std::complex<double>* to, const Packet2cd& from, Index stride) 320 { 321 __m128d low = _mm256_extractf128_pd(from.v, 0); 322 to[stride*0] = std::complex<double>(_mm_cvtsd_f64(low), _mm_cvtsd_f64(_mm_shuffle_pd(low, low, 1))); 323 __m128d high = _mm256_extractf128_pd(from.v, 1); 324 to[stride*1] = std::complex<double>(_mm_cvtsd_f64(high), _mm_cvtsd_f64(_mm_shuffle_pd(high, high, 1))); 325 } 326 327 template<> EIGEN_STRONG_INLINE std::complex<double> pfirst<Packet2cd>(const Packet2cd& a) 328 { 329 __m128d low = _mm256_extractf128_pd(a.v, 0); 330 EIGEN_ALIGN16 double res[2]; 331 _mm_store_pd(res, low); 332 return std::complex<double>(res[0],res[1]); 333 } 334 335 template<> EIGEN_STRONG_INLINE Packet2cd preverse(const Packet2cd& a) { 336 __m256d result = _mm256_permute2f128_pd(a.v, a.v, 1); 337 return Packet2cd(result); 338 } 339 340 template<> EIGEN_STRONG_INLINE std::complex<double> predux<Packet2cd>(const Packet2cd& a) 341 { 342 return predux(padd(Packet1cd(_mm256_extractf128_pd(a.v,0)), 343 Packet1cd(_mm256_extractf128_pd(a.v,1)))); 344 } 345 346 template<> EIGEN_STRONG_INLINE Packet2cd preduxp<Packet2cd>(const Packet2cd* vecs) 347 { 348 Packet4d t0 = _mm256_permute2f128_pd(vecs[0].v,vecs[1].v, 0 + (2<<4)); 349 Packet4d t1 = _mm256_permute2f128_pd(vecs[0].v,vecs[1].v, 1 + (3<<4)); 350 351 return Packet2cd(_mm256_add_pd(t0,t1)); 352 } 353 354 template<> EIGEN_STRONG_INLINE std::complex<double> predux_mul<Packet2cd>(const Packet2cd& a) 355 { 356 return predux(pmul(Packet1cd(_mm256_extractf128_pd(a.v,0)), 357 Packet1cd(_mm256_extractf128_pd(a.v,1)))); 358 } 359 360 template<int Offset> 361 struct palign_impl<Offset,Packet2cd> 362 { 363 static EIGEN_STRONG_INLINE void run(Packet2cd& first, const Packet2cd& second) 364 { 365 if (Offset==0) return; 366 palign_impl<Offset*2,Packet4d>::run(first.v, second.v); 367 } 368 }; 369 370 template<> struct conj_helper<Packet2cd, Packet2cd, false,true> 371 { 372 EIGEN_STRONG_INLINE Packet2cd pmadd(const Packet2cd& x, const Packet2cd& y, const Packet2cd& c) const 373 { return padd(pmul(x,y),c); } 374 375 EIGEN_STRONG_INLINE Packet2cd pmul(const Packet2cd& a, const Packet2cd& b) const 376 { 377 return internal::pmul(a, pconj(b)); 378 } 379 }; 380 381 template<> struct conj_helper<Packet2cd, Packet2cd, true,false> 382 { 383 EIGEN_STRONG_INLINE Packet2cd pmadd(const Packet2cd& x, const Packet2cd& y, const Packet2cd& c) const 384 { return padd(pmul(x,y),c); } 385 386 EIGEN_STRONG_INLINE Packet2cd pmul(const Packet2cd& a, const Packet2cd& b) const 387 { 388 return internal::pmul(pconj(a), b); 389 } 390 }; 391 392 template<> struct conj_helper<Packet2cd, Packet2cd, true,true> 393 { 394 EIGEN_STRONG_INLINE Packet2cd pmadd(const Packet2cd& x, const Packet2cd& y, const Packet2cd& c) const 395 { return padd(pmul(x,y),c); } 396 397 EIGEN_STRONG_INLINE Packet2cd pmul(const Packet2cd& a, const Packet2cd& b) const 398 { 399 return pconj(internal::pmul(a, b)); 400 } 401 }; 402 403 template<> struct conj_helper<Packet4d, Packet2cd, false,false> 404 { 405 EIGEN_STRONG_INLINE Packet2cd pmadd(const Packet4d& x, const Packet2cd& y, const Packet2cd& c) const 406 { return padd(c, pmul(x,y)); } 407 408 EIGEN_STRONG_INLINE Packet2cd pmul(const Packet4d& x, const Packet2cd& y) const 409 { return Packet2cd(Eigen::internal::pmul(x, y.v)); } 410 }; 411 412 template<> struct conj_helper<Packet2cd, Packet4d, false,false> 413 { 414 EIGEN_STRONG_INLINE Packet2cd pmadd(const Packet2cd& x, const Packet4d& y, const Packet2cd& c) const 415 { return padd(c, pmul(x,y)); } 416 417 EIGEN_STRONG_INLINE Packet2cd pmul(const Packet2cd& x, const Packet4d& y) const 418 { return Packet2cd(Eigen::internal::pmul(x.v, y)); } 419 }; 420 421 template<> EIGEN_STRONG_INLINE Packet2cd pdiv<Packet2cd>(const Packet2cd& a, const Packet2cd& b) 422 { 423 Packet2cd num = pmul(a, pconj(b)); 424 __m256d tmp = _mm256_mul_pd(b.v, b.v); 425 __m256d denom = _mm256_hadd_pd(tmp, tmp); 426 return Packet2cd(_mm256_div_pd(num.v, denom)); 427 } 428 429 template<> EIGEN_STRONG_INLINE Packet2cd pcplxflip<Packet2cd>(const Packet2cd& x) 430 { 431 return Packet2cd(_mm256_shuffle_pd(x.v, x.v, 0x5)); 432 } 433 434 EIGEN_DEVICE_FUNC inline void 435 ptranspose(PacketBlock<Packet4cf,4>& kernel) { 436 __m256d P0 = _mm256_castps_pd(kernel.packet[0].v); 437 __m256d P1 = _mm256_castps_pd(kernel.packet[1].v); 438 __m256d P2 = _mm256_castps_pd(kernel.packet[2].v); 439 __m256d P3 = _mm256_castps_pd(kernel.packet[3].v); 440 441 __m256d T0 = _mm256_shuffle_pd(P0, P1, 15); 442 __m256d T1 = _mm256_shuffle_pd(P0, P1, 0); 443 __m256d T2 = _mm256_shuffle_pd(P2, P3, 15); 444 __m256d T3 = _mm256_shuffle_pd(P2, P3, 0); 445 446 kernel.packet[1].v = _mm256_castpd_ps(_mm256_permute2f128_pd(T0, T2, 32)); 447 kernel.packet[3].v = _mm256_castpd_ps(_mm256_permute2f128_pd(T0, T2, 49)); 448 kernel.packet[0].v = _mm256_castpd_ps(_mm256_permute2f128_pd(T1, T3, 32)); 449 kernel.packet[2].v = _mm256_castpd_ps(_mm256_permute2f128_pd(T1, T3, 49)); 450 } 451 452 EIGEN_DEVICE_FUNC inline void 453 ptranspose(PacketBlock<Packet2cd,2>& kernel) { 454 __m256d tmp = _mm256_permute2f128_pd(kernel.packet[0].v, kernel.packet[1].v, 0+(2<<4)); 455 kernel.packet[1].v = _mm256_permute2f128_pd(kernel.packet[0].v, kernel.packet[1].v, 1+(3<<4)); 456 kernel.packet[0].v = tmp; 457 } 458 459 template<> EIGEN_STRONG_INLINE Packet4cf pinsertfirst(const Packet4cf& a, std::complex<float> b) 460 { 461 return Packet4cf(_mm256_blend_ps(a.v,pset1<Packet4cf>(b).v,1|2)); 462 } 463 464 template<> EIGEN_STRONG_INLINE Packet2cd pinsertfirst(const Packet2cd& a, std::complex<double> b) 465 { 466 return Packet2cd(_mm256_blend_pd(a.v,pset1<Packet2cd>(b).v,1|2)); 467 } 468 469 template<> EIGEN_STRONG_INLINE Packet4cf pinsertlast(const Packet4cf& a, std::complex<float> b) 470 { 471 return Packet4cf(_mm256_blend_ps(a.v,pset1<Packet4cf>(b).v,(1<<7)|(1<<6))); 472 } 473 474 template<> EIGEN_STRONG_INLINE Packet2cd pinsertlast(const Packet2cd& a, std::complex<double> b) 475 { 476 return Packet2cd(_mm256_blend_pd(a.v,pset1<Packet2cd>(b).v,(1<<3)|(1<<2))); 477 } 478 479 } // end namespace internal 480 481 } // end namespace Eigen 482 483 #endif // EIGEN_COMPLEX_AVX_H 484
