1 /************************************************************************** 2 * 3 * Copyright 2008 VMware, Inc. 4 * All Rights Reserved. 5 * 6 * Permission is hereby granted, free of charge, to any person obtaining a 7 * copy of this software and associated documentation files (the 8 * "Software"), to deal in the Software without restriction, including 9 * without limitation the rights to use, copy, modify, merge, publish, 10 * distribute, sub license, and/or sell copies of the Software, and to 11 * permit persons to whom the Software is furnished to do so, subject to 12 * the following conditions: 13 * 14 * The above copyright notice and this permission notice (including the 15 * next paragraph) shall be included in all copies or substantial portions 16 * of the Software. 17 * 18 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS 19 * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 20 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. 21 * IN NO EVENT SHALL VMWARE AND/OR ITS SUPPLIERS BE LIABLE FOR 22 * ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, 23 * TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE 24 * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 25 * 26 **************************************************************************/ 27 28 /** 29 * @file 30 * SSE intrinsics portability header. 31 * 32 * Although the SSE intrinsics are support by all modern x86 and x86-64 33 * compilers, there are some intrisincs missing in some implementations 34 * (especially older MSVC versions). This header abstracts that away. 35 */ 36 37 #ifndef U_SSE_H_ 38 #define U_SSE_H_ 39 40 #include "pipe/p_config.h" 41 42 #if defined(PIPE_ARCH_SSE) 43 44 #include <emmintrin.h> 45 46 47 union m128i { 48 __m128i m; 49 ubyte ub[16]; 50 ushort us[8]; 51 uint ui[4]; 52 }; 53 54 static inline void u_print_epi8(const char *name, __m128i r) 55 { 56 union { __m128i m; ubyte ub[16]; } u; 57 u.m = r; 58 59 debug_printf("%s: " 60 "%02x/" 61 "%02x/" 62 "%02x/" 63 "%02x/" 64 "%02x/" 65 "%02x/" 66 "%02x/" 67 "%02x/" 68 "%02x/" 69 "%02x/" 70 "%02x/" 71 "%02x/" 72 "%02x/" 73 "%02x/" 74 "%02x/" 75 "%02x\n", 76 name, 77 u.ub[0], u.ub[1], u.ub[2], u.ub[3], 78 u.ub[4], u.ub[5], u.ub[6], u.ub[7], 79 u.ub[8], u.ub[9], u.ub[10], u.ub[11], 80 u.ub[12], u.ub[13], u.ub[14], u.ub[15]); 81 } 82 83 static inline void u_print_epi16(const char *name, __m128i r) 84 { 85 union { __m128i m; ushort us[8]; } u; 86 u.m = r; 87 88 debug_printf("%s: " 89 "%04x/" 90 "%04x/" 91 "%04x/" 92 "%04x/" 93 "%04x/" 94 "%04x/" 95 "%04x/" 96 "%04x\n", 97 name, 98 u.us[0], u.us[1], u.us[2], u.us[3], 99 u.us[4], u.us[5], u.us[6], u.us[7]); 100 } 101 102 static inline void u_print_epi32(const char *name, __m128i r) 103 { 104 union { __m128i m; uint ui[4]; } u; 105 u.m = r; 106 107 debug_printf("%s: " 108 "%08x/" 109 "%08x/" 110 "%08x/" 111 "%08x\n", 112 name, 113 u.ui[0], u.ui[1], u.ui[2], u.ui[3]); 114 } 115 116 static inline void u_print_ps(const char *name, __m128 r) 117 { 118 union { __m128 m; float f[4]; } u; 119 u.m = r; 120 121 debug_printf("%s: " 122 "%f/" 123 "%f/" 124 "%f/" 125 "%f\n", 126 name, 127 u.f[0], u.f[1], u.f[2], u.f[3]); 128 } 129 130 131 #define U_DUMP_EPI32(a) u_print_epi32(#a, a) 132 #define U_DUMP_EPI16(a) u_print_epi16(#a, a) 133 #define U_DUMP_EPI8(a) u_print_epi8(#a, a) 134 #define U_DUMP_PS(a) u_print_ps(#a, a) 135 136 137 138 #if defined(PIPE_ARCH_SSSE3) 139 140 #include <tmmintrin.h> 141 142 #else /* !PIPE_ARCH_SSSE3 */ 143 144 /** 145 * Describe _mm_shuffle_epi8() with gcc extended inline assembly, for cases 146 * where -mssse3 is not supported/enabled. 147 * 148 * MSVC will never get in here as its intrinsics support do not rely on 149 * compiler command line options. 150 */ 151 static __inline __m128i 152 #ifdef __clang__ 153 __attribute__((__always_inline__, __nodebug__)) 154 #else 155 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) 156 #endif 157 _mm_shuffle_epi8(__m128i a, __m128i mask) 158 { 159 __m128i result; 160 __asm__("pshufb %1, %0" 161 : "=x" (result) 162 : "xm" (mask), "0" (a)); 163 return result; 164 } 165 166 #endif /* !PIPE_ARCH_SSSE3 */ 167 168 169 /* 170 * Provide an SSE implementation of _mm_mul_epi32() in terms of 171 * _mm_mul_epu32(). 172 * 173 * Basically, albeit surprising at first (and second, and third...) look 174 * if a * b is done signed instead of unsigned, can just 175 * subtract b from the high bits of the result if a is negative 176 * (and the same for a if b is negative). Modular arithmetic at its best! 177 * 178 * So for int32 a,b in crude pseudo-code ("*" here denoting a widening mul) 179 * fixupb = (signmask(b) & a) << 32ULL 180 * fixupa = (signmask(a) & b) << 32ULL 181 * a * b = (unsigned)a * (unsigned)b - fixupb - fixupa 182 * = (unsigned)a * (unsigned)b -(fixupb + fixupa) 183 * 184 * This does both lo (dwords 0/2) and hi parts (1/3) at the same time due 185 * to some optimization potential. 186 */ 187 static inline __m128i 188 mm_mullohi_epi32(const __m128i a, const __m128i b, __m128i *res13) 189 { 190 __m128i a13, b13, mul02, mul13; 191 __m128i anegmask, bnegmask, fixup, fixup02, fixup13; 192 a13 = _mm_shuffle_epi32(a, _MM_SHUFFLE(2,3,0,1)); 193 b13 = _mm_shuffle_epi32(b, _MM_SHUFFLE(2,3,0,1)); 194 anegmask = _mm_srai_epi32(a, 31); 195 bnegmask = _mm_srai_epi32(b, 31); 196 fixup = _mm_add_epi32(_mm_and_si128(anegmask, b), 197 _mm_and_si128(bnegmask, a)); 198 mul02 = _mm_mul_epu32(a, b); 199 mul13 = _mm_mul_epu32(a13, b13); 200 fixup02 = _mm_slli_epi64(fixup, 32); 201 fixup13 = _mm_and_si128(fixup, _mm_set_epi32(-1,0,-1,0)); 202 *res13 = _mm_sub_epi64(mul13, fixup13); 203 return _mm_sub_epi64(mul02, fixup02); 204 } 205 206 207 /* Provide an SSE2 implementation of _mm_mullo_epi32() in terms of 208 * _mm_mul_epu32(). 209 * 210 * This always works regardless the signs of the operands, since 211 * the high bits (which would be different) aren't used. 212 * 213 * This seems close enough to the speed of SSE4 and the real 214 * _mm_mullo_epi32() intrinsic as to not justify adding an sse4 215 * dependency at this point. 216 */ 217 static inline __m128i mm_mullo_epi32(const __m128i a, const __m128i b) 218 { 219 __m128i a4 = _mm_srli_epi64(a, 32); /* shift by one dword */ 220 __m128i b4 = _mm_srli_epi64(b, 32); /* shift by one dword */ 221 __m128i ba = _mm_mul_epu32(b, a); /* multply dwords 0, 2 */ 222 __m128i b4a4 = _mm_mul_epu32(b4, a4); /* multiply dwords 1, 3 */ 223 224 /* Interleave the results, either with shuffles or (slightly 225 * faster) direct bit operations: 226 * XXX: might be only true for some cpus (in particular 65nm 227 * Core 2). On most cpus (including that Core 2, but not Nehalem...) 228 * using _mm_shuffle_ps/_mm_shuffle_epi32 might also be faster 229 * than using the 3 instructions below. But logic should be fine 230 * as well, we can't have optimal solution for all cpus (if anything, 231 * should just use _mm_mullo_epi32() if sse41 is available...). 232 */ 233 #if 0 234 __m128i ba8 = _mm_shuffle_epi32(ba, 8); 235 __m128i b4a48 = _mm_shuffle_epi32(b4a4, 8); 236 __m128i result = _mm_unpacklo_epi32(ba8, b4a48); 237 #else 238 __m128i mask = _mm_setr_epi32(~0,0,~0,0); 239 __m128i ba_mask = _mm_and_si128(ba, mask); 240 __m128i b4a4_mask_shift = _mm_slli_epi64(b4a4, 32); 241 __m128i result = _mm_or_si128(ba_mask, b4a4_mask_shift); 242 #endif 243 244 return result; 245 } 246 247 248 static inline void 249 transpose4_epi32(const __m128i * restrict a, 250 const __m128i * restrict b, 251 const __m128i * restrict c, 252 const __m128i * restrict d, 253 __m128i * restrict o, 254 __m128i * restrict p, 255 __m128i * restrict q, 256 __m128i * restrict r) 257 { 258 __m128i t0 = _mm_unpacklo_epi32(*a, *b); 259 __m128i t1 = _mm_unpacklo_epi32(*c, *d); 260 __m128i t2 = _mm_unpackhi_epi32(*a, *b); 261 __m128i t3 = _mm_unpackhi_epi32(*c, *d); 262 263 *o = _mm_unpacklo_epi64(t0, t1); 264 *p = _mm_unpackhi_epi64(t0, t1); 265 *q = _mm_unpacklo_epi64(t2, t3); 266 *r = _mm_unpackhi_epi64(t2, t3); 267 } 268 269 270 /* 271 * Same as above, except the first two values are already interleaved 272 * (i.e. contain 64bit values). 273 */ 274 static inline void 275 transpose2_64_2_32(const __m128i * restrict a01, 276 const __m128i * restrict a23, 277 const __m128i * restrict c, 278 const __m128i * restrict d, 279 __m128i * restrict o, 280 __m128i * restrict p, 281 __m128i * restrict q, 282 __m128i * restrict r) 283 { 284 __m128i t0 = *a01; 285 __m128i t1 = _mm_unpacklo_epi32(*c, *d); 286 __m128i t2 = *a23; 287 __m128i t3 = _mm_unpackhi_epi32(*c, *d); 288 289 *o = _mm_unpacklo_epi64(t0, t1); 290 *p = _mm_unpackhi_epi64(t0, t1); 291 *q = _mm_unpacklo_epi64(t2, t3); 292 *r = _mm_unpackhi_epi64(t2, t3); 293 } 294 295 296 #define SCALAR_EPI32(m, i) _mm_shuffle_epi32((m), _MM_SHUFFLE(i,i,i,i)) 297 298 299 #endif /* PIPE_ARCH_SSE */ 300 301 #endif /* U_SSE_H_ */ 302
