1 /************************************************************************** 2 * 3 * Copyright 2008 Tungsten Graphics, Inc., Cedar Park, Texas. 4 * All Rights Reserved. 5 * 6 * Permission is hereby granted, free of charge, to any person obtaining a 7 * copy of this software and associated documentation files (the 8 * "Software"), to deal in the Software without restriction, including 9 * without limitation the rights to use, copy, modify, merge, publish, 10 * distribute, sub license, and/or sell copies of the Software, and to 11 * permit persons to whom the Software is furnished to do so, subject to 12 * the following conditions: 13 * 14 * The above copyright notice and this permission notice (including the 15 * next paragraph) shall be included in all copies or substantial portions 16 * of the Software. 17 * 18 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS 19 * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 20 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. 21 * IN NO EVENT SHALL TUNGSTEN GRAPHICS AND/OR ITS SUPPLIERS BE LIABLE FOR 22 * ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, 23 * TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE 24 * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 25 * 26 **************************************************************************/ 27 28 /** 29 * @file 30 * SSE intrinsics portability header. 31 * 32 * Although the SSE intrinsics are support by all modern x86 and x86-64 33 * compilers, there are some intrisincs missing in some implementations 34 * (especially older MSVC versions). This header abstracts that away. 35 */ 36 37 #ifndef U_SSE_H_ 38 #define U_SSE_H_ 39 40 #include "pipe/p_config.h" 41 42 #if defined(PIPE_ARCH_SSE) 43 44 #include <emmintrin.h> 45 46 47 /* MSVC before VC8 does not support the _mm_castxxx_yyy */ 48 #if defined(_MSC_VER) && _MSC_VER < 1500 49 50 union __declspec(align(16)) m128_types { 51 __m128 m128; 52 __m128i m128i; 53 __m128d m128d; 54 }; 55 56 static __inline __m128 57 _mm_castsi128_ps(__m128i a) 58 { 59 union m128_types u; 60 u.m128i = a; 61 return u.m128; 62 } 63 64 static __inline __m128i 65 _mm_castps_si128(__m128 a) 66 { 67 union m128_types u; 68 u.m128 = a; 69 return u.m128i; 70 } 71 72 #endif /* defined(_MSC_VER) && _MSC_VER < 1500 */ 73 74 union m128i { 75 __m128i m; 76 ubyte ub[16]; 77 ushort us[8]; 78 uint ui[4]; 79 }; 80 81 static INLINE void u_print_epi8(const char *name, __m128i r) 82 { 83 union { __m128i m; ubyte ub[16]; } u; 84 u.m = r; 85 86 debug_printf("%s: " 87 "%02x/" 88 "%02x/" 89 "%02x/" 90 "%02x/" 91 "%02x/" 92 "%02x/" 93 "%02x/" 94 "%02x/" 95 "%02x/" 96 "%02x/" 97 "%02x/" 98 "%02x/" 99 "%02x/" 100 "%02x/" 101 "%02x/" 102 "%02x\n", 103 name, 104 u.ub[0], u.ub[1], u.ub[2], u.ub[3], 105 u.ub[4], u.ub[5], u.ub[6], u.ub[7], 106 u.ub[8], u.ub[9], u.ub[10], u.ub[11], 107 u.ub[12], u.ub[13], u.ub[14], u.ub[15]); 108 } 109 110 static INLINE void u_print_epi16(const char *name, __m128i r) 111 { 112 union { __m128i m; ushort us[8]; } u; 113 u.m = r; 114 115 debug_printf("%s: " 116 "%04x/" 117 "%04x/" 118 "%04x/" 119 "%04x/" 120 "%04x/" 121 "%04x/" 122 "%04x/" 123 "%04x\n", 124 name, 125 u.us[0], u.us[1], u.us[2], u.us[3], 126 u.us[4], u.us[5], u.us[6], u.us[7]); 127 } 128 129 static INLINE void u_print_epi32(const char *name, __m128i r) 130 { 131 union { __m128i m; uint ui[4]; } u; 132 u.m = r; 133 134 debug_printf("%s: " 135 "%08x/" 136 "%08x/" 137 "%08x/" 138 "%08x\n", 139 name, 140 u.ui[0], u.ui[1], u.ui[2], u.ui[3]); 141 } 142 143 static INLINE void u_print_ps(const char *name, __m128 r) 144 { 145 union { __m128 m; float f[4]; } u; 146 u.m = r; 147 148 debug_printf("%s: " 149 "%f/" 150 "%f/" 151 "%f/" 152 "%f\n", 153 name, 154 u.f[0], u.f[1], u.f[2], u.f[3]); 155 } 156 157 158 #define U_DUMP_EPI32(a) u_print_epi32(#a, a) 159 #define U_DUMP_EPI16(a) u_print_epi16(#a, a) 160 #define U_DUMP_EPI8(a) u_print_epi8(#a, a) 161 #define U_DUMP_PS(a) u_print_ps(#a, a) 162 163 164 165 #if defined(PIPE_ARCH_SSSE3) 166 167 #include <tmmintrin.h> 168 169 #else /* !PIPE_ARCH_SSSE3 */ 170 171 /** 172 * Describe _mm_shuffle_epi8() with gcc extended inline assembly, for cases 173 * where -mssse3 is not supported/enabled. 174 * 175 * MSVC will never get in here as its intrinsics support do not rely on 176 * compiler command line options. 177 */ 178 static __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) 179 _mm_shuffle_epi8(__m128i a, __m128i mask) 180 { 181 __m128i result; 182 __asm__("pshufb %1, %0" 183 : "=x" (result) 184 : "xm" (mask), "0" (a)); 185 return result; 186 } 187 188 #endif /* !PIPE_ARCH_SSSE3 */ 189 190 191 192 193 /* Provide an SSE2 implementation of _mm_mullo_epi32() in terms of 194 * _mm_mul_epu32(). 195 * 196 * I suspect this works fine for us because one of our operands is 197 * always positive, but not sure that this can be used for general 198 * signed integer multiplication. 199 * 200 * This seems close enough to the speed of SSE4 and the real 201 * _mm_mullo_epi32() intrinsic as to not justify adding an sse4 202 * dependency at this point. 203 */ 204 static INLINE __m128i mm_mullo_epi32(const __m128i a, const __m128i b) 205 { 206 __m128i a4 = _mm_srli_epi64(a, 32); /* shift by one dword */ 207 __m128i b4 = _mm_srli_epi64(b, 32); /* shift by one dword */ 208 __m128i ba = _mm_mul_epu32(b, a); /* multply dwords 0, 2 */ 209 __m128i b4a4 = _mm_mul_epu32(b4, a4); /* multiply dwords 1, 3 */ 210 211 /* Interleave the results, either with shuffles or (slightly 212 * faster) direct bit operations: 213 */ 214 #if 0 215 __m128i ba8 = _mm_shuffle_epi32(ba, 8); 216 __m128i b4a48 = _mm_shuffle_epi32(b4a4, 8); 217 __m128i result = _mm_unpacklo_epi32(ba8, b4a48); 218 #else 219 __m128i mask = _mm_setr_epi32(~0,0,~0,0); 220 __m128i ba_mask = _mm_and_si128(ba, mask); 221 __m128i b4a4_mask_shift = _mm_slli_epi64(b4a4, 32); 222 __m128i result = _mm_or_si128(ba_mask, b4a4_mask_shift); 223 #endif 224 225 return result; 226 } 227 228 229 static INLINE void 230 transpose4_epi32(const __m128i * restrict a, 231 const __m128i * restrict b, 232 const __m128i * restrict c, 233 const __m128i * restrict d, 234 __m128i * restrict o, 235 __m128i * restrict p, 236 __m128i * restrict q, 237 __m128i * restrict r) 238 { 239 __m128i t0 = _mm_unpacklo_epi32(*a, *b); 240 __m128i t1 = _mm_unpacklo_epi32(*c, *d); 241 __m128i t2 = _mm_unpackhi_epi32(*a, *b); 242 __m128i t3 = _mm_unpackhi_epi32(*c, *d); 243 244 *o = _mm_unpacklo_epi64(t0, t1); 245 *p = _mm_unpackhi_epi64(t0, t1); 246 *q = _mm_unpacklo_epi64(t2, t3); 247 *r = _mm_unpackhi_epi64(t2, t3); 248 } 249 250 #define SCALAR_EPI32(m, i) _mm_shuffle_epi32((m), _MM_SHUFFLE(i,i,i,i)) 251 252 253 #endif /* PIPE_ARCH_SSE */ 254 255 #endif /* U_SSE_H_ */ 256
