1 /************************************************************************** 2 * 3 * Copyright 2008 Dennis Smit 4 * All Rights Reserved. 5 * 6 * Permission is hereby granted, free of charge, to any person obtaining a 7 * copy of this software and associated documentation files (the "Software"), 8 * to deal in the Software without restriction, including without limitation 9 * on the rights to use, copy, modify, merge, publish, distribute, sub 10 * license, and/or sell copies of the Software, and to permit persons to whom 11 * the Software is furnished to do so, subject to the following conditions: 12 * 13 * The above copyright notice and this permission notice (including the next 14 * paragraph) shall be included in all copies or substantial portions of the 15 * Software. 16 * 17 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 18 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 19 * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL 20 * AUTHORS, COPYRIGHT HOLDERS, AND/OR THEIR SUPPLIERS BE LIABLE FOR ANY CLAIM, 21 * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR 22 * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE 23 * USE OR OTHER DEALINGS IN THE SOFTWARE. 24 * 25 **************************************************************************/ 26 27 /** 28 * @file 29 * CPU feature detection. 30 * 31 * @author Dennis Smit 32 * @author Based on the work of Eric Anholt <anholt (at) FreeBSD.org> 33 */ 34 35 #include "pipe/p_config.h" 36 37 #include "u_debug.h" 38 #include "u_cpu_detect.h" 39 40 #if defined(PIPE_ARCH_PPC) 41 #if defined(PIPE_OS_APPLE) 42 #include <sys/sysctl.h> 43 #else 44 #include <signal.h> 45 #include <setjmp.h> 46 #endif 47 #endif 48 49 #if defined(PIPE_OS_NETBSD) || defined(PIPE_OS_OPENBSD) 50 #include <sys/param.h> 51 #include <sys/sysctl.h> 52 #include <machine/cpu.h> 53 #endif 54 55 #if defined(PIPE_OS_FREEBSD) || defined(PIPE_OS_DRAGONFLY) 56 #include <sys/types.h> 57 #include <sys/sysctl.h> 58 #endif 59 60 #if defined(PIPE_OS_LINUX) 61 #include <signal.h> 62 #endif 63 64 #ifdef PIPE_OS_UNIX 65 #include <unistd.h> 66 #endif 67 68 #if defined(PIPE_OS_WINDOWS) 69 #include <windows.h> 70 #if defined(PIPE_CC_MSVC) 71 #include <intrin.h> 72 #endif 73 #endif 74 75 76 #ifdef DEBUG 77 DEBUG_GET_ONCE_BOOL_OPTION(dump_cpu, "GALLIUM_DUMP_CPU", FALSE) 78 #endif 79 80 81 struct util_cpu_caps util_cpu_caps; 82 83 #if defined(PIPE_ARCH_X86) || defined(PIPE_ARCH_X86_64) 84 static int has_cpuid(void); 85 #endif 86 87 88 #if defined(PIPE_ARCH_PPC) && !defined(PIPE_OS_APPLE) 89 static jmp_buf __lv_powerpc_jmpbuf; 90 static volatile sig_atomic_t __lv_powerpc_canjump = 0; 91 92 static void 93 sigill_handler(int sig) 94 { 95 if (!__lv_powerpc_canjump) { 96 signal (sig, SIG_DFL); 97 raise (sig); 98 } 99 100 __lv_powerpc_canjump = 0; 101 longjmp(__lv_powerpc_jmpbuf, 1); 102 } 103 #endif 104 105 #if defined(PIPE_ARCH_PPC) 106 static void 107 check_os_altivec_support(void) 108 { 109 #if defined(PIPE_OS_APPLE) 110 int sels[2] = {CTL_HW, HW_VECTORUNIT}; 111 int has_vu = 0; 112 int len = sizeof (has_vu); 113 int err; 114 115 err = sysctl(sels, 2, &has_vu, &len, NULL, 0); 116 117 if (err == 0) { 118 if (has_vu != 0) { 119 util_cpu_caps.has_altivec = 1; 120 } 121 } 122 #else /* !PIPE_OS_APPLE */ 123 /* not on Apple/Darwin, do it the brute-force way */ 124 /* this is borrowed from the libmpeg2 library */ 125 signal(SIGILL, sigill_handler); 126 if (setjmp(__lv_powerpc_jmpbuf)) { 127 signal(SIGILL, SIG_DFL); 128 } else { 129 __lv_powerpc_canjump = 1; 130 131 __asm __volatile 132 ("mtspr 256, %0\n\t" 133 "vand %%v0, %%v0, %%v0" 134 : 135 : "r" (-1)); 136 137 signal(SIGILL, SIG_DFL); 138 util_cpu_caps.has_altivec = 1; 139 } 140 #endif /* !PIPE_OS_APPLE */ 141 } 142 #endif /* PIPE_ARCH_PPC */ 143 144 145 #if defined(PIPE_ARCH_X86) || defined (PIPE_ARCH_X86_64) 146 static int has_cpuid(void) 147 { 148 #if defined(PIPE_ARCH_X86) 149 #if defined(PIPE_OS_GCC) 150 int a, c; 151 152 __asm __volatile 153 ("pushf\n" 154 "popl %0\n" 155 "movl %0, %1\n" 156 "xorl $0x200000, %0\n" 157 "push %0\n" 158 "popf\n" 159 "pushf\n" 160 "popl %0\n" 161 : "=a" (a), "=c" (c) 162 : 163 : "cc"); 164 165 return a != c; 166 #else 167 /* FIXME */ 168 return 1; 169 #endif 170 #elif defined(PIPE_ARCH_X86_64) 171 return 1; 172 #else 173 return 0; 174 #endif 175 } 176 177 178 /** 179 * @sa cpuid.h included in gcc-4.3 onwards. 180 * @sa http://msdn.microsoft.com/en-us/library/hskdteyh.aspx 181 */ 182 static inline void 183 cpuid(uint32_t ax, uint32_t *p) 184 { 185 #if defined(PIPE_CC_GCC) && defined(PIPE_ARCH_X86) 186 __asm __volatile ( 187 "xchgl %%ebx, %1\n\t" 188 "cpuid\n\t" 189 "xchgl %%ebx, %1" 190 : "=a" (p[0]), 191 "=S" (p[1]), 192 "=c" (p[2]), 193 "=d" (p[3]) 194 : "0" (ax) 195 ); 196 #elif defined(PIPE_CC_GCC) && defined(PIPE_ARCH_X86_64) 197 __asm __volatile ( 198 "cpuid\n\t" 199 : "=a" (p[0]), 200 "=b" (p[1]), 201 "=c" (p[2]), 202 "=d" (p[3]) 203 : "0" (ax) 204 ); 205 #elif defined(PIPE_CC_MSVC) 206 __cpuid(p, ax); 207 #else 208 p[0] = 0; 209 p[1] = 0; 210 p[2] = 0; 211 p[3] = 0; 212 #endif 213 } 214 215 /** 216 * @sa cpuid.h included in gcc-4.4 onwards. 217 * @sa http://msdn.microsoft.com/en-us/library/hskdteyh%28v=vs.90%29.aspx 218 */ 219 static inline void 220 cpuid_count(uint32_t ax, uint32_t cx, uint32_t *p) 221 { 222 #if defined(PIPE_CC_GCC) && defined(PIPE_ARCH_X86) 223 __asm __volatile ( 224 "xchgl %%ebx, %1\n\t" 225 "cpuid\n\t" 226 "xchgl %%ebx, %1" 227 : "=a" (p[0]), 228 "=S" (p[1]), 229 "=c" (p[2]), 230 "=d" (p[3]) 231 : "0" (ax), "2" (cx) 232 ); 233 #elif defined(PIPE_CC_GCC) && defined(PIPE_ARCH_X86_64) 234 __asm __volatile ( 235 "cpuid\n\t" 236 : "=a" (p[0]), 237 "=b" (p[1]), 238 "=c" (p[2]), 239 "=d" (p[3]) 240 : "0" (ax), "2" (cx) 241 ); 242 #elif defined(PIPE_CC_MSVC) 243 __cpuidex(p, ax, cx); 244 #else 245 p[0] = 0; 246 p[1] = 0; 247 p[2] = 0; 248 p[3] = 0; 249 #endif 250 } 251 252 253 static inline uint64_t xgetbv(void) 254 { 255 #if defined(PIPE_CC_GCC) 256 uint32_t eax, edx; 257 258 __asm __volatile ( 259 ".byte 0x0f, 0x01, 0xd0" // xgetbv isn't supported on gcc < 4.4 260 : "=a"(eax), 261 "=d"(edx) 262 : "c"(0) 263 ); 264 265 return ((uint64_t)edx << 32) | eax; 266 #elif defined(PIPE_CC_MSVC) && defined(_MSC_FULL_VER) && defined(_XCR_XFEATURE_ENABLED_MASK) 267 return _xgetbv(_XCR_XFEATURE_ENABLED_MASK); 268 #else 269 return 0; 270 #endif 271 } 272 273 274 #if defined(PIPE_ARCH_X86) 275 PIPE_ALIGN_STACK static inline boolean sse2_has_daz(void) 276 { 277 struct { 278 uint32_t pad1[7]; 279 uint32_t mxcsr_mask; 280 uint32_t pad2[128-8]; 281 } PIPE_ALIGN_VAR(16) fxarea; 282 283 fxarea.mxcsr_mask = 0; 284 #if defined(PIPE_CC_GCC) 285 __asm __volatile ("fxsave %0" : "+m" (fxarea)); 286 #elif defined(PIPE_CC_MSVC) || defined(PIPE_CC_ICL) 287 _fxsave(&fxarea); 288 #else 289 fxarea.mxcsr_mask = 0; 290 #endif 291 return !!(fxarea.mxcsr_mask & (1 << 6)); 292 } 293 #endif 294 295 #endif /* X86 or X86_64 */ 296 297 void 298 util_cpu_detect(void) 299 { 300 static boolean util_cpu_detect_initialized = FALSE; 301 302 if(util_cpu_detect_initialized) 303 return; 304 305 memset(&util_cpu_caps, 0, sizeof util_cpu_caps); 306 307 /* Count the number of CPUs in system */ 308 #if defined(PIPE_OS_WINDOWS) 309 { 310 SYSTEM_INFO system_info; 311 GetSystemInfo(&system_info); 312 util_cpu_caps.nr_cpus = system_info.dwNumberOfProcessors; 313 } 314 #elif defined(PIPE_OS_UNIX) && defined(_SC_NPROCESSORS_ONLN) 315 util_cpu_caps.nr_cpus = sysconf(_SC_NPROCESSORS_ONLN); 316 if (util_cpu_caps.nr_cpus == ~0u) 317 util_cpu_caps.nr_cpus = 1; 318 #elif defined(PIPE_OS_BSD) 319 { 320 int mib[2], ncpu; 321 int len; 322 323 mib[0] = CTL_HW; 324 mib[1] = HW_NCPU; 325 326 len = sizeof (ncpu); 327 sysctl(mib, 2, &ncpu, &len, NULL, 0); 328 util_cpu_caps.nr_cpus = ncpu; 329 } 330 #else 331 util_cpu_caps.nr_cpus = 1; 332 #endif 333 334 /* Make the fallback cacheline size nonzero so that it can be 335 * safely passed to align(). 336 */ 337 util_cpu_caps.cacheline = sizeof(void *); 338 339 #if defined(PIPE_ARCH_X86) || defined(PIPE_ARCH_X86_64) 340 if (has_cpuid()) { 341 uint32_t regs[4]; 342 uint32_t regs2[4]; 343 344 util_cpu_caps.cacheline = 32; 345 346 /* Get max cpuid level */ 347 cpuid(0x00000000, regs); 348 349 if (regs[0] >= 0x00000001) { 350 unsigned int cacheline; 351 352 cpuid (0x00000001, regs2); 353 354 util_cpu_caps.x86_cpu_type = (regs2[0] >> 8) & 0xf; 355 if (util_cpu_caps.x86_cpu_type == 0xf) 356 util_cpu_caps.x86_cpu_type = 8 + ((regs2[0] >> 20) & 255); /* use extended family (P4, IA64) */ 357 358 /* general feature flags */ 359 util_cpu_caps.has_tsc = (regs2[3] >> 4) & 1; /* 0x0000010 */ 360 util_cpu_caps.has_mmx = (regs2[3] >> 23) & 1; /* 0x0800000 */ 361 util_cpu_caps.has_sse = (regs2[3] >> 25) & 1; /* 0x2000000 */ 362 util_cpu_caps.has_sse2 = (regs2[3] >> 26) & 1; /* 0x4000000 */ 363 util_cpu_caps.has_sse3 = (regs2[2] >> 0) & 1; /* 0x0000001 */ 364 util_cpu_caps.has_ssse3 = (regs2[2] >> 9) & 1; /* 0x0000020 */ 365 util_cpu_caps.has_sse4_1 = (regs2[2] >> 19) & 1; 366 util_cpu_caps.has_sse4_2 = (regs2[2] >> 20) & 1; 367 util_cpu_caps.has_popcnt = (regs2[2] >> 23) & 1; 368 util_cpu_caps.has_avx = ((regs2[2] >> 28) & 1) && // AVX 369 ((regs2[2] >> 27) & 1) && // OSXSAVE 370 ((xgetbv() & 6) == 6); // XMM & YMM 371 util_cpu_caps.has_f16c = ((regs2[2] >> 29) & 1) && util_cpu_caps.has_avx; 372 util_cpu_caps.has_fma = ((regs2[2] >> 12) & 1) && util_cpu_caps.has_avx; 373 util_cpu_caps.has_mmx2 = util_cpu_caps.has_sse; /* SSE cpus supports mmxext too */ 374 #if defined(PIPE_ARCH_X86_64) 375 util_cpu_caps.has_daz = 1; 376 #else 377 util_cpu_caps.has_daz = util_cpu_caps.has_sse3 || 378 (util_cpu_caps.has_sse2 && sse2_has_daz()); 379 #endif 380 381 cacheline = ((regs2[1] >> 8) & 0xFF) * 8; 382 if (cacheline > 0) 383 util_cpu_caps.cacheline = cacheline; 384 } 385 if (util_cpu_caps.has_avx && regs[0] >= 0x00000007) { 386 uint32_t regs7[4]; 387 cpuid_count(0x00000007, 0x00000000, regs7); 388 util_cpu_caps.has_avx2 = (regs7[1] >> 5) & 1; 389 } 390 391 // check for avx512 392 if (((regs2[2] >> 27) & 1) && // OSXSAVE 393 (xgetbv() & (0x7 << 5)) && // OPMASK: upper-256 enabled by OS 394 ((xgetbv() & 6) == 6)) { // XMM/YMM enabled by OS 395 uint32_t regs3[4]; 396 cpuid(0x00000007, regs3); 397 util_cpu_caps.has_avx512f = (regs3[1] >> 16) & 1; 398 util_cpu_caps.has_avx512dq = (regs3[1] >> 17) & 1; 399 util_cpu_caps.has_avx512ifma = (regs3[1] >> 21) & 1; 400 util_cpu_caps.has_avx512pf = (regs3[1] >> 26) & 1; 401 util_cpu_caps.has_avx512er = (regs3[1] >> 27) & 1; 402 util_cpu_caps.has_avx512cd = (regs3[1] >> 28) & 1; 403 util_cpu_caps.has_avx512bw = (regs3[1] >> 30) & 1; 404 util_cpu_caps.has_avx512vl = (regs3[1] >> 31) & 1; 405 util_cpu_caps.has_avx512vbmi = (regs3[2] >> 1) & 1; 406 } 407 408 if (regs[1] == 0x756e6547 && regs[2] == 0x6c65746e && regs[3] == 0x49656e69) { 409 /* GenuineIntel */ 410 util_cpu_caps.has_intel = 1; 411 } 412 413 cpuid(0x80000000, regs); 414 415 if (regs[0] >= 0x80000001) { 416 417 cpuid(0x80000001, regs2); 418 419 util_cpu_caps.has_mmx |= (regs2[3] >> 23) & 1; 420 util_cpu_caps.has_mmx2 |= (regs2[3] >> 22) & 1; 421 util_cpu_caps.has_3dnow = (regs2[3] >> 31) & 1; 422 util_cpu_caps.has_3dnow_ext = (regs2[3] >> 30) & 1; 423 424 util_cpu_caps.has_xop = util_cpu_caps.has_avx && 425 ((regs2[2] >> 11) & 1); 426 } 427 428 if (regs[0] >= 0x80000006) { 429 /* should we really do this if the clflush size above worked? */ 430 unsigned int cacheline; 431 cpuid(0x80000006, regs2); 432 cacheline = regs2[2] & 0xFF; 433 if (cacheline > 0) 434 util_cpu_caps.cacheline = cacheline; 435 } 436 437 if (!util_cpu_caps.has_sse) { 438 util_cpu_caps.has_sse2 = 0; 439 util_cpu_caps.has_sse3 = 0; 440 util_cpu_caps.has_ssse3 = 0; 441 util_cpu_caps.has_sse4_1 = 0; 442 } 443 } 444 #endif /* PIPE_ARCH_X86 || PIPE_ARCH_X86_64 */ 445 446 #if defined(PIPE_ARCH_PPC) 447 check_os_altivec_support(); 448 #endif /* PIPE_ARCH_PPC */ 449 450 #ifdef DEBUG 451 if (debug_get_option_dump_cpu()) { 452 debug_printf("util_cpu_caps.nr_cpus = %u\n", util_cpu_caps.nr_cpus); 453 454 debug_printf("util_cpu_caps.x86_cpu_type = %u\n", util_cpu_caps.x86_cpu_type); 455 debug_printf("util_cpu_caps.cacheline = %u\n", util_cpu_caps.cacheline); 456 457 debug_printf("util_cpu_caps.has_tsc = %u\n", util_cpu_caps.has_tsc); 458 debug_printf("util_cpu_caps.has_mmx = %u\n", util_cpu_caps.has_mmx); 459 debug_printf("util_cpu_caps.has_mmx2 = %u\n", util_cpu_caps.has_mmx2); 460 debug_printf("util_cpu_caps.has_sse = %u\n", util_cpu_caps.has_sse); 461 debug_printf("util_cpu_caps.has_sse2 = %u\n", util_cpu_caps.has_sse2); 462 debug_printf("util_cpu_caps.has_sse3 = %u\n", util_cpu_caps.has_sse3); 463 debug_printf("util_cpu_caps.has_ssse3 = %u\n", util_cpu_caps.has_ssse3); 464 debug_printf("util_cpu_caps.has_sse4_1 = %u\n", util_cpu_caps.has_sse4_1); 465 debug_printf("util_cpu_caps.has_sse4_2 = %u\n", util_cpu_caps.has_sse4_2); 466 debug_printf("util_cpu_caps.has_avx = %u\n", util_cpu_caps.has_avx); 467 debug_printf("util_cpu_caps.has_avx2 = %u\n", util_cpu_caps.has_avx2); 468 debug_printf("util_cpu_caps.has_f16c = %u\n", util_cpu_caps.has_f16c); 469 debug_printf("util_cpu_caps.has_popcnt = %u\n", util_cpu_caps.has_popcnt); 470 debug_printf("util_cpu_caps.has_3dnow = %u\n", util_cpu_caps.has_3dnow); 471 debug_printf("util_cpu_caps.has_3dnow_ext = %u\n", util_cpu_caps.has_3dnow_ext); 472 debug_printf("util_cpu_caps.has_xop = %u\n", util_cpu_caps.has_xop); 473 debug_printf("util_cpu_caps.has_altivec = %u\n", util_cpu_caps.has_altivec); 474 debug_printf("util_cpu_caps.has_daz = %u\n", util_cpu_caps.has_daz); 475 debug_printf("util_cpu_caps.has_avx512f = %u\n", util_cpu_caps.has_avx512f); 476 debug_printf("util_cpu_caps.has_avx512dq = %u\n", util_cpu_caps.has_avx512dq); 477 debug_printf("util_cpu_caps.has_avx512ifma = %u\n", util_cpu_caps.has_avx512ifma); 478 debug_printf("util_cpu_caps.has_avx512pf = %u\n", util_cpu_caps.has_avx512pf); 479 debug_printf("util_cpu_caps.has_avx512er = %u\n", util_cpu_caps.has_avx512er); 480 debug_printf("util_cpu_caps.has_avx512cd = %u\n", util_cpu_caps.has_avx512cd); 481 debug_printf("util_cpu_caps.has_avx512bw = %u\n", util_cpu_caps.has_avx512bw); 482 debug_printf("util_cpu_caps.has_avx512vl = %u\n", util_cpu_caps.has_avx512vl); 483 debug_printf("util_cpu_caps.has_avx512vbmi = %u\n", util_cpu_caps.has_avx512vbmi); 484 } 485 #endif 486 487 util_cpu_detect_initialized = TRUE; 488 } 489
