1 """Shared OS X support functions.""" 2 3 import os 4 import re 5 import sys 6 7 __all__ = [ 8 'compiler_fixup', 9 'customize_config_vars', 10 'customize_compiler', 11 'get_platform_osx', 12 ] 13 14 # configuration variables that may contain universal build flags, 15 # like "-arch" or "-isdkroot", that may need customization for 16 # the user environment 17 _UNIVERSAL_CONFIG_VARS = ('CFLAGS', 'LDFLAGS', 'CPPFLAGS', 'BASECFLAGS', 18 'BLDSHARED', 'LDSHARED', 'CC', 'CXX', 19 'PY_CFLAGS', 'PY_LDFLAGS', 'PY_CPPFLAGS', 20 'PY_CORE_CFLAGS') 21 22 # configuration variables that may contain compiler calls 23 _COMPILER_CONFIG_VARS = ('BLDSHARED', 'LDSHARED', 'CC', 'CXX') 24 25 # prefix added to original configuration variable names 26 _INITPRE = '_OSX_SUPPORT_INITIAL_' 27 28 29 def _find_executable(executable, path=None): 30 """Tries to find 'executable' in the directories listed in 'path'. 31 32 A string listing directories separated by 'os.pathsep'; defaults to 33 os.environ['PATH']. Returns the complete filename or None if not found. 34 """ 35 if path is None: 36 path = os.environ['PATH'] 37 38 paths = path.split(os.pathsep) 39 base, ext = os.path.splitext(executable) 40 41 if (sys.platform == 'win32' or os.name == 'os2') and (ext != '.exe'): 42 executable = executable + '.exe' 43 44 if not os.path.isfile(executable): 45 for p in paths: 46 f = os.path.join(p, executable) 47 if os.path.isfile(f): 48 # the file exists, we have a shot at spawn working 49 return f 50 return None 51 else: 52 return executable 53 54 55 def _read_output(commandstring): 56 """Output from succesful command execution or None""" 57 # Similar to os.popen(commandstring, "r").read(), 58 # but without actually using os.popen because that 59 # function is not usable during python bootstrap. 60 # tempfile is also not available then. 61 import contextlib 62 try: 63 import tempfile 64 fp = tempfile.NamedTemporaryFile() 65 except ImportError: 66 fp = open("/tmp/_osx_support.%s"%( 67 os.getpid(),), "w+b") 68 69 with contextlib.closing(fp) as fp: 70 cmd = "%s 2>/dev/null >'%s'" % (commandstring, fp.name) 71 return fp.read().decode('utf-8').strip() if not os.system(cmd) else None 72 73 74 def _find_build_tool(toolname): 75 """Find a build tool on current path or using xcrun""" 76 return (_find_executable(toolname) 77 or _read_output("/usr/bin/xcrun -find %s" % (toolname,)) 78 or '' 79 ) 80 81 _SYSTEM_VERSION = None 82 83 def _get_system_version(): 84 """Return the OS X system version as a string""" 85 # Reading this plist is a documented way to get the system 86 # version (see the documentation for the Gestalt Manager) 87 # We avoid using platform.mac_ver to avoid possible bootstrap issues during 88 # the build of Python itself (distutils is used to build standard library 89 # extensions). 90 91 global _SYSTEM_VERSION 92 93 if _SYSTEM_VERSION is None: 94 _SYSTEM_VERSION = '' 95 try: 96 f = open('/System/Library/CoreServices/SystemVersion.plist') 97 except IOError: 98 # We're on a plain darwin box, fall back to the default 99 # behaviour. 100 pass 101 else: 102 try: 103 m = re.search(r'<key>ProductUserVisibleVersion</key>\s*' 104 r'<string>(.*?)</string>', f.read()) 105 finally: 106 f.close() 107 if m is not None: 108 _SYSTEM_VERSION = '.'.join(m.group(1).split('.')[:2]) 109 # else: fall back to the default behaviour 110 111 return _SYSTEM_VERSION 112 113 def _remove_original_values(_config_vars): 114 """Remove original unmodified values for testing""" 115 # This is needed for higher-level cross-platform tests of get_platform. 116 for k in list(_config_vars): 117 if k.startswith(_INITPRE): 118 del _config_vars[k] 119 120 def _save_modified_value(_config_vars, cv, newvalue): 121 """Save modified and original unmodified value of configuration var""" 122 123 oldvalue = _config_vars.get(cv, '') 124 if (oldvalue != newvalue) and (_INITPRE + cv not in _config_vars): 125 _config_vars[_INITPRE + cv] = oldvalue 126 _config_vars[cv] = newvalue 127 128 def _supports_universal_builds(): 129 """Returns True if universal builds are supported on this system""" 130 # As an approximation, we assume that if we are running on 10.4 or above, 131 # then we are running with an Xcode environment that supports universal 132 # builds, in particular -isysroot and -arch arguments to the compiler. This 133 # is in support of allowing 10.4 universal builds to run on 10.3.x systems. 134 135 osx_version = _get_system_version() 136 if osx_version: 137 try: 138 osx_version = tuple(int(i) for i in osx_version.split('.')) 139 except ValueError: 140 osx_version = '' 141 return bool(osx_version >= (10, 4)) if osx_version else False 142 143 144 def _find_appropriate_compiler(_config_vars): 145 """Find appropriate C compiler for extension module builds""" 146 147 # Issue #13590: 148 # The OSX location for the compiler varies between OSX 149 # (or rather Xcode) releases. With older releases (up-to 10.5) 150 # the compiler is in /usr/bin, with newer releases the compiler 151 # can only be found inside Xcode.app if the "Command Line Tools" 152 # are not installed. 153 # 154 # Futhermore, the compiler that can be used varies between 155 # Xcode releases. Upto Xcode 4 it was possible to use 'gcc-4.2' 156 # as the compiler, after that 'clang' should be used because 157 # gcc-4.2 is either not present, or a copy of 'llvm-gcc' that 158 # miscompiles Python. 159 160 # skip checks if the compiler was overriden with a CC env variable 161 if 'CC' in os.environ: 162 return _config_vars 163 164 # The CC config var might contain additional arguments. 165 # Ignore them while searching. 166 cc = oldcc = _config_vars['CC'].split()[0] 167 if not _find_executable(cc): 168 # Compiler is not found on the shell search PATH. 169 # Now search for clang, first on PATH (if the Command LIne 170 # Tools have been installed in / or if the user has provided 171 # another location via CC). If not found, try using xcrun 172 # to find an uninstalled clang (within a selected Xcode). 173 174 # NOTE: Cannot use subprocess here because of bootstrap 175 # issues when building Python itself (and os.popen is 176 # implemented on top of subprocess and is therefore not 177 # usable as well) 178 179 cc = _find_build_tool('clang') 180 181 elif os.path.basename(cc).startswith('gcc'): 182 # Compiler is GCC, check if it is LLVM-GCC 183 data = _read_output("'%s' --version" 184 % (cc.replace("'", "'\"'\"'"),)) 185 if 'llvm-gcc' in data: 186 # Found LLVM-GCC, fall back to clang 187 cc = _find_build_tool('clang') 188 189 if not cc: 190 raise SystemError( 191 "Cannot locate working compiler") 192 193 if cc != oldcc: 194 # Found a replacement compiler. 195 # Modify config vars using new compiler, if not already explictly 196 # overriden by an env variable, preserving additional arguments. 197 for cv in _COMPILER_CONFIG_VARS: 198 if cv in _config_vars and cv not in os.environ: 199 cv_split = _config_vars[cv].split() 200 cv_split[0] = cc if cv != 'CXX' else cc + '++' 201 _save_modified_value(_config_vars, cv, ' '.join(cv_split)) 202 203 return _config_vars 204 205 206 def _remove_universal_flags(_config_vars): 207 """Remove all universal build arguments from config vars""" 208 209 for cv in _UNIVERSAL_CONFIG_VARS: 210 # Do not alter a config var explicitly overriden by env var 211 if cv in _config_vars and cv not in os.environ: 212 flags = _config_vars[cv] 213 flags = re.sub('-arch\s+\w+\s', ' ', flags) 214 flags = re.sub('-isysroot [^ \t]*', ' ', flags) 215 _save_modified_value(_config_vars, cv, flags) 216 217 return _config_vars 218 219 220 def _remove_unsupported_archs(_config_vars): 221 """Remove any unsupported archs from config vars""" 222 # Different Xcode releases support different sets for '-arch' 223 # flags. In particular, Xcode 4.x no longer supports the 224 # PPC architectures. 225 # 226 # This code automatically removes '-arch ppc' and '-arch ppc64' 227 # when these are not supported. That makes it possible to 228 # build extensions on OSX 10.7 and later with the prebuilt 229 # 32-bit installer on the python.org website. 230 231 # skip checks if the compiler was overriden with a CC env variable 232 if 'CC' in os.environ: 233 return _config_vars 234 235 if re.search('-arch\s+ppc', _config_vars['CFLAGS']) is not None: 236 # NOTE: Cannot use subprocess here because of bootstrap 237 # issues when building Python itself 238 status = os.system("'%s' -arch ppc -x c /dev/null 2>/dev/null"%( 239 _config_vars['CC'].replace("'", "'\"'\"'"),)) 240 # The Apple compiler drivers return status 255 if no PPC 241 if (status >> 8) == 255: 242 # Compiler doesn't support PPC, remove the related 243 # '-arch' flags if not explicitly overridden by an 244 # environment variable 245 for cv in _UNIVERSAL_CONFIG_VARS: 246 if cv in _config_vars and cv not in os.environ: 247 flags = _config_vars[cv] 248 flags = re.sub('-arch\s+ppc\w*\s', ' ', flags) 249 _save_modified_value(_config_vars, cv, flags) 250 251 return _config_vars 252 253 254 def _override_all_archs(_config_vars): 255 """Allow override of all archs with ARCHFLAGS env var""" 256 # NOTE: This name was introduced by Apple in OSX 10.5 and 257 # is used by several scripting languages distributed with 258 # that OS release. 259 if 'ARCHFLAGS' in os.environ: 260 arch = os.environ['ARCHFLAGS'] 261 for cv in _UNIVERSAL_CONFIG_VARS: 262 if cv in _config_vars and '-arch' in _config_vars[cv]: 263 flags = _config_vars[cv] 264 flags = re.sub('-arch\s+\w+\s', ' ', flags) 265 flags = flags + ' ' + arch 266 _save_modified_value(_config_vars, cv, flags) 267 268 return _config_vars 269 270 271 def _check_for_unavailable_sdk(_config_vars): 272 """Remove references to any SDKs not available""" 273 # If we're on OSX 10.5 or later and the user tries to 274 # compile an extension using an SDK that is not present 275 # on the current machine it is better to not use an SDK 276 # than to fail. This is particularly important with 277 # the standalong Command Line Tools alternative to a 278 # full-blown Xcode install since the CLT packages do not 279 # provide SDKs. If the SDK is not present, it is assumed 280 # that the header files and dev libs have been installed 281 # to /usr and /System/Library by either a standalone CLT 282 # package or the CLT component within Xcode. 283 cflags = _config_vars.get('CFLAGS', '') 284 m = re.search(r'-isysroot\s+(\S+)', cflags) 285 if m is not None: 286 sdk = m.group(1) 287 if not os.path.exists(sdk): 288 for cv in _UNIVERSAL_CONFIG_VARS: 289 # Do not alter a config var explicitly overriden by env var 290 if cv in _config_vars and cv not in os.environ: 291 flags = _config_vars[cv] 292 flags = re.sub(r'-isysroot\s+\S+(?:\s|$)', ' ', flags) 293 _save_modified_value(_config_vars, cv, flags) 294 295 return _config_vars 296 297 298 def compiler_fixup(compiler_so, cc_args): 299 """ 300 This function will strip '-isysroot PATH' and '-arch ARCH' from the 301 compile flags if the user has specified one them in extra_compile_flags. 302 303 This is needed because '-arch ARCH' adds another architecture to the 304 build, without a way to remove an architecture. Furthermore GCC will 305 barf if multiple '-isysroot' arguments are present. 306 """ 307 stripArch = stripSysroot = False 308 309 compiler_so = list(compiler_so) 310 311 if not _supports_universal_builds(): 312 # OSX before 10.4.0, these don't support -arch and -isysroot at 313 # all. 314 stripArch = stripSysroot = True 315 else: 316 stripArch = '-arch' in cc_args 317 stripSysroot = '-isysroot' in cc_args 318 319 if stripArch or 'ARCHFLAGS' in os.environ: 320 while True: 321 try: 322 index = compiler_so.index('-arch') 323 # Strip this argument and the next one: 324 del compiler_so[index:index+2] 325 except ValueError: 326 break 327 328 if 'ARCHFLAGS' in os.environ and not stripArch: 329 # User specified different -arch flags in the environ, 330 # see also distutils.sysconfig 331 compiler_so = compiler_so + os.environ['ARCHFLAGS'].split() 332 333 if stripSysroot: 334 while True: 335 try: 336 index = compiler_so.index('-isysroot') 337 # Strip this argument and the next one: 338 del compiler_so[index:index+2] 339 except ValueError: 340 break 341 342 # Check if the SDK that is used during compilation actually exists, 343 # the universal build requires the usage of a universal SDK and not all 344 # users have that installed by default. 345 sysroot = None 346 if '-isysroot' in cc_args: 347 idx = cc_args.index('-isysroot') 348 sysroot = cc_args[idx+1] 349 elif '-isysroot' in compiler_so: 350 idx = compiler_so.index('-isysroot') 351 sysroot = compiler_so[idx+1] 352 353 if sysroot and not os.path.isdir(sysroot): 354 from distutils import log 355 log.warn("Compiling with an SDK that doesn't seem to exist: %s", 356 sysroot) 357 log.warn("Please check your Xcode installation") 358 359 return compiler_so 360 361 362 def customize_config_vars(_config_vars): 363 """Customize Python build configuration variables. 364 365 Called internally from sysconfig with a mutable mapping 366 containing name/value pairs parsed from the configured 367 makefile used to build this interpreter. Returns 368 the mapping updated as needed to reflect the environment 369 in which the interpreter is running; in the case of 370 a Python from a binary installer, the installed 371 environment may be very different from the build 372 environment, i.e. different OS levels, different 373 built tools, different available CPU architectures. 374 375 This customization is performed whenever 376 distutils.sysconfig.get_config_vars() is first 377 called. It may be used in environments where no 378 compilers are present, i.e. when installing pure 379 Python dists. Customization of compiler paths 380 and detection of unavailable archs is deferred 381 until the first extention module build is 382 requested (in distutils.sysconfig.customize_compiler). 383 384 Currently called from distutils.sysconfig 385 """ 386 387 if not _supports_universal_builds(): 388 # On Mac OS X before 10.4, check if -arch and -isysroot 389 # are in CFLAGS or LDFLAGS and remove them if they are. 390 # This is needed when building extensions on a 10.3 system 391 # using a universal build of python. 392 _remove_universal_flags(_config_vars) 393 394 # Allow user to override all archs with ARCHFLAGS env var 395 _override_all_archs(_config_vars) 396 397 # Remove references to sdks that are not found 398 _check_for_unavailable_sdk(_config_vars) 399 400 return _config_vars 401 402 403 def customize_compiler(_config_vars): 404 """Customize compiler path and configuration variables. 405 406 This customization is performed when the first 407 extension module build is requested 408 in distutils.sysconfig.customize_compiler). 409 """ 410 411 # Find a compiler to use for extension module builds 412 _find_appropriate_compiler(_config_vars) 413 414 # Remove ppc arch flags if not supported here 415 _remove_unsupported_archs(_config_vars) 416 417 # Allow user to override all archs with ARCHFLAGS env var 418 _override_all_archs(_config_vars) 419 420 return _config_vars 421 422 423 def get_platform_osx(_config_vars, osname, release, machine): 424 """Filter values for get_platform()""" 425 # called from get_platform() in sysconfig and distutils.util 426 # 427 # For our purposes, we'll assume that the system version from 428 # distutils' perspective is what MACOSX_DEPLOYMENT_TARGET is set 429 # to. This makes the compatibility story a bit more sane because the 430 # machine is going to compile and link as if it were 431 # MACOSX_DEPLOYMENT_TARGET. 432 433 macver = _config_vars.get('MACOSX_DEPLOYMENT_TARGET', '') 434 macrelease = _get_system_version() or macver 435 macver = macver or macrelease 436 437 if macver: 438 release = macver 439 osname = "macosx" 440 441 # Use the original CFLAGS value, if available, so that we 442 # return the same machine type for the platform string. 443 # Otherwise, distutils may consider this a cross-compiling 444 # case and disallow installs. 445 cflags = _config_vars.get(_INITPRE+'CFLAGS', 446 _config_vars.get('CFLAGS', '')) 447 if ((macrelease + '.') >= '10.4.' and 448 '-arch' in cflags.strip()): 449 # The universal build will build fat binaries, but not on 450 # systems before 10.4 451 452 machine = 'fat' 453 454 archs = re.findall('-arch\s+(\S+)', cflags) 455 archs = tuple(sorted(set(archs))) 456 457 if len(archs) == 1: 458 machine = archs[0] 459 elif archs == ('i386', 'ppc'): 460 machine = 'fat' 461 elif archs == ('i386', 'x86_64'): 462 machine = 'intel' 463 elif archs == ('i386', 'ppc', 'x86_64'): 464 machine = 'fat3' 465 elif archs == ('ppc64', 'x86_64'): 466 machine = 'fat64' 467 elif archs == ('i386', 'ppc', 'ppc64', 'x86_64'): 468 machine = 'universal' 469 else: 470 raise ValueError( 471 "Don't know machine value for archs=%r" % (archs,)) 472 473 elif machine == 'i386': 474 # On OSX the machine type returned by uname is always the 475 # 32-bit variant, even if the executable architecture is 476 # the 64-bit variant 477 if sys.maxint >= 2**32: 478 machine = 'x86_64' 479 480 elif machine in ('PowerPC', 'Power_Macintosh'): 481 # Pick a sane name for the PPC architecture. 482 # See 'i386' case 483 if sys.maxint >= 2**32: 484 machine = 'ppc64' 485 else: 486 machine = 'ppc' 487 488 return (osname, release, machine) 489
