1 #!/usr/bin/env python 2 """ 3 This script is used to build "official" universal installers on macOS. 4 5 NEW for 3.7.0: 6 - support Intel 64-bit-only () and 32-bit-only installer builds 7 - build and use internal Tcl/Tk 8.6 for 10.6+ builds 8 - deprecate use of explicit SDK (--sdk-path=) since all but the oldest 9 versions of Xcode support implicit setting of an SDK via environment 10 variables (SDKROOT and friends, see the xcrun man page for more info). 11 The SDK stuff was primarily needed for building universal installers 12 for 10.4; so as of 3.7.0, building installers for 10.4 is no longer 13 supported with build-installer. 14 - use generic "gcc" as compiler (CC env var) rather than "gcc-4.2" 15 16 TODO: 17 - support SDKROOT and DEVELOPER_DIR xcrun env variables 18 - test with 10.5 and 10.4 and determine support status 19 20 Please ensure that this script keeps working with Python 2.5, to avoid 21 bootstrap issues (/usr/bin/python is Python 2.5 on OSX 10.5). Doc builds 22 use current versions of Sphinx and require a reasonably current python3. 23 Sphinx and dependencies are installed into a venv using the python3's pip 24 so will fetch them from PyPI if necessary. Since python3 is now used for 25 Sphinx, build-installer.py should also be converted to use python3! 26 27 For 3.7.0, when building for a 10.6 or higher deployment target, 28 build-installer builds and links with its own copy of Tcl/Tk 8.6. 29 Otherwise, it requires an installed third-party version of 30 Tcl/Tk 8.4 (for OS X 10.4 and 10.5 deployment targets), Tcl/TK 8.5 31 (for 10.6 or later), or Tcl/TK 8.6 (for 10.9 or later) 32 installed in /Library/Frameworks. When installed, 33 the Python built by this script will attempt to dynamically link first to 34 Tcl and Tk frameworks in /Library/Frameworks if available otherwise fall 35 back to the ones in /System/Library/Framework. For the build, we recommend 36 installing the most recent ActiveTcl 8.6. 8.5, or 8.4 version, depending 37 on the deployment target. The actual version linked to depends on the 38 path of /Library/Frameworks/{Tcl,Tk}.framework/Versions/Current. 39 40 Usage: see USAGE variable in the script. 41 """ 42 import platform, os, sys, getopt, textwrap, shutil, stat, time, pwd, grp 43 try: 44 import urllib2 as urllib_request 45 except ImportError: 46 import urllib.request as urllib_request 47 48 STAT_0o755 = ( stat.S_IRUSR | stat.S_IWUSR | stat.S_IXUSR 49 | stat.S_IRGRP | stat.S_IXGRP 50 | stat.S_IROTH | stat.S_IXOTH ) 51 52 STAT_0o775 = ( stat.S_IRUSR | stat.S_IWUSR | stat.S_IXUSR 53 | stat.S_IRGRP | stat.S_IWGRP | stat.S_IXGRP 54 | stat.S_IROTH | stat.S_IXOTH ) 55 56 INCLUDE_TIMESTAMP = 1 57 VERBOSE = 1 58 59 from plistlib import Plist 60 61 try: 62 from plistlib import writePlist 63 except ImportError: 64 # We're run using python2.3 65 def writePlist(plist, path): 66 plist.write(path) 67 68 def shellQuote(value): 69 """ 70 Return the string value in a form that can safely be inserted into 71 a shell command. 72 """ 73 return "'%s'"%(value.replace("'", "'\"'\"'")) 74 75 def grepValue(fn, variable): 76 """ 77 Return the unquoted value of a variable from a file.. 78 QUOTED_VALUE='quotes' -> str('quotes') 79 UNQUOTED_VALUE=noquotes -> str('noquotes') 80 """ 81 variable = variable + '=' 82 for ln in open(fn, 'r'): 83 if ln.startswith(variable): 84 value = ln[len(variable):].strip() 85 return value.strip("\"'") 86 raise RuntimeError("Cannot find variable %s" % variable[:-1]) 87 88 _cache_getVersion = None 89 90 def getVersion(): 91 global _cache_getVersion 92 if _cache_getVersion is None: 93 _cache_getVersion = grepValue( 94 os.path.join(SRCDIR, 'configure'), 'PACKAGE_VERSION') 95 return _cache_getVersion 96 97 def getVersionMajorMinor(): 98 return tuple([int(n) for n in getVersion().split('.', 2)]) 99 100 _cache_getFullVersion = None 101 102 def getFullVersion(): 103 global _cache_getFullVersion 104 if _cache_getFullVersion is not None: 105 return _cache_getFullVersion 106 fn = os.path.join(SRCDIR, 'Include', 'patchlevel.h') 107 for ln in open(fn): 108 if 'PY_VERSION' in ln: 109 _cache_getFullVersion = ln.split()[-1][1:-1] 110 return _cache_getFullVersion 111 raise RuntimeError("Cannot find full version??") 112 113 FW_PREFIX = ["Library", "Frameworks", "Python.framework"] 114 FW_VERSION_PREFIX = "--undefined--" # initialized in parseOptions 115 FW_SSL_DIRECTORY = "--undefined--" # initialized in parseOptions 116 117 # The directory we'll use to create the build (will be erased and recreated) 118 WORKDIR = "/tmp/_py" 119 120 # The directory we'll use to store third-party sources. Set this to something 121 # else if you don't want to re-fetch required libraries every time. 122 DEPSRC = os.path.join(WORKDIR, 'third-party') 123 DEPSRC = os.path.expanduser('~/Universal/other-sources') 124 125 universal_opts_map = { '32-bit': ('i386', 'ppc',), 126 '64-bit': ('x86_64', 'ppc64',), 127 'intel': ('i386', 'x86_64'), 128 'intel-32': ('i386',), 129 'intel-64': ('x86_64',), 130 '3-way': ('ppc', 'i386', 'x86_64'), 131 'all': ('i386', 'ppc', 'x86_64', 'ppc64',) } 132 default_target_map = { 133 '64-bit': '10.5', 134 '3-way': '10.5', 135 'intel': '10.5', 136 'intel-32': '10.4', 137 'intel-64': '10.5', 138 'all': '10.5', 139 } 140 141 UNIVERSALOPTS = tuple(universal_opts_map.keys()) 142 143 UNIVERSALARCHS = '32-bit' 144 145 ARCHLIST = universal_opts_map[UNIVERSALARCHS] 146 147 # Source directory (assume we're in Mac/BuildScript) 148 SRCDIR = os.path.dirname( 149 os.path.dirname( 150 os.path.dirname( 151 os.path.abspath(__file__ 152 )))) 153 154 # $MACOSX_DEPLOYMENT_TARGET -> minimum OS X level 155 DEPTARGET = '10.5' 156 157 def getDeptargetTuple(): 158 return tuple([int(n) for n in DEPTARGET.split('.')[0:2]]) 159 160 def getTargetCompilers(): 161 target_cc_map = { 162 '10.4': ('gcc-4.0', 'g++-4.0'), 163 '10.5': ('gcc', 'g++'), 164 '10.6': ('gcc', 'g++'), 165 } 166 return target_cc_map.get(DEPTARGET, ('gcc', 'g++') ) 167 168 CC, CXX = getTargetCompilers() 169 170 PYTHON_3 = getVersionMajorMinor() >= (3, 0) 171 172 USAGE = textwrap.dedent("""\ 173 Usage: build_python [options] 174 175 Options: 176 -? or -h: Show this message 177 -b DIR 178 --build-dir=DIR: Create build here (default: %(WORKDIR)r) 179 --third-party=DIR: Store third-party sources here (default: %(DEPSRC)r) 180 --sdk-path=DIR: Location of the SDK (deprecated, use SDKROOT env variable) 181 --src-dir=DIR: Location of the Python sources (default: %(SRCDIR)r) 182 --dep-target=10.n macOS deployment target (default: %(DEPTARGET)r) 183 --universal-archs=x universal architectures (options: %(UNIVERSALOPTS)r, default: %(UNIVERSALARCHS)r) 184 """)% globals() 185 186 # Dict of object file names with shared library names to check after building. 187 # This is to ensure that we ended up dynamically linking with the shared 188 # library paths and versions we expected. For example: 189 # EXPECTED_SHARED_LIBS['_tkinter.so'] = [ 190 # '/Library/Frameworks/Tcl.framework/Versions/8.5/Tcl', 191 # '/Library/Frameworks/Tk.framework/Versions/8.5/Tk'] 192 EXPECTED_SHARED_LIBS = {} 193 194 # Are we building and linking with our own copy of Tcl/TK? 195 # For now, do so if deployment target is 10.6+. 196 def internalTk(): 197 return getDeptargetTuple() >= (10, 6) 198 199 # List of names of third party software built with this installer. 200 # The names will be inserted into the rtf version of the License. 201 THIRD_PARTY_LIBS = [] 202 203 # Instructions for building libraries that are necessary for building a 204 # batteries included python. 205 # [The recipes are defined here for convenience but instantiated later after 206 # command line options have been processed.] 207 def library_recipes(): 208 result = [] 209 210 LT_10_5 = bool(getDeptargetTuple() < (10, 5)) 211 212 # Since Apple removed the header files for the deprecated system 213 # OpenSSL as of the Xcode 7 release (for OS X 10.10+), we do not 214 # have much choice but to build our own copy here, too. 215 216 result.extend([ 217 dict( 218 name="OpenSSL 1.1.0j", 219 url="https://www.openssl.org/source/openssl-1.1.0j.tar.gz", 220 checksum='b4ca5b78ae6ae79da80790b30dbedbdc', 221 buildrecipe=build_universal_openssl, 222 configure=None, 223 install=None, 224 ), 225 ]) 226 227 if internalTk(): 228 result.extend([ 229 dict( 230 name="Tcl 8.6.8", 231 url="ftp://ftp.tcl.tk/pub/tcl//tcl8_6/tcl8.6.8-src.tar.gz", 232 checksum='81656d3367af032e0ae6157eff134f89', 233 buildDir="unix", 234 configure_pre=[ 235 '--enable-shared', 236 '--enable-threads', 237 '--libdir=/Library/Frameworks/Python.framework/Versions/%s/lib'%(getVersion(),), 238 ], 239 useLDFlags=False, 240 install='make TCL_LIBRARY=%(TCL_LIBRARY)s && make install TCL_LIBRARY=%(TCL_LIBRARY)s DESTDIR=%(DESTDIR)s'%{ 241 "DESTDIR": shellQuote(os.path.join(WORKDIR, 'libraries')), 242 "TCL_LIBRARY": shellQuote('/Library/Frameworks/Python.framework/Versions/%s/lib/tcl8.6'%(getVersion())), 243 }, 244 ), 245 dict( 246 name="Tk 8.6.8", 247 url="ftp://ftp.tcl.tk/pub/tcl//tcl8_6/tk8.6.8-src.tar.gz", 248 checksum='5e0faecba458ee1386078fb228d008ba', 249 patches=[ 250 "tk868_on_10_8_10_9.patch", 251 ], 252 buildDir="unix", 253 configure_pre=[ 254 '--enable-aqua', 255 '--enable-shared', 256 '--enable-threads', 257 '--libdir=/Library/Frameworks/Python.framework/Versions/%s/lib'%(getVersion(),), 258 ], 259 useLDFlags=False, 260 install='make TCL_LIBRARY=%(TCL_LIBRARY)s TK_LIBRARY=%(TK_LIBRARY)s && make install TCL_LIBRARY=%(TCL_LIBRARY)s TK_LIBRARY=%(TK_LIBRARY)s DESTDIR=%(DESTDIR)s'%{ 261 "DESTDIR": shellQuote(os.path.join(WORKDIR, 'libraries')), 262 "TCL_LIBRARY": shellQuote('/Library/Frameworks/Python.framework/Versions/%s/lib/tcl8.6'%(getVersion())), 263 "TK_LIBRARY": shellQuote('/Library/Frameworks/Python.framework/Versions/%s/lib/tk8.6'%(getVersion())), 264 }, 265 ), 266 ]) 267 268 if PYTHON_3: 269 result.extend([ 270 dict( 271 name="XZ 5.2.3", 272 url="http://tukaani.org/xz/xz-5.2.3.tar.gz", 273 checksum='ef68674fb47a8b8e741b34e429d86e9d', 274 configure_pre=[ 275 '--disable-dependency-tracking', 276 ] 277 ), 278 ]) 279 280 result.extend([ 281 dict( 282 name="NCurses 5.9", 283 url="http://ftp.gnu.org/pub/gnu/ncurses/ncurses-5.9.tar.gz", 284 checksum='8cb9c412e5f2d96bc6f459aa8c6282a1', 285 configure_pre=[ 286 "--enable-widec", 287 "--without-cxx", 288 "--without-cxx-binding", 289 "--without-ada", 290 "--without-curses-h", 291 "--enable-shared", 292 "--with-shared", 293 "--without-debug", 294 "--without-normal", 295 "--without-tests", 296 "--without-manpages", 297 "--datadir=/usr/share", 298 "--sysconfdir=/etc", 299 "--sharedstatedir=/usr/com", 300 "--with-terminfo-dirs=/usr/share/terminfo", 301 "--with-default-terminfo-dir=/usr/share/terminfo", 302 "--libdir=/Library/Frameworks/Python.framework/Versions/%s/lib"%(getVersion(),), 303 ], 304 patchscripts=[ 305 ("ftp://invisible-island.net/ncurses//5.9/ncurses-5.9-20120616-patch.sh.bz2", 306 "f54bf02a349f96a7c4f0d00922f3a0d4"), 307 ], 308 useLDFlags=False, 309 install='make && make install DESTDIR=%s && cd %s/usr/local/lib && ln -fs ../../../Library/Frameworks/Python.framework/Versions/%s/lib/lib* .'%( 310 shellQuote(os.path.join(WORKDIR, 'libraries')), 311 shellQuote(os.path.join(WORKDIR, 'libraries')), 312 getVersion(), 313 ), 314 ), 315 dict( 316 name="SQLite 3.22.0", 317 url="https://www.sqlite.org/2018/sqlite-autoconf-3220000.tar.gz", 318 checksum='96b5648d542e8afa6ab7ffb8db8ddc3d', 319 extra_cflags=('-Os ' 320 '-DSQLITE_ENABLE_FTS5 ' 321 '-DSQLITE_ENABLE_FTS4 ' 322 '-DSQLITE_ENABLE_FTS3_PARENTHESIS ' 323 '-DSQLITE_ENABLE_JSON1 ' 324 '-DSQLITE_ENABLE_RTREE ' 325 '-DSQLITE_TCL=0 ' 326 '%s' % ('','-DSQLITE_WITHOUT_ZONEMALLOC ')[LT_10_5]), 327 configure_pre=[ 328 '--enable-threadsafe', 329 '--enable-shared=no', 330 '--enable-static=yes', 331 '--disable-readline', 332 '--disable-dependency-tracking', 333 ] 334 ), 335 ]) 336 337 if getDeptargetTuple() < (10, 5): 338 result.extend([ 339 dict( 340 name="Bzip2 1.0.6", 341 url="http://bzip.org/1.0.6/bzip2-1.0.6.tar.gz", 342 checksum='00b516f4704d4a7cb50a1d97e6e8e15b', 343 configure=None, 344 install='make install CC=%s CXX=%s, PREFIX=%s/usr/local/ CFLAGS="-arch %s"'%( 345 CC, CXX, 346 shellQuote(os.path.join(WORKDIR, 'libraries')), 347 ' -arch '.join(ARCHLIST), 348 ), 349 ), 350 dict( 351 name="ZLib 1.2.3", 352 url="http://www.gzip.org/zlib/zlib-1.2.3.tar.gz", 353 checksum='debc62758716a169df9f62e6ab2bc634', 354 configure=None, 355 install='make install CC=%s CXX=%s, prefix=%s/usr/local/ CFLAGS="-arch %s"'%( 356 CC, CXX, 357 shellQuote(os.path.join(WORKDIR, 'libraries')), 358 ' -arch '.join(ARCHLIST), 359 ), 360 ), 361 dict( 362 # Note that GNU readline is GPL'd software 363 name="GNU Readline 6.1.2", 364 url="http://ftp.gnu.org/pub/gnu/readline/readline-6.1.tar.gz" , 365 checksum='fc2f7e714fe792db1ce6ddc4c9fb4ef3', 366 patchlevel='0', 367 patches=[ 368 # The readline maintainers don't do actual micro releases, but 369 # just ship a set of patches. 370 ('http://ftp.gnu.org/pub/gnu/readline/readline-6.1-patches/readline61-001', 371 'c642f2e84d820884b0bf9fd176bc6c3f'), 372 ('http://ftp.gnu.org/pub/gnu/readline/readline-6.1-patches/readline61-002', 373 '1a76781a1ea734e831588285db7ec9b1'), 374 ] 375 ), 376 ]) 377 378 if not PYTHON_3: 379 result.extend([ 380 dict( 381 name="Sleepycat DB 4.7.25", 382 url="http://download.oracle.com/berkeley-db/db-4.7.25.tar.gz", 383 checksum='ec2b87e833779681a0c3a814aa71359e', 384 buildDir="build_unix", 385 configure="../dist/configure", 386 configure_pre=[ 387 '--includedir=/usr/local/include/db4', 388 ] 389 ), 390 ]) 391 392 return result 393 394 395 # Instructions for building packages inside the .mpkg. 396 def pkg_recipes(): 397 unselected_for_python3 = ('selected', 'unselected')[PYTHON_3] 398 result = [ 399 dict( 400 name="PythonFramework", 401 long_name="Python Framework", 402 source="/Library/Frameworks/Python.framework", 403 readme="""\ 404 This package installs Python.framework, that is the python 405 interpreter and the standard library. 406 """, 407 postflight="scripts/postflight.framework", 408 selected='selected', 409 ), 410 dict( 411 name="PythonApplications", 412 long_name="GUI Applications", 413 source="/Applications/Python %(VER)s", 414 readme="""\ 415 This package installs IDLE (an interactive Python IDE), 416 Python Launcher and Build Applet (create application bundles 417 from python scripts). 418 419 It also installs a number of examples and demos. 420 """, 421 required=False, 422 selected='selected', 423 ), 424 dict( 425 name="PythonUnixTools", 426 long_name="UNIX command-line tools", 427 source="/usr/local/bin", 428 readme="""\ 429 This package installs the unix tools in /usr/local/bin for 430 compatibility with older releases of Python. This package 431 is not necessary to use Python. 432 """, 433 required=False, 434 selected='selected', 435 ), 436 dict( 437 name="PythonDocumentation", 438 long_name="Python Documentation", 439 topdir="/Library/Frameworks/Python.framework/Versions/%(VER)s/Resources/English.lproj/Documentation", 440 source="/pydocs", 441 readme="""\ 442 This package installs the python documentation at a location 443 that is useable for pydoc and IDLE. 444 """, 445 postflight="scripts/postflight.documentation", 446 required=False, 447 selected='selected', 448 ), 449 dict( 450 name="PythonProfileChanges", 451 long_name="Shell profile updater", 452 readme="""\ 453 This packages updates your shell profile to make sure that 454 the Python tools are found by your shell in preference of 455 the system provided Python tools. 456 457 If you don't install this package you'll have to add 458 "/Library/Frameworks/Python.framework/Versions/%(VER)s/bin" 459 to your PATH by hand. 460 """, 461 postflight="scripts/postflight.patch-profile", 462 topdir="/Library/Frameworks/Python.framework", 463 source="/empty-dir", 464 required=False, 465 selected='selected', 466 ), 467 dict( 468 name="PythonInstallPip", 469 long_name="Install or upgrade pip", 470 readme="""\ 471 This package installs (or upgrades from an earlier version) 472 pip, a tool for installing and managing Python packages. 473 """, 474 postflight="scripts/postflight.ensurepip", 475 topdir="/Library/Frameworks/Python.framework", 476 source="/empty-dir", 477 required=False, 478 selected='selected', 479 ), 480 ] 481 482 return result 483 484 def fatal(msg): 485 """ 486 A fatal error, bail out. 487 """ 488 sys.stderr.write('FATAL: ') 489 sys.stderr.write(msg) 490 sys.stderr.write('\n') 491 sys.exit(1) 492 493 def fileContents(fn): 494 """ 495 Return the contents of the named file 496 """ 497 return open(fn, 'r').read() 498 499 def runCommand(commandline): 500 """ 501 Run a command and raise RuntimeError if it fails. Output is suppressed 502 unless the command fails. 503 """ 504 fd = os.popen(commandline, 'r') 505 data = fd.read() 506 xit = fd.close() 507 if xit is not None: 508 sys.stdout.write(data) 509 raise RuntimeError("command failed: %s"%(commandline,)) 510 511 if VERBOSE: 512 sys.stdout.write(data); sys.stdout.flush() 513 514 def captureCommand(commandline): 515 fd = os.popen(commandline, 'r') 516 data = fd.read() 517 xit = fd.close() 518 if xit is not None: 519 sys.stdout.write(data) 520 raise RuntimeError("command failed: %s"%(commandline,)) 521 522 return data 523 524 def getTclTkVersion(configfile, versionline): 525 """ 526 search Tcl or Tk configuration file for version line 527 """ 528 try: 529 f = open(configfile, "r") 530 except OSError: 531 fatal("Framework configuration file not found: %s" % configfile) 532 533 for l in f: 534 if l.startswith(versionline): 535 f.close() 536 return l 537 538 fatal("Version variable %s not found in framework configuration file: %s" 539 % (versionline, configfile)) 540 541 def checkEnvironment(): 542 """ 543 Check that we're running on a supported system. 544 """ 545 546 if sys.version_info[0:2] < (2, 5): 547 fatal("This script must be run with Python 2.5 (or later)") 548 549 if platform.system() != 'Darwin': 550 fatal("This script should be run on a macOS 10.5 (or later) system") 551 552 if int(platform.release().split('.')[0]) < 8: 553 fatal("This script should be run on a macOS 10.5 (or later) system") 554 555 # Because we only support dynamic load of only one major/minor version of 556 # Tcl/Tk, if we are not using building and using our own private copy of 557 # Tcl/Tk, ensure: 558 # 1. there is a user-installed framework (usually ActiveTcl) in (or linked 559 # in) SDKROOT/Library/Frameworks. As of Python 3.7.0, we no longer 560 # enforce that the version of the user-installed framework also 561 # exists in the system-supplied Tcl/Tk frameworks. Time to support 562 # Tcl/Tk 8.6 even if Apple does not. 563 if not internalTk(): 564 frameworks = {} 565 for framework in ['Tcl', 'Tk']: 566 fwpth = 'Library/Frameworks/%s.framework/Versions/Current' % framework 567 libfw = os.path.join('/', fwpth) 568 usrfw = os.path.join(os.getenv('HOME'), fwpth) 569 frameworks[framework] = os.readlink(libfw) 570 if not os.path.exists(libfw): 571 fatal("Please install a link to a current %s %s as %s so " 572 "the user can override the system framework." 573 % (framework, frameworks[framework], libfw)) 574 if os.path.exists(usrfw): 575 fatal("Please rename %s to avoid possible dynamic load issues." 576 % usrfw) 577 578 if frameworks['Tcl'] != frameworks['Tk']: 579 fatal("The Tcl and Tk frameworks are not the same version.") 580 581 print(" -- Building with external Tcl/Tk %s frameworks" 582 % frameworks['Tk']) 583 584 # add files to check after build 585 EXPECTED_SHARED_LIBS['_tkinter.so'] = [ 586 "/Library/Frameworks/Tcl.framework/Versions/%s/Tcl" 587 % frameworks['Tcl'], 588 "/Library/Frameworks/Tk.framework/Versions/%s/Tk" 589 % frameworks['Tk'], 590 ] 591 else: 592 print(" -- Building private copy of Tcl/Tk") 593 print("") 594 595 # Remove inherited environment variables which might influence build 596 environ_var_prefixes = ['CPATH', 'C_INCLUDE_', 'DYLD_', 'LANG', 'LC_', 597 'LD_', 'LIBRARY_', 'PATH', 'PYTHON'] 598 for ev in list(os.environ): 599 for prefix in environ_var_prefixes: 600 if ev.startswith(prefix) : 601 print("INFO: deleting environment variable %s=%s" % ( 602 ev, os.environ[ev])) 603 del os.environ[ev] 604 605 base_path = '/bin:/sbin:/usr/bin:/usr/sbin' 606 if 'SDK_TOOLS_BIN' in os.environ: 607 base_path = os.environ['SDK_TOOLS_BIN'] + ':' + base_path 608 # Xcode 2.5 on OS X 10.4 does not include SetFile in its usr/bin; 609 # add its fixed location here if it exists 610 OLD_DEVELOPER_TOOLS = '/Developer/Tools' 611 if os.path.isdir(OLD_DEVELOPER_TOOLS): 612 base_path = base_path + ':' + OLD_DEVELOPER_TOOLS 613 os.environ['PATH'] = base_path 614 print("Setting default PATH: %s"%(os.environ['PATH'])) 615 # Ensure we have access to sphinx-build. 616 # You may have to create a link in /usr/bin for it. 617 runCommand('sphinx-build --version') 618 619 def parseOptions(args=None): 620 """ 621 Parse arguments and update global settings. 622 """ 623 global WORKDIR, DEPSRC, SRCDIR, DEPTARGET 624 global UNIVERSALOPTS, UNIVERSALARCHS, ARCHLIST, CC, CXX 625 global FW_VERSION_PREFIX 626 global FW_SSL_DIRECTORY 627 628 if args is None: 629 args = sys.argv[1:] 630 631 try: 632 options, args = getopt.getopt(args, '?hb', 633 [ 'build-dir=', 'third-party=', 'sdk-path=' , 'src-dir=', 634 'dep-target=', 'universal-archs=', 'help' ]) 635 except getopt.GetoptError: 636 print(sys.exc_info()[1]) 637 sys.exit(1) 638 639 if args: 640 print("Additional arguments") 641 sys.exit(1) 642 643 deptarget = None 644 for k, v in options: 645 if k in ('-h', '-?', '--help'): 646 print(USAGE) 647 sys.exit(0) 648 649 elif k in ('-d', '--build-dir'): 650 WORKDIR=v 651 652 elif k in ('--third-party',): 653 DEPSRC=v 654 655 elif k in ('--sdk-path',): 656 print(" WARNING: --sdk-path is no longer supported") 657 658 elif k in ('--src-dir',): 659 SRCDIR=v 660 661 elif k in ('--dep-target', ): 662 DEPTARGET=v 663 deptarget=v 664 665 elif k in ('--universal-archs', ): 666 if v in UNIVERSALOPTS: 667 UNIVERSALARCHS = v 668 ARCHLIST = universal_opts_map[UNIVERSALARCHS] 669 if deptarget is None: 670 # Select alternate default deployment 671 # target 672 DEPTARGET = default_target_map.get(v, '10.5') 673 else: 674 raise NotImplementedError(v) 675 676 else: 677 raise NotImplementedError(k) 678 679 SRCDIR=os.path.abspath(SRCDIR) 680 WORKDIR=os.path.abspath(WORKDIR) 681 DEPSRC=os.path.abspath(DEPSRC) 682 683 CC, CXX = getTargetCompilers() 684 685 FW_VERSION_PREFIX = FW_PREFIX[:] + ["Versions", getVersion()] 686 FW_SSL_DIRECTORY = FW_VERSION_PREFIX[:] + ["etc", "openssl"] 687 688 print("-- Settings:") 689 print(" * Source directory: %s" % SRCDIR) 690 print(" * Build directory: %s" % WORKDIR) 691 print(" * Third-party source: %s" % DEPSRC) 692 print(" * Deployment target: %s" % DEPTARGET) 693 print(" * Universal archs: %s" % str(ARCHLIST)) 694 print(" * C compiler: %s" % CC) 695 print(" * C++ compiler: %s" % CXX) 696 print("") 697 print(" -- Building a Python %s framework at patch level %s" 698 % (getVersion(), getFullVersion())) 699 print("") 700 701 def extractArchive(builddir, archiveName): 702 """ 703 Extract a source archive into 'builddir'. Returns the path of the 704 extracted archive. 705 706 XXX: This function assumes that archives contain a toplevel directory 707 that is has the same name as the basename of the archive. This is 708 safe enough for almost anything we use. Unfortunately, it does not 709 work for current Tcl and Tk source releases where the basename of 710 the archive ends with "-src" but the uncompressed directory does not. 711 For now, just special case Tcl and Tk tar.gz downloads. 712 """ 713 curdir = os.getcwd() 714 try: 715 os.chdir(builddir) 716 if archiveName.endswith('.tar.gz'): 717 retval = os.path.basename(archiveName[:-7]) 718 if ((retval.startswith('tcl') or retval.startswith('tk')) 719 and retval.endswith('-src')): 720 retval = retval[:-4] 721 if os.path.exists(retval): 722 shutil.rmtree(retval) 723 fp = os.popen("tar zxf %s 2>&1"%(shellQuote(archiveName),), 'r') 724 725 elif archiveName.endswith('.tar.bz2'): 726 retval = os.path.basename(archiveName[:-8]) 727 if os.path.exists(retval): 728 shutil.rmtree(retval) 729 fp = os.popen("tar jxf %s 2>&1"%(shellQuote(archiveName),), 'r') 730 731 elif archiveName.endswith('.tar'): 732 retval = os.path.basename(archiveName[:-4]) 733 if os.path.exists(retval): 734 shutil.rmtree(retval) 735 fp = os.popen("tar xf %s 2>&1"%(shellQuote(archiveName),), 'r') 736 737 elif archiveName.endswith('.zip'): 738 retval = os.path.basename(archiveName[:-4]) 739 if os.path.exists(retval): 740 shutil.rmtree(retval) 741 fp = os.popen("unzip %s 2>&1"%(shellQuote(archiveName),), 'r') 742 743 data = fp.read() 744 xit = fp.close() 745 if xit is not None: 746 sys.stdout.write(data) 747 raise RuntimeError("Cannot extract %s"%(archiveName,)) 748 749 return os.path.join(builddir, retval) 750 751 finally: 752 os.chdir(curdir) 753 754 def downloadURL(url, fname): 755 """ 756 Download the contents of the url into the file. 757 """ 758 fpIn = urllib_request.urlopen(url) 759 fpOut = open(fname, 'wb') 760 block = fpIn.read(10240) 761 try: 762 while block: 763 fpOut.write(block) 764 block = fpIn.read(10240) 765 fpIn.close() 766 fpOut.close() 767 except: 768 try: 769 os.unlink(fname) 770 except OSError: 771 pass 772 773 def verifyThirdPartyFile(url, checksum, fname): 774 """ 775 Download file from url to filename fname if it does not already exist. 776 Abort if file contents does not match supplied md5 checksum. 777 """ 778 name = os.path.basename(fname) 779 if os.path.exists(fname): 780 print("Using local copy of %s"%(name,)) 781 else: 782 print("Did not find local copy of %s"%(name,)) 783 print("Downloading %s"%(name,)) 784 downloadURL(url, fname) 785 print("Archive for %s stored as %s"%(name, fname)) 786 if os.system( 787 'MD5=$(openssl md5 %s) ; test "${MD5##*= }" = "%s"' 788 % (shellQuote(fname), checksum) ): 789 fatal('MD5 checksum mismatch for file %s' % fname) 790 791 def build_universal_openssl(basedir, archList): 792 """ 793 Special case build recipe for universal build of openssl. 794 795 The upstream OpenSSL build system does not directly support 796 OS X universal builds. We need to build each architecture 797 separately then lipo them together into fat libraries. 798 """ 799 800 # OpenSSL fails to build with Xcode 2.5 (on OS X 10.4). 801 # If we are building on a 10.4.x or earlier system, 802 # unilaterally disable assembly code building to avoid the problem. 803 no_asm = int(platform.release().split(".")[0]) < 9 804 805 def build_openssl_arch(archbase, arch): 806 "Build one architecture of openssl" 807 arch_opts = { 808 "i386": ["darwin-i386-cc"], 809 "x86_64": ["darwin64-x86_64-cc", "enable-ec_nistp_64_gcc_128"], 810 "ppc": ["darwin-ppc-cc"], 811 "ppc64": ["darwin64-ppc-cc"], 812 } 813 configure_opts = [ 814 "no-idea", 815 "no-mdc2", 816 "no-rc5", 817 "no-zlib", 818 "no-ssl3", 819 # "enable-unit-test", 820 "shared", 821 "--prefix=%s"%os.path.join("/", *FW_VERSION_PREFIX), 822 "--openssldir=%s"%os.path.join("/", *FW_SSL_DIRECTORY), 823 ] 824 if no_asm: 825 configure_opts.append("no-asm") 826 runCommand(" ".join(["perl", "Configure"] 827 + arch_opts[arch] + configure_opts)) 828 runCommand("make depend") 829 runCommand("make all") 830 runCommand("make install_sw DESTDIR=%s"%shellQuote(archbase)) 831 # runCommand("make test") 832 return 833 834 srcdir = os.getcwd() 835 universalbase = os.path.join(srcdir, "..", 836 os.path.basename(srcdir) + "-universal") 837 os.mkdir(universalbase) 838 archbasefws = [] 839 for arch in archList: 840 # fresh copy of the source tree 841 archsrc = os.path.join(universalbase, arch, "src") 842 shutil.copytree(srcdir, archsrc, symlinks=True) 843 # install base for this arch 844 archbase = os.path.join(universalbase, arch, "root") 845 os.mkdir(archbase) 846 # Python framework base within install_prefix: 847 # the build will install into this framework.. 848 # This is to ensure that the resulting shared libs have 849 # the desired real install paths built into them. 850 archbasefw = os.path.join(archbase, *FW_VERSION_PREFIX) 851 852 # build one architecture 853 os.chdir(archsrc) 854 build_openssl_arch(archbase, arch) 855 os.chdir(srcdir) 856 archbasefws.append(archbasefw) 857 858 # copy arch-independent files from last build into the basedir framework 859 basefw = os.path.join(basedir, *FW_VERSION_PREFIX) 860 shutil.copytree( 861 os.path.join(archbasefw, "include", "openssl"), 862 os.path.join(basefw, "include", "openssl") 863 ) 864 865 shlib_version_number = grepValue(os.path.join(archsrc, "Makefile"), 866 "SHLIB_VERSION_NUMBER") 867 # e.g. -> "1.0.0" 868 libcrypto = "libcrypto.dylib" 869 libcrypto_versioned = libcrypto.replace(".", "."+shlib_version_number+".") 870 # e.g. -> "libcrypto.1.0.0.dylib" 871 libssl = "libssl.dylib" 872 libssl_versioned = libssl.replace(".", "."+shlib_version_number+".") 873 # e.g. -> "libssl.1.0.0.dylib" 874 875 try: 876 os.mkdir(os.path.join(basefw, "lib")) 877 except OSError: 878 pass 879 880 # merge the individual arch-dependent shared libs into a fat shared lib 881 archbasefws.insert(0, basefw) 882 for (lib_unversioned, lib_versioned) in [ 883 (libcrypto, libcrypto_versioned), 884 (libssl, libssl_versioned) 885 ]: 886 runCommand("lipo -create -output " + 887 " ".join(shellQuote( 888 os.path.join(fw, "lib", lib_versioned)) 889 for fw in archbasefws)) 890 # and create an unversioned symlink of it 891 os.symlink(lib_versioned, os.path.join(basefw, "lib", lib_unversioned)) 892 893 # Create links in the temp include and lib dirs that will be injected 894 # into the Python build so that setup.py can find them while building 895 # and the versioned links so that the setup.py post-build import test 896 # does not fail. 897 relative_path = os.path.join("..", "..", "..", *FW_VERSION_PREFIX) 898 for fn in [ 899 ["include", "openssl"], 900 ["lib", libcrypto], 901 ["lib", libssl], 902 ["lib", libcrypto_versioned], 903 ["lib", libssl_versioned], 904 ]: 905 os.symlink( 906 os.path.join(relative_path, *fn), 907 os.path.join(basedir, "usr", "local", *fn) 908 ) 909 910 return 911 912 def buildRecipe(recipe, basedir, archList): 913 """ 914 Build software using a recipe. This function does the 915 'configure;make;make install' dance for C software, with a possibility 916 to customize this process, basically a poor-mans DarwinPorts. 917 """ 918 curdir = os.getcwd() 919 920 name = recipe['name'] 921 THIRD_PARTY_LIBS.append(name) 922 url = recipe['url'] 923 configure = recipe.get('configure', './configure') 924 buildrecipe = recipe.get('buildrecipe', None) 925 install = recipe.get('install', 'make && make install DESTDIR=%s'%( 926 shellQuote(basedir))) 927 928 archiveName = os.path.split(url)[-1] 929 sourceArchive = os.path.join(DEPSRC, archiveName) 930 931 if not os.path.exists(DEPSRC): 932 os.mkdir(DEPSRC) 933 934 verifyThirdPartyFile(url, recipe['checksum'], sourceArchive) 935 print("Extracting archive for %s"%(name,)) 936 buildDir=os.path.join(WORKDIR, '_bld') 937 if not os.path.exists(buildDir): 938 os.mkdir(buildDir) 939 940 workDir = extractArchive(buildDir, sourceArchive) 941 os.chdir(workDir) 942 943 for patch in recipe.get('patches', ()): 944 if isinstance(patch, tuple): 945 url, checksum = patch 946 fn = os.path.join(DEPSRC, os.path.basename(url)) 947 verifyThirdPartyFile(url, checksum, fn) 948 else: 949 # patch is a file in the source directory 950 fn = os.path.join(curdir, patch) 951 runCommand('patch -p%s < %s'%(recipe.get('patchlevel', 1), 952 shellQuote(fn),)) 953 954 for patchscript in recipe.get('patchscripts', ()): 955 if isinstance(patchscript, tuple): 956 url, checksum = patchscript 957 fn = os.path.join(DEPSRC, os.path.basename(url)) 958 verifyThirdPartyFile(url, checksum, fn) 959 else: 960 # patch is a file in the source directory 961 fn = os.path.join(curdir, patchscript) 962 if fn.endswith('.bz2'): 963 runCommand('bunzip2 -fk %s' % shellQuote(fn)) 964 fn = fn[:-4] 965 runCommand('sh %s' % shellQuote(fn)) 966 os.unlink(fn) 967 968 if 'buildDir' in recipe: 969 os.chdir(recipe['buildDir']) 970 971 if configure is not None: 972 configure_args = [ 973 "--prefix=/usr/local", 974 "--enable-static", 975 "--disable-shared", 976 #"CPP=gcc -arch %s -E"%(' -arch '.join(archList,),), 977 ] 978 979 if 'configure_pre' in recipe: 980 args = list(recipe['configure_pre']) 981 if '--disable-static' in args: 982 configure_args.remove('--enable-static') 983 if '--enable-shared' in args: 984 configure_args.remove('--disable-shared') 985 configure_args.extend(args) 986 987 if recipe.get('useLDFlags', 1): 988 configure_args.extend([ 989 "CFLAGS=%s-mmacosx-version-min=%s -arch %s " 990 "-I%s/usr/local/include"%( 991 recipe.get('extra_cflags', ''), 992 DEPTARGET, 993 ' -arch '.join(archList), 994 shellQuote(basedir)[1:-1],), 995 "LDFLAGS=-mmacosx-version-min=%s -L%s/usr/local/lib -arch %s"%( 996 DEPTARGET, 997 shellQuote(basedir)[1:-1], 998 ' -arch '.join(archList)), 999 ]) 1000 else: 1001 configure_args.extend([ 1002 "CFLAGS=%s-mmacosx-version-min=%s -arch %s " 1003 "-I%s/usr/local/include"%( 1004 recipe.get('extra_cflags', ''), 1005 DEPTARGET, 1006 ' -arch '.join(archList), 1007 shellQuote(basedir)[1:-1],), 1008 ]) 1009 1010 if 'configure_post' in recipe: 1011 configure_args = configure_args + list(recipe['configure_post']) 1012 1013 configure_args.insert(0, configure) 1014 configure_args = [ shellQuote(a) for a in configure_args ] 1015 1016 print("Running configure for %s"%(name,)) 1017 runCommand(' '.join(configure_args) + ' 2>&1') 1018 1019 if buildrecipe is not None: 1020 # call special-case build recipe, e.g. for openssl 1021 buildrecipe(basedir, archList) 1022 1023 if install is not None: 1024 print("Running install for %s"%(name,)) 1025 runCommand('{ ' + install + ' ;} 2>&1') 1026 1027 print("Done %s"%(name,)) 1028 print("") 1029 1030 os.chdir(curdir) 1031 1032 def buildLibraries(): 1033 """ 1034 Build our dependencies into $WORKDIR/libraries/usr/local 1035 """ 1036 print("") 1037 print("Building required libraries") 1038 print("") 1039 universal = os.path.join(WORKDIR, 'libraries') 1040 os.mkdir(universal) 1041 os.makedirs(os.path.join(universal, 'usr', 'local', 'lib')) 1042 os.makedirs(os.path.join(universal, 'usr', 'local', 'include')) 1043 1044 for recipe in library_recipes(): 1045 buildRecipe(recipe, universal, ARCHLIST) 1046 1047 1048 1049 def buildPythonDocs(): 1050 # This stores the documentation as Resources/English.lproj/Documentation 1051 # inside the framework. pydoc and IDLE will pick it up there. 1052 print("Install python documentation") 1053 rootDir = os.path.join(WORKDIR, '_root') 1054 buildDir = os.path.join('../../Doc') 1055 docdir = os.path.join(rootDir, 'pydocs') 1056 curDir = os.getcwd() 1057 os.chdir(buildDir) 1058 runCommand('make clean') 1059 # Create virtual environment for docs builds with blurb and sphinx 1060 runCommand('make venv') 1061 runCommand('make html PYTHON=venv/bin/python') 1062 os.chdir(curDir) 1063 if not os.path.exists(docdir): 1064 os.mkdir(docdir) 1065 os.rename(os.path.join(buildDir, 'build', 'html'), docdir) 1066 1067 1068 def buildPython(): 1069 print("Building a universal python for %s architectures" % UNIVERSALARCHS) 1070 1071 buildDir = os.path.join(WORKDIR, '_bld', 'python') 1072 rootDir = os.path.join(WORKDIR, '_root') 1073 1074 if os.path.exists(buildDir): 1075 shutil.rmtree(buildDir) 1076 if os.path.exists(rootDir): 1077 shutil.rmtree(rootDir) 1078 os.makedirs(buildDir) 1079 os.makedirs(rootDir) 1080 os.makedirs(os.path.join(rootDir, 'empty-dir')) 1081 curdir = os.getcwd() 1082 os.chdir(buildDir) 1083 1084 # Extract the version from the configure file, needed to calculate 1085 # several paths. 1086 version = getVersion() 1087 1088 # Since the extra libs are not in their installed framework location 1089 # during the build, augment the library path so that the interpreter 1090 # will find them during its extension import sanity checks. 1091 os.environ['DYLD_LIBRARY_PATH'] = os.path.join(WORKDIR, 1092 'libraries', 'usr', 'local', 'lib') 1093 print("Running configure...") 1094 runCommand("%s -C --enable-framework --enable-universalsdk=/ " 1095 "--with-universal-archs=%s " 1096 "%s " 1097 "%s " 1098 "%s " 1099 "%s " 1100 "LDFLAGS='-g -L%s/libraries/usr/local/lib' " 1101 "CFLAGS='-g -I%s/libraries/usr/local/include' 2>&1"%( 1102 shellQuote(os.path.join(SRCDIR, 'configure')), 1103 UNIVERSALARCHS, 1104 (' ', '--with-computed-gotos ')[PYTHON_3], 1105 (' ', '--without-ensurepip ')[PYTHON_3], 1106 (' ', "--with-tcltk-includes='-I%s/libraries/usr/local/include'"%( 1107 shellQuote(WORKDIR)[1:-1],))[internalTk()], 1108 (' ', "--with-tcltk-libs='-L%s/libraries/usr/local/lib -ltcl8.6 -ltk8.6'"%( 1109 shellQuote(WORKDIR)[1:-1],))[internalTk()], 1110 shellQuote(WORKDIR)[1:-1], 1111 shellQuote(WORKDIR)[1:-1])) 1112 1113 # Look for environment value BUILDINSTALLER_BUILDPYTHON_MAKE_EXTRAS 1114 # and, if defined, append its value to the make command. This allows 1115 # us to pass in version control tags, like GITTAG, to a build from a 1116 # tarball rather than from a vcs checkout, thus eliminating the need 1117 # to have a working copy of the vcs program on the build machine. 1118 # 1119 # A typical use might be: 1120 # export BUILDINSTALLER_BUILDPYTHON_MAKE_EXTRAS=" \ 1121 # GITVERSION='echo 123456789a' \ 1122 # GITTAG='echo v3.6.0' \ 1123 # GITBRANCH='echo 3.6'" 1124 1125 make_extras = os.getenv("BUILDINSTALLER_BUILDPYTHON_MAKE_EXTRAS") 1126 if make_extras: 1127 make_cmd = "make " + make_extras 1128 else: 1129 make_cmd = "make" 1130 print("Running " + make_cmd) 1131 runCommand(make_cmd) 1132 1133 print("Running make install") 1134 runCommand("make install DESTDIR=%s"%( 1135 shellQuote(rootDir))) 1136 1137 print("Running make frameworkinstallextras") 1138 runCommand("make frameworkinstallextras DESTDIR=%s"%( 1139 shellQuote(rootDir))) 1140 1141 del os.environ['DYLD_LIBRARY_PATH'] 1142 print("Copying required shared libraries") 1143 if os.path.exists(os.path.join(WORKDIR, 'libraries', 'Library')): 1144 build_lib_dir = os.path.join( 1145 WORKDIR, 'libraries', 'Library', 'Frameworks', 1146 'Python.framework', 'Versions', getVersion(), 'lib') 1147 fw_lib_dir = os.path.join( 1148 WORKDIR, '_root', 'Library', 'Frameworks', 1149 'Python.framework', 'Versions', getVersion(), 'lib') 1150 if internalTk(): 1151 # move Tcl and Tk pkgconfig files 1152 runCommand("mv %s/pkgconfig/* %s/pkgconfig"%( 1153 shellQuote(build_lib_dir), 1154 shellQuote(fw_lib_dir) )) 1155 runCommand("rm -r %s/pkgconfig"%( 1156 shellQuote(build_lib_dir), )) 1157 runCommand("mv %s/* %s"%( 1158 shellQuote(build_lib_dir), 1159 shellQuote(fw_lib_dir) )) 1160 1161 frmDir = os.path.join(rootDir, 'Library', 'Frameworks', 'Python.framework') 1162 frmDirVersioned = os.path.join(frmDir, 'Versions', version) 1163 path_to_lib = os.path.join(frmDirVersioned, 'lib', 'python%s'%(version,)) 1164 # create directory for OpenSSL certificates 1165 sslDir = os.path.join(frmDirVersioned, 'etc', 'openssl') 1166 os.makedirs(sslDir) 1167 1168 print("Fix file modes") 1169 gid = grp.getgrnam('admin').gr_gid 1170 1171 shared_lib_error = False 1172 for dirpath, dirnames, filenames in os.walk(frmDir): 1173 for dn in dirnames: 1174 os.chmod(os.path.join(dirpath, dn), STAT_0o775) 1175 os.chown(os.path.join(dirpath, dn), -1, gid) 1176 1177 for fn in filenames: 1178 if os.path.islink(fn): 1179 continue 1180 1181 # "chmod g+w $fn" 1182 p = os.path.join(dirpath, fn) 1183 st = os.stat(p) 1184 os.chmod(p, stat.S_IMODE(st.st_mode) | stat.S_IWGRP) 1185 os.chown(p, -1, gid) 1186 1187 if fn in EXPECTED_SHARED_LIBS: 1188 # check to see that this file was linked with the 1189 # expected library path and version 1190 data = captureCommand("otool -L %s" % shellQuote(p)) 1191 for sl in EXPECTED_SHARED_LIBS[fn]: 1192 if ("\t%s " % sl) not in data: 1193 print("Expected shared lib %s was not linked with %s" 1194 % (sl, p)) 1195 shared_lib_error = True 1196 1197 if shared_lib_error: 1198 fatal("Unexpected shared library errors.") 1199 1200 if PYTHON_3: 1201 LDVERSION=None 1202 VERSION=None 1203 ABIFLAGS=None 1204 1205 fp = open(os.path.join(buildDir, 'Makefile'), 'r') 1206 for ln in fp: 1207 if ln.startswith('VERSION='): 1208 VERSION=ln.split()[1] 1209 if ln.startswith('ABIFLAGS='): 1210 ABIFLAGS=ln.split()[1] 1211 if ln.startswith('LDVERSION='): 1212 LDVERSION=ln.split()[1] 1213 fp.close() 1214 1215 LDVERSION = LDVERSION.replace('$(VERSION)', VERSION) 1216 LDVERSION = LDVERSION.replace('$(ABIFLAGS)', ABIFLAGS) 1217 config_suffix = '-' + LDVERSION 1218 if getVersionMajorMinor() >= (3, 6): 1219 config_suffix = config_suffix + '-darwin' 1220 else: 1221 config_suffix = '' # Python 2.x 1222 1223 # We added some directories to the search path during the configure 1224 # phase. Remove those because those directories won't be there on 1225 # the end-users system. Also remove the directories from _sysconfigdata.py 1226 # (added in 3.3) if it exists. 1227 1228 include_path = '-I%s/libraries/usr/local/include' % (WORKDIR,) 1229 lib_path = '-L%s/libraries/usr/local/lib' % (WORKDIR,) 1230 1231 # fix Makefile 1232 path = os.path.join(path_to_lib, 'config' + config_suffix, 'Makefile') 1233 fp = open(path, 'r') 1234 data = fp.read() 1235 fp.close() 1236 1237 for p in (include_path, lib_path): 1238 data = data.replace(" " + p, '') 1239 data = data.replace(p + " ", '') 1240 1241 fp = open(path, 'w') 1242 fp.write(data) 1243 fp.close() 1244 1245 # fix _sysconfigdata 1246 # 1247 # TODO: make this more robust! test_sysconfig_module of 1248 # distutils.tests.test_sysconfig.SysconfigTestCase tests that 1249 # the output from get_config_var in both sysconfig and 1250 # distutils.sysconfig is exactly the same for both CFLAGS and 1251 # LDFLAGS. The fixing up is now complicated by the pretty 1252 # printing in _sysconfigdata.py. Also, we are using the 1253 # pprint from the Python running the installer build which 1254 # may not cosmetically format the same as the pprint in the Python 1255 # being built (and which is used to originally generate 1256 # _sysconfigdata.py). 1257 1258 import pprint 1259 if getVersionMajorMinor() >= (3, 6): 1260 # XXX this is extra-fragile 1261 path = os.path.join(path_to_lib, '_sysconfigdata_m_darwin_darwin.py') 1262 else: 1263 path = os.path.join(path_to_lib, '_sysconfigdata.py') 1264 fp = open(path, 'r') 1265 data = fp.read() 1266 fp.close() 1267 # create build_time_vars dict 1268 exec(data) 1269 vars = {} 1270 for k, v in build_time_vars.items(): 1271 if type(v) == type(''): 1272 for p in (include_path, lib_path): 1273 v = v.replace(' ' + p, '') 1274 v = v.replace(p + ' ', '') 1275 vars[k] = v 1276 1277 fp = open(path, 'w') 1278 # duplicated from sysconfig._generate_posix_vars() 1279 fp.write('# system configuration generated and used by' 1280 ' the sysconfig module\n') 1281 fp.write('build_time_vars = ') 1282 pprint.pprint(vars, stream=fp) 1283 fp.close() 1284 1285 # Add symlinks in /usr/local/bin, using relative links 1286 usr_local_bin = os.path.join(rootDir, 'usr', 'local', 'bin') 1287 to_framework = os.path.join('..', '..', '..', 'Library', 'Frameworks', 1288 'Python.framework', 'Versions', version, 'bin') 1289 if os.path.exists(usr_local_bin): 1290 shutil.rmtree(usr_local_bin) 1291 os.makedirs(usr_local_bin) 1292 for fn in os.listdir( 1293 os.path.join(frmDir, 'Versions', version, 'bin')): 1294 os.symlink(os.path.join(to_framework, fn), 1295 os.path.join(usr_local_bin, fn)) 1296 1297 os.chdir(curdir) 1298 1299 if PYTHON_3: 1300 # Remove the 'Current' link, that way we don't accidentally mess 1301 # with an already installed version of python 2 1302 os.unlink(os.path.join(rootDir, 'Library', 'Frameworks', 1303 'Python.framework', 'Versions', 'Current')) 1304 1305 def patchFile(inPath, outPath): 1306 data = fileContents(inPath) 1307 data = data.replace('$FULL_VERSION', getFullVersion()) 1308 data = data.replace('$VERSION', getVersion()) 1309 data = data.replace('$MACOSX_DEPLOYMENT_TARGET', ''.join((DEPTARGET, ' or later'))) 1310 data = data.replace('$ARCHITECTURES', ", ".join(universal_opts_map[UNIVERSALARCHS])) 1311 data = data.replace('$INSTALL_SIZE', installSize()) 1312 data = data.replace('$THIRD_PARTY_LIBS', "\\\n".join(THIRD_PARTY_LIBS)) 1313 1314 # This one is not handy as a template variable 1315 data = data.replace('$PYTHONFRAMEWORKINSTALLDIR', '/Library/Frameworks/Python.framework') 1316 fp = open(outPath, 'w') 1317 fp.write(data) 1318 fp.close() 1319 1320 def patchScript(inPath, outPath): 1321 major, minor = getVersionMajorMinor() 1322 data = fileContents(inPath) 1323 data = data.replace('@PYMAJOR@', str(major)) 1324 data = data.replace('@PYVER@', getVersion()) 1325 fp = open(outPath, 'w') 1326 fp.write(data) 1327 fp.close() 1328 os.chmod(outPath, STAT_0o755) 1329 1330 1331 1332 def packageFromRecipe(targetDir, recipe): 1333 curdir = os.getcwd() 1334 try: 1335 # The major version (such as 2.5) is included in the package name 1336 # because having two version of python installed at the same time is 1337 # common. 1338 pkgname = '%s-%s'%(recipe['name'], getVersion()) 1339 srcdir = recipe.get('source') 1340 pkgroot = recipe.get('topdir', srcdir) 1341 postflight = recipe.get('postflight') 1342 readme = textwrap.dedent(recipe['readme']) 1343 isRequired = recipe.get('required', True) 1344 1345 print("- building package %s"%(pkgname,)) 1346 1347 # Substitute some variables 1348 textvars = dict( 1349 VER=getVersion(), 1350 FULLVER=getFullVersion(), 1351 ) 1352 readme = readme % textvars 1353 1354 if pkgroot is not None: 1355 pkgroot = pkgroot % textvars 1356 else: 1357 pkgroot = '/' 1358 1359 if srcdir is not None: 1360 srcdir = os.path.join(WORKDIR, '_root', srcdir[1:]) 1361 srcdir = srcdir % textvars 1362 1363 if postflight is not None: 1364 postflight = os.path.abspath(postflight) 1365 1366 packageContents = os.path.join(targetDir, pkgname + '.pkg', 'Contents') 1367 os.makedirs(packageContents) 1368 1369 if srcdir is not None: 1370 os.chdir(srcdir) 1371 runCommand("pax -wf %s . 2>&1"%(shellQuote(os.path.join(packageContents, 'Archive.pax')),)) 1372 runCommand("gzip -9 %s 2>&1"%(shellQuote(os.path.join(packageContents, 'Archive.pax')),)) 1373 runCommand("mkbom . %s 2>&1"%(shellQuote(os.path.join(packageContents, 'Archive.bom')),)) 1374 1375 fn = os.path.join(packageContents, 'PkgInfo') 1376 fp = open(fn, 'w') 1377 fp.write('pmkrpkg1') 1378 fp.close() 1379 1380 rsrcDir = os.path.join(packageContents, "Resources") 1381 os.mkdir(rsrcDir) 1382 fp = open(os.path.join(rsrcDir, 'ReadMe.txt'), 'w') 1383 fp.write(readme) 1384 fp.close() 1385 1386 if postflight is not None: 1387 patchScript(postflight, os.path.join(rsrcDir, 'postflight')) 1388 1389 vers = getFullVersion() 1390 major, minor = getVersionMajorMinor() 1391 pl = Plist( 1392 CFBundleGetInfoString="Python.%s %s"%(pkgname, vers,), 1393 CFBundleIdentifier='org.python.Python.%s'%(pkgname,), 1394 CFBundleName='Python.%s'%(pkgname,), 1395 CFBundleShortVersionString=vers, 1396 IFMajorVersion=major, 1397 IFMinorVersion=minor, 1398 IFPkgFormatVersion=0.10000000149011612, 1399 IFPkgFlagAllowBackRev=False, 1400 IFPkgFlagAuthorizationAction="RootAuthorization", 1401 IFPkgFlagDefaultLocation=pkgroot, 1402 IFPkgFlagFollowLinks=True, 1403 IFPkgFlagInstallFat=True, 1404 IFPkgFlagIsRequired=isRequired, 1405 IFPkgFlagOverwritePermissions=False, 1406 IFPkgFlagRelocatable=False, 1407 IFPkgFlagRestartAction="NoRestart", 1408 IFPkgFlagRootVolumeOnly=True, 1409 IFPkgFlagUpdateInstalledLangauges=False, 1410 ) 1411 writePlist(pl, os.path.join(packageContents, 'Info.plist')) 1412 1413 pl = Plist( 1414 IFPkgDescriptionDescription=readme, 1415 IFPkgDescriptionTitle=recipe.get('long_name', "Python.%s"%(pkgname,)), 1416 IFPkgDescriptionVersion=vers, 1417 ) 1418 writePlist(pl, os.path.join(packageContents, 'Resources', 'Description.plist')) 1419 1420 finally: 1421 os.chdir(curdir) 1422 1423 1424 def makeMpkgPlist(path): 1425 1426 vers = getFullVersion() 1427 major, minor = getVersionMajorMinor() 1428 1429 pl = Plist( 1430 CFBundleGetInfoString="Python %s"%(vers,), 1431 CFBundleIdentifier='org.python.Python', 1432 CFBundleName='Python', 1433 CFBundleShortVersionString=vers, 1434 IFMajorVersion=major, 1435 IFMinorVersion=minor, 1436 IFPkgFlagComponentDirectory="Contents/Packages", 1437 IFPkgFlagPackageList=[ 1438 dict( 1439 IFPkgFlagPackageLocation='%s-%s.pkg'%(item['name'], getVersion()), 1440 IFPkgFlagPackageSelection=item.get('selected', 'selected'), 1441 ) 1442 for item in pkg_recipes() 1443 ], 1444 IFPkgFormatVersion=0.10000000149011612, 1445 IFPkgFlagBackgroundScaling="proportional", 1446 IFPkgFlagBackgroundAlignment="left", 1447 IFPkgFlagAuthorizationAction="RootAuthorization", 1448 ) 1449 1450 writePlist(pl, path) 1451 1452 1453 def buildInstaller(): 1454 1455 # Zap all compiled files 1456 for dirpath, _, filenames in os.walk(os.path.join(WORKDIR, '_root')): 1457 for fn in filenames: 1458 if fn.endswith('.pyc') or fn.endswith('.pyo'): 1459 os.unlink(os.path.join(dirpath, fn)) 1460 1461 outdir = os.path.join(WORKDIR, 'installer') 1462 if os.path.exists(outdir): 1463 shutil.rmtree(outdir) 1464 os.mkdir(outdir) 1465 1466 pkgroot = os.path.join(outdir, 'Python.mpkg', 'Contents') 1467 pkgcontents = os.path.join(pkgroot, 'Packages') 1468 os.makedirs(pkgcontents) 1469 for recipe in pkg_recipes(): 1470 packageFromRecipe(pkgcontents, recipe) 1471 1472 rsrcDir = os.path.join(pkgroot, 'Resources') 1473 1474 fn = os.path.join(pkgroot, 'PkgInfo') 1475 fp = open(fn, 'w') 1476 fp.write('pmkrpkg1') 1477 fp.close() 1478 1479 os.mkdir(rsrcDir) 1480 1481 makeMpkgPlist(os.path.join(pkgroot, 'Info.plist')) 1482 pl = Plist( 1483 IFPkgDescriptionTitle="Python", 1484 IFPkgDescriptionVersion=getVersion(), 1485 ) 1486 1487 writePlist(pl, os.path.join(pkgroot, 'Resources', 'Description.plist')) 1488 for fn in os.listdir('resources'): 1489 if fn == '.svn': continue 1490 if fn.endswith('.jpg'): 1491 shutil.copy(os.path.join('resources', fn), os.path.join(rsrcDir, fn)) 1492 else: 1493 patchFile(os.path.join('resources', fn), os.path.join(rsrcDir, fn)) 1494 1495 1496 def installSize(clear=False, _saved=[]): 1497 if clear: 1498 del _saved[:] 1499 if not _saved: 1500 data = captureCommand("du -ks %s"%( 1501 shellQuote(os.path.join(WORKDIR, '_root')))) 1502 _saved.append("%d"%((0.5 + (int(data.split()[0]) / 1024.0)),)) 1503 return _saved[0] 1504 1505 1506 def buildDMG(): 1507 """ 1508 Create DMG containing the rootDir. 1509 """ 1510 outdir = os.path.join(WORKDIR, 'diskimage') 1511 if os.path.exists(outdir): 1512 shutil.rmtree(outdir) 1513 1514 imagepath = os.path.join(outdir, 1515 'python-%s-macosx%s'%(getFullVersion(),DEPTARGET)) 1516 if INCLUDE_TIMESTAMP: 1517 imagepath = imagepath + '-%04d-%02d-%02d'%(time.localtime()[:3]) 1518 imagepath = imagepath + '.dmg' 1519 1520 os.mkdir(outdir) 1521 1522 # Try to mitigate race condition in certain versions of macOS, e.g. 10.9, 1523 # when hdiutil create fails with "Resource busy". For now, just retry 1524 # the create a few times and hope that it eventually works. 1525 1526 volname='Python %s'%(getFullVersion()) 1527 cmd = ("hdiutil create -format UDRW -volname %s -srcfolder %s -size 100m %s"%( 1528 shellQuote(volname), 1529 shellQuote(os.path.join(WORKDIR, 'installer')), 1530 shellQuote(imagepath + ".tmp.dmg" ))) 1531 for i in range(5): 1532 fd = os.popen(cmd, 'r') 1533 data = fd.read() 1534 xit = fd.close() 1535 if not xit: 1536 break 1537 sys.stdout.write(data) 1538 print(" -- retrying hdiutil create") 1539 time.sleep(5) 1540 else: 1541 raise RuntimeError("command failed: %s"%(commandline,)) 1542 1543 if not os.path.exists(os.path.join(WORKDIR, "mnt")): 1544 os.mkdir(os.path.join(WORKDIR, "mnt")) 1545 runCommand("hdiutil attach %s -mountroot %s"%( 1546 shellQuote(imagepath + ".tmp.dmg"), shellQuote(os.path.join(WORKDIR, "mnt")))) 1547 1548 # Custom icon for the DMG, shown when the DMG is mounted. 1549 shutil.copy("../Icons/Disk Image.icns", 1550 os.path.join(WORKDIR, "mnt", volname, ".VolumeIcon.icns")) 1551 runCommand("SetFile -a C %s/"%( 1552 shellQuote(os.path.join(WORKDIR, "mnt", volname)),)) 1553 1554 runCommand("hdiutil detach %s"%(shellQuote(os.path.join(WORKDIR, "mnt", volname)))) 1555 1556 setIcon(imagepath + ".tmp.dmg", "../Icons/Disk Image.icns") 1557 runCommand("hdiutil convert %s -format UDZO -o %s"%( 1558 shellQuote(imagepath + ".tmp.dmg"), shellQuote(imagepath))) 1559 setIcon(imagepath, "../Icons/Disk Image.icns") 1560 1561 os.unlink(imagepath + ".tmp.dmg") 1562 1563 return imagepath 1564 1565 1566 def setIcon(filePath, icnsPath): 1567 """ 1568 Set the custom icon for the specified file or directory. 1569 """ 1570 1571 dirPath = os.path.normpath(os.path.dirname(__file__)) 1572 toolPath = os.path.join(dirPath, "seticon.app/Contents/MacOS/seticon") 1573 if not os.path.exists(toolPath) or os.stat(toolPath).st_mtime < os.stat(dirPath + '/seticon.m').st_mtime: 1574 # NOTE: The tool is created inside an .app bundle, otherwise it won't work due 1575 # to connections to the window server. 1576 appPath = os.path.join(dirPath, "seticon.app/Contents/MacOS") 1577 if not os.path.exists(appPath): 1578 os.makedirs(appPath) 1579 runCommand("cc -o %s %s/seticon.m -framework Cocoa"%( 1580 shellQuote(toolPath), shellQuote(dirPath))) 1581 1582 runCommand("%s %s %s"%(shellQuote(os.path.abspath(toolPath)), shellQuote(icnsPath), 1583 shellQuote(filePath))) 1584 1585 def main(): 1586 # First parse options and check if we can perform our work 1587 parseOptions() 1588 checkEnvironment() 1589 1590 os.environ['MACOSX_DEPLOYMENT_TARGET'] = DEPTARGET 1591 os.environ['CC'] = CC 1592 os.environ['CXX'] = CXX 1593 1594 if os.path.exists(WORKDIR): 1595 shutil.rmtree(WORKDIR) 1596 os.mkdir(WORKDIR) 1597 1598 os.environ['LC_ALL'] = 'C' 1599 1600 # Then build third-party libraries such as sleepycat DB4. 1601 buildLibraries() 1602 1603 # Now build python itself 1604 buildPython() 1605 1606 # And then build the documentation 1607 # Remove the Deployment Target from the shell 1608 # environment, it's no longer needed and 1609 # an unexpected build target can cause problems 1610 # when Sphinx and its dependencies need to 1611 # be (re-)installed. 1612 del os.environ['MACOSX_DEPLOYMENT_TARGET'] 1613 buildPythonDocs() 1614 1615 1616 # Prepare the applications folder 1617 folder = os.path.join(WORKDIR, "_root", "Applications", "Python %s"%( 1618 getVersion(),)) 1619 fn = os.path.join(folder, "License.rtf") 1620 patchFile("resources/License.rtf", fn) 1621 fn = os.path.join(folder, "ReadMe.rtf") 1622 patchFile("resources/ReadMe.rtf", fn) 1623 fn = os.path.join(folder, "Update Shell Profile.command") 1624 patchScript("scripts/postflight.patch-profile", fn) 1625 fn = os.path.join(folder, "Install Certificates.command") 1626 patchScript("resources/install_certificates.command", fn) 1627 os.chmod(folder, STAT_0o755) 1628 setIcon(folder, "../Icons/Python Folder.icns") 1629 1630 # Create the installer 1631 buildInstaller() 1632 1633 # And copy the readme into the directory containing the installer 1634 patchFile('resources/ReadMe.rtf', 1635 os.path.join(WORKDIR, 'installer', 'ReadMe.rtf')) 1636 1637 # Ditto for the license file. 1638 patchFile('resources/License.rtf', 1639 os.path.join(WORKDIR, 'installer', 'License.rtf')) 1640 1641 fp = open(os.path.join(WORKDIR, 'installer', 'Build.txt'), 'w') 1642 fp.write("# BUILD INFO\n") 1643 fp.write("# Date: %s\n" % time.ctime()) 1644 fp.write("# By: %s\n" % pwd.getpwuid(os.getuid()).pw_gecos) 1645 fp.close() 1646 1647 # And copy it to a DMG 1648 buildDMG() 1649 1650 if __name__ == "__main__": 1651 main() 1652
