1 #!/bin/sh 2 # 3 # Copyright (C) 2012 The Android Open Source Project 4 # Copyright (C) 2012 Ray Donnelly <mingw.android at gmail.com> 5 # 6 # Licensed under the Apache License, Version 2.0 (the "License"); 7 # you may not use this file except in compliance with the License. 8 # You may obtain a copy of the License at 9 # 10 # http://www.apache.org/licenses/LICENSE-2.0 11 # 12 # Unless required by applicable law or agreed to in writing, software 13 # distributed under the License is distributed on an "AS IS" BASIS, 14 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 # See the License for the specific language governing permissions and 16 # limitations under the License. 17 # 18 # Rebuild the host Python binaries from sources. 19 # Also copies any gnu libstdc++ pretty-printers 20 # found in $TOOLCHAIN_SRC_DIR/gcc/gcc-* 21 # and Joachim Reichel's stlport pretty printers 22 # found in sources/host-tools/gdb-pretty-printers/stlport 23 # 24 25 # include common function and variable definitions 26 NDK_BUILDTOOLS_PATH="$(dirname $0)" 27 . "$NDK_BUILDTOOLS_PATH/prebuilt-common.sh" 28 . "$NDK_BUILDTOOLS_PATH/common-build-host-funcs.sh" 29 30 PROGRAM_PARAMETERS="" 31 PROGRAM_DESCRIPTION="\ 32 This program is used to rebuild one or more Python client programs from 33 sources. To use it, you will need a working set of toolchain sources, like 34 those downloaded with download-toolchain-sources.sh, then pass the 35 corresponding directory with the --toolchain-src-dir=<path> option. 36 37 By default, the script rebuilds Python for you host system [$HOST_TAG], 38 but you can use --systems=<tag1>,<tag2>,.. to ask binaries that can run on 39 several distinct systems. Each <tag> value in the list can be one of the 40 following: 41 42 linux-x86 43 linux-x86_64 44 windows 45 windows-x86 (equivalent to 'windows') 46 windows-x86_64 47 darwin-x86 48 darwin-x86_64 49 50 For example, here's how to rebuild Python 2.7.5 on Linux 51 for six different systems: 52 53 $PROGNAME --build-dir=/path/to/toolchain/src \n \ 54 --python-version=2.7.5 \n \ 55 --systems=linux-x86,linux-x86_64,windows,windows-x86_64,darwin-x86,darwin-x86_64" 56 57 TOOLCHAIN_SRC_DIR= 58 register_var_option "--toolchain-src-dir=<path>" TOOLCHAIN_SRC_DIR "Select toolchain source directory" 59 60 PYTHON_VERSION=$DEFAULT_PYTHON_VERSION 61 register_var_option "--python-version=<version>" PYTHON_VERSION "Select Python version." 62 63 NDK_DIR=$ANDROID_NDK_ROOT 64 register_var_option "--ndk-dir=<path>" NDK_DIR "Select NDK install directory." 65 66 PACKAGE_DIR= 67 register_var_option "--package-dir=<path>" PACKAGE_DIR "Package prebuilt tarballs into directory." 68 69 BUILD_DIR= 70 register_var_option "--build-dir=<path>" BUILD_DIR "Build Python into directory" 71 72 bh_register_options 73 register_try64_option 74 register_canadian_option 75 register_jobs_option 76 77 extract_parameters "$@" 78 79 if [ -n "$PARAMETERS" ]; then 80 panic "This script doesn't take parameters, only options. See --help" 81 fi 82 83 if [ -z "$TOOLCHAIN_SRC_DIR" ]; then 84 panic "Please use --toolchain-src-dir=<path> to select toolchain source directory." 85 fi 86 check_toolchain_src_dir "$TOOLCHAIN_SRC_DIR" 87 TOOLCHAIN_SRC_DIR=`cd $TOOLCHAIN_SRC_DIR; pwd` 88 89 BH_HOST_SYSTEMS=$(commas_to_spaces $BH_HOST_SYSTEMS) 90 AUTO_BUILD="no" 91 92 if [ "$MINGW" = "yes" ]; then 93 BH_HOST_SYSTEMS="windows" 94 log "Auto-config: --systems=windows" 95 fi 96 97 if [ "$DARWIN" = "yes" ]; then 98 BH_HOST_SYSTEMS="darwin-x86" 99 log "Auto-config: --systems=darwin-x86" 100 fi 101 102 determine_systems () 103 { 104 local IN_SYSTEMS="$1" 105 local OUT_SYSTEMS 106 107 for SYSTEM in $IN_SYSTEMS; do 108 if [ "$TRY64" = "yes" ]; then 109 case $SYSTEM in 110 darwin-x86|linux-x86|windows-x86) 111 SYSTEM=${SYSTEM%%x86}x86_64 112 ;; 113 windows) 114 SYSTEM=windows-x86_64 115 ;; 116 esac 117 else 118 # 'windows-x86' causes substitution 119 # failure at the packing stage. 120 case $SYSTEM in 121 windows-x86) 122 SYSTEM=windows 123 ;; 124 esac 125 fi 126 OUT_SYSTEMS="$OUT_SYSTEMS $SYSTEM" 127 done 128 echo $OUT_SYSTEMS 129 } 130 131 BH_HOST_SYSTEMS=$(determine_systems "$BH_HOST_SYSTEMS") 132 133 # Build python for build machine automatically 134 if [ "$(bh_list_contains $BH_BUILD_TAG $BH_HOST_SYSTEMS)" = "no" ]; then 135 BH_HOST_SYSTEMS="$BH_BUILD_TAG $BH_HOST_SYSTEMS" 136 AUTO_BUILD="yes" 137 fi 138 139 # Python needs to execute itself during its build process, so must build the build 140 # Python first. It should also be an error if not asked to build for build machine. 141 BH_HOST_SYSTEMS=$(bh_sort_systems_build_first "$BH_HOST_SYSTEMS") 142 143 download_package () 144 { 145 # Assume the packages are already downloaded under $ARCHIVE_DIR 146 local PKG_URL=$1 147 local PKG_NAME=$(basename $PKG_URL) 148 149 case $PKG_NAME in 150 *.tar.bz2) 151 PKG_BASENAME=${PKG_NAME%%.tar.bz2} 152 ;; 153 *.tar.gz) 154 PKG_BASENAME=${PKG_NAME%%.tar.gz} 155 ;; 156 *) 157 panic "Unknown archive type: $PKG_NAME" 158 esac 159 160 if [ ! -f "$ARCHIVE_DIR/$PKG_NAME" ]; then 161 log "Downloading $PKG_URL..." 162 (cd $ARCHIVE_DIR && run curl -L -o "$PKG_NAME" "$PKG_URL") 163 fail_panic "Can't download '$PKG_URL'" 164 fi 165 166 if [ ! -d "$SRC_DIR/$PKG_BASENAME" ]; then 167 log "Uncompressing $PKG_URL into $SRC_DIR" 168 case $PKG_NAME in 169 *.tar.bz2) 170 run tar xjf $ARCHIVE_DIR/$PKG_NAME -C $SRC_DIR 171 ;; 172 *.tar.gz) 173 run tar xzf $ARCHIVE_DIR/$PKG_NAME -C $SRC_DIR 174 ;; 175 *) 176 panic "Unknown archive type: $PKG_NAME" 177 ;; 178 esac 179 fail_panic "Can't uncompress $ARCHIVE_DIR/$PKG_NAME" 180 fi 181 } 182 183 if [ -z "$BUILD_DIR" ] ; then 184 BUILD_DIR=/tmp/ndk-$USER/buildhost 185 fi 186 187 bh_setup_build_dir $BUILD_DIR 188 189 if [ "$BH_BUILD_MODE" = "debug" ] ; then 190 PYDEBUG="-with-pydebug" 191 fi 192 193 # Sanity check that we have the right compilers for all hosts 194 for SYSTEM in $BH_HOST_SYSTEMS; do 195 bh_setup_build_for_host $SYSTEM 196 done 197 198 TEMP_DIR=$BUILD_DIR/tmp 199 # Download and unpack source packages from official sites 200 ARCHIVE_DIR=$TEMP_DIR/archive 201 STAMP_DIR=$TEMP_DIR/timestamps 202 BUILD_DIR=$TEMP_DIR/build-$HOST_TAG 203 204 mkdir -p $BUILD_DIR 205 206 PROGDIR=`dirname $0` 207 PROGDIR=$(cd $PROGDIR && pwd) 208 209 # Sanity check for all Python versions. 210 for VERSION in $(commas_to_spaces $PYTHON_VERSION); do 211 PYTHON_SRCDIR=$TOOLCHAIN_SRC_DIR/python/Python-$VERSION 212 if [ ! -d "$PYTHON_SRCDIR" ]; then 213 panic "Missing source directory: $PYTHON_SRCDIR" 214 fi 215 done 216 217 arch_to_qemu_arch () 218 { 219 case $1 in 220 x86) 221 echo i386 222 ;; 223 *) 224 echo $1 225 ;; 226 esac 227 } 228 229 # $1: host system tag 230 # $2: python version 231 build_host_python () 232 { 233 local SRCDIR=$TOOLCHAIN_SRC_DIR/python/Python-$2 234 local BUILDDIR=$BH_BUILD_DIR/build-python-$1-$2 235 local INSTALLDIR=$(python_build_install_dir $1 $2) 236 local ARGS TEXT 237 238 if [ ! -f "$SRCDIR/configure" ]; then 239 panic "Missing configure script in $SRCDIR" 240 fi 241 242 # Currently, 2.7.5 and 3.3.0 builds generate $SRCDIR/Lib/_sysconfigdata.py, unless it 243 # already exists (in which case it ends up wrong anyway!)... this should really be in 244 # the build directory instead. 245 if [ ! -f "$SRCDIR/Lib/_sysconfigdata.py" ]; then 246 log "Removing old $SRCDIR/Lib/_sysconfigdata.py" 247 rm -f $SRCDIR/Lib/_sysconfigdata.py 248 fi 249 250 ARGS=" --prefix=$INSTALLDIR" 251 252 ARGS=$ARGS" --build=$BH_BUILD_CONFIG" 253 ARGS=$ARGS" --host=$BH_HOST_CONFIG" 254 ARGS=$ARGS" --with-build-sysroot" 255 ARGS=$ARGS" $PYDEBUG" 256 ARGS=$ARGS" --disable-ipv6" 257 258 mkdir -p "$BUILDDIR" && rm -rf "$BUILDDIR"/* 259 cd "$BUILDDIR" && 260 bh_setup_host_env 261 262 CFG_SITE= 263 # Need to add -L$HOST_STATIC_LIBDIR to LDSHARED if need 264 # any static host libs. 265 export LDSHARED="$CC -shared " 266 if [ ! $BH_HOST_TAG = $BH_BUILD_TAG ]; then 267 268 # Cross compiling. 269 CFG_SITE=$BUILDDIR/config.site 270 271 # Ideally would remove all of these configury hacks by 272 # patching the issues. 273 274 if [ $1 = darwin-x86 -o $1 = darwin-x86_64 ]; then 275 echo "ac_cv_file__dev_ptmx=no" > $CFG_SITE 276 echo "ac_cv_file__dev_ptc=no" >> $CFG_SITE 277 echo "ac_cv_have_long_long_format=yes" >> $CFG_SITE 278 if [ $1 = darwin-x86 ] ; then 279 echo "ac_osx_32bit=yes" >> $CFG_SITE 280 elif [ $1 = darwin-x86_64 ] ; then 281 echo "ac_osx_32bit=no" >> $CFG_SITE 282 fi 283 echo "ac_cv_have_sendfile=no" >> $CFG_SITE 284 # I could change AC_MSG_CHECKING(LDSHARED) in configure.ac 285 # to check $host instead of $ac_sys_system/$ac_sys_release 286 # but it handles loads of platforms 287 # and I can only test on three, so instead... 288 export LDSHARED="$CC -bundle -undefined dynamic_lookup" 289 elif [ $1 = windows-x86 -o $1 = windows-x86_64 ]; then 290 echo "ac_cv_file__dev_ptmx=no" > $CFG_SITE 291 echo "ac_cv_file__dev_ptc=no" >> $CFG_SITE 292 CFLAGS=$CFLAGS" -D__USE_MINGW_ANSI_STDIO=1" 293 CXXFLAGS=$CXXFLAGS" -D__USE_MINGW_ANSI_STDIO=1" 294 elif [ $1 = linux-x86 -o $1 = linux-x86_64 ]; then 295 echo "ac_cv_file__dev_ptmx=yes" > $CFG_SITE 296 echo "ac_cv_file__dev_ptc=no" >> $CFG_SITE 297 echo "ac_cv_have_long_long_format=yes" >> $CFG_SITE 298 echo "ac_cv_pthread_system_supported=yes" >> $CFG_SITE 299 echo "ac_cv_working_tzset=yes" >> $CFG_SITE 300 echo "ac_cv_little_endian_double=yes" >> $CFG_SITE 301 fi 302 303 if [ "$BH_HOST_OS" = "$BH_BUILD_OS" ]; then 304 # Only cross compiling from arch perspective. 305 # qemu causes failures as cross-compilation is not detected 306 # if a test executable can be run successfully, so we test 307 # for qemu-${BH_HOST_ARCH} and qemu-${BH_HOST_ARCH}-static 308 # and panic if either are found. 309 QEMU_HOST_ARCH=$(arch_to_qemu_arch $BH_HOST_ARCH) 310 if [ ! -z "$(which qemu-$QEMU_HOST_ARCH 2>/dev/null)" -o \ 311 ! -z "$(which qemu-$QEMU_HOST_ARCH-static 2>/dev/null)" ] ; then 312 panic "Installed qemu(s) ($(which qemu-$QEMU_HOST_ARCH 2>/dev/null) $(which qemu-$QEMU_HOST_ARCH-static 2>/dev/null))" \ 313 "will prevent this build from working." 314 fi 315 fi 316 else 317 if [ $1 = darwin-x86 -o $1 = darwin-x86_64 ]; then 318 export LDSHARED="$CC -bundle -undefined dynamic_lookup" 319 fi 320 fi 321 322 TEXT="$(bh_host_text) python-$BH_HOST_CONFIG-$2:" 323 324 touch $SRCDIR/Include/graminit.h 325 touch $SRCDIR/Python/graminit.c 326 echo "" > $SRCDIR/Parser/pgen.stamp 327 touch $SRCDIR/Parser/Python.asdl 328 touch $SRCDIR/Parser/asdl.py 329 touch $SRCDIR/Parser/asdl_c.py 330 touch $SRCDIR/Include/Python-ast.h 331 touch $SRCDIR/Python/Python-ast.c 332 333 # By default, the Python build will force the following compiler flags 334 # after our own CFLAGS: 335 # -g -fwrap -O3 -Wall -Wstrict-prototypes 336 # 337 # The '-g' is unfortunate because it makes the generated binaries 338 # much larger than necessary, and stripping them after the fact is 339 # a bit delicate when cross-compiling. To avoid this, define a 340 # custom OPT variable here (see Python-2.7.5/configure.ac) when 341 # generating non stripped builds. 342 if [ "$BH_BUILD_MODE" = "release" ]; then 343 OPT="-fwrapv -O3 -Wall -Wstrict-prototypes" 344 export OPT 345 fi 346 347 dump "$TEXT Building" 348 export CONFIG_SITE=$CFG_SITE && 349 run2 "$SRCDIR"/configure $ARGS && 350 # 351 # Note 1: 352 # sharedmods is a phony target, but it's a dependency of both "make all" and also 353 # "make install", this causes it to fail on Windows as it tries to rename pydoc3 354 # to pydoc3.3 twice, and the second time aroud the file exists. So instead, we 355 # just do make install. 356 # 357 # Note 2: 358 # Can't run make install with -j as from the Makefile: 359 # install: altinstall bininstall maninstall 360 # meaning altinstall and bininstall are kicked off at the same time 361 # but actually, bininstall depends on altinstall being run first 362 # due to libainstall: doing 363 # $(INSTALL_SCRIPT) python-config $(DESTDIR)$(BINDIR)/python$(VERSION)-config 364 # and bininstall: doing 365 # (cd $(DESTDIR)$(BINDIR); $(LN) -s python$(VERSION)-config python2-config) 366 # Though the real fix is to make bininstall depend on libainstall. 367 run2 make -j$NUM_JOBS && 368 run2 make install 369 370 # Pretty printers. 371 PYPPDIR="$INSTALLDIR/share/pretty-printers/" 372 373 # .. for gnu stdlibc++ 374 GCC_DIRS=$(find $TOOLCHAIN_SRC_DIR/gcc/ -maxdepth 1 -name "gcc-*" -type d) 375 for GCC_DIR in $GCC_DIRS; do 376 ( 377 if [ -d "$GCC_DIR/libstdc++-v3/python" ]; then 378 cd "$GCC_DIR/libstdc++-v3/python" 379 [ -d "$PYPPDIR/libstdcxx/$(basename $GCC_DIR)" ] || mkdir -p "$PYPPDIR/libstdcxx/$(basename $GCC_DIR)" 380 run2 find . -path "*.py" -exec cp {} "$PYPPDIR/libstdcxx/$(basename $GCC_DIR)/" \; 381 fi 382 ) 383 done 384 385 # .. for STLPort 386 run2 cp -rf $NDK_DIR/sources/host-tools/gdb-pretty-printers/stlport/gppfs-0.2 $PYPPDIR/stlport 387 } 388 389 need_build_host_python () 390 { 391 bh_stamps_do host-python-$1-$2 build_host_python $1 $2 392 } 393 394 # Install host Python binaries and support files to the NDK install dir. 395 # $1: host tag 396 # $2: python version 397 install_host_python () 398 { 399 local SRCDIR="$(python_build_install_dir $1 $2)" 400 local DSTDIR="$NDK_DIR/$(python_ndk_install_dir $1 $2)" 401 402 need_build_host_python $1 $2 403 404 if [ $AUTO_BUILD != "yes" -o $1 != $BH_BUILD_TAG ]; then 405 dump "$(bh_host_text) python-$BH_HOST_ARCH-$2: Installing" 406 run copy_directory "$SRCDIR/bin" "$DSTDIR/bin" 407 run copy_directory "$SRCDIR/lib" "$DSTDIR/lib" 408 run copy_directory "$SRCDIR/share" "$DSTDIR/share" 409 run copy_directory "$SRCDIR/include" "$DSTDIR/include" 410 # remove unneeded files 411 run rm -rf "$DSTDIR/share/man" 412 run rm -rf "$DSTDIR/share/pretty-printers/libstdcxx/gcc-4.9.*" 413 run rm -rf "$DSTDIR/share/pretty-printers/libstdcxx/gcc-4.9-*" 414 run rm -rf "$DSTDIR/share/pretty-printers/libstdcxx/gcc-[lm]*" 415 fi 416 } 417 418 need_install_host_python () 419 { 420 local SRCDIR="$(python_build_install_dir $1 $2)" 421 422 bh_stamps_do install-host-python-$1-$2 install_host_python $1 $2 423 424 # make sharedmods (setup.py) needs to use the build machine's Python 425 # for the other hosts to build correctly. 426 if [ $BH_BUILD_TAG = $BH_HOST_TAG ]; then 427 export PATH=$SRCDIR/bin:$PATH 428 fi 429 } 430 431 # Package host Python binaries into a tarball 432 # $1: host tag 433 # $2: python version 434 package_host_python () 435 { 436 local BLDDIR="$(python_build_install_dir $1 $2)" 437 local SRCDIR="$(python_ndk_install_dir $1 $2)" 438 # This is similar to BLDDIR=${BLDDIR%%$SRCDIR} (and requires we use windows and not windows-x86) 439 BLDDIR=$(echo "$BLDDIR" | sed "s/$(echo "$SRCDIR" | sed -e 's/\\/\\\\/g' -e 's/\//\\\//g' -e 's/&/\\\&/g')//g") 440 local PACKAGENAME=ndk-python-$(install_dir_from_host_tag $1).tar.bz2 441 local PACKAGE="$PACKAGE_DIR/$PACKAGENAME" 442 443 need_install_host_python $1 $2 444 445 dump "$(bh_host_text) $PACKAGENAME: Packaging" 446 run pack_archive "$PACKAGE" "$BLDDIR" "$SRCDIR" 447 } 448 449 need_package_host_python () 450 { 451 bh_stamps_do package-host-python-$1-$2 package_host_python $1 $2 452 } 453 454 PYTHON_VERSION=$(commas_to_spaces $PYTHON_VERSION) 455 ARCHS=$(commas_to_spaces $ARCHS) 456 457 # Let's build this 458 for SYSTEM in $BH_HOST_SYSTEMS; do 459 bh_setup_build_for_host $SYSTEM 460 for VERSION in $PYTHON_VERSION; do 461 need_install_host_python $SYSTEM $VERSION 462 done 463 done 464 465 if [ "$PACKAGE_DIR" ]; then 466 for SYSTEM in $BH_HOST_SYSTEMS; do 467 bh_setup_build_for_host $SYSTEM 468 if [ $AUTO_BUILD != "yes" -o $SYSTEM != $BH_BUILD_TAG ]; then 469 for VERSION in $PYTHON_VERSION; do 470 need_package_host_python $SYSTEM $VERSION 471 done 472 fi 473 done 474 fi 475
