Home | History | Annotate | Download | only in sysroot_scripts 
      1 # Copyright 2014 The Chromium Authors. All rights reserved.
      2 # Use of this source code is governed by a BSD-style license that can be
      3 # found in the LICENSE file.
      4 #
      5 # This script should not be run directly but sourced by the other
      6 # scripts (e.g. sysroot-creator-trusty.sh).  Its up to the parent scripts
      7 # to define certain environment variables: e.g.
      8 #  DIST=trusty
      9 #  APT_REPO=http://archive.ubuntu.com/ubuntu
     10 #  KEYRING_FILE=/usr/share/keyrings/ubuntu-archive-keyring.gpg
     11 #  DEBIAN_PACKAGES="gcc libz libssl"
     12 
     13 #@ This script builds a Debian sysroot images for building Google Chrome.
     14 #@
     15 #@  Generally this script is invoked as:
     16 #@  sysroot-creator-<flavour>.sh <mode> <args>*
     17 #@  Available modes are shown below.
     18 #@
     19 #@ List of modes:
     20 
     21 ######################################################################
     22 # Config
     23 ######################################################################
     24 
     25 set -o nounset
     26 set -o errexit
     27 
     28 SCRIPT_DIR=$(cd $(dirname $0) && pwd)
     29 
     30 if [ -z "${DIST:-}" ]; then
     31   echo "error: DIST not defined"
     32   exit 1
     33 fi
     34 
     35 if [ -z "${APT_REPO:-}" ]; then
     36   echo "error: APT_REPO not defined"
     37   exit 1
     38 fi
     39 
     40 if [ -z "${KEYRING_FILE:-}" ]; then
     41   echo "error: KEYRING_FILE not defined"
     42   exit 1
     43 fi
     44 
     45 if [ -z "${DEBIAN_PACKAGES:-}" ]; then
     46   echo "error: DEBIAN_PACKAGES not defined"
     47   exit 1
     48 fi
     49 
     50 readonly REPO_BASEDIR="${APT_REPO}/dists/${DIST}"
     51 
     52 # This is where the staging sysroot is.
     53 readonly INSTALL_ROOT_AMD64=$(pwd)/${DIST}_amd64_staging
     54 readonly INSTALL_ROOT_I386=$(pwd)/${DIST}_i386_staging
     55 readonly INSTALL_ROOT_ARM=$(pwd)/${DIST}_arm_staging
     56 
     57 readonly REQUIRED_TOOLS="wget"
     58 
     59 ######################################################################
     60 # Package Config
     61 ######################################################################
     62 
     63 readonly RELEASE_FILE="Release"
     64 readonly RELEASE_FILE_GPG="Release.gpg"
     65 readonly RELEASE_LIST="${REPO_BASEDIR}/${RELEASE_FILE}"
     66 readonly RELEASE_LIST_GPG="${REPO_BASEDIR}/${RELEASE_FILE_GPG}"
     67 readonly PACKAGE_FILE_AMD64="main/binary-amd64/Packages.bz2"
     68 readonly PACKAGE_FILE_I386="main/binary-i386/Packages.bz2"
     69 readonly PACKAGE_FILE_ARM="main/binary-armhf/Packages.bz2"
     70 readonly PACKAGE_LIST_AMD64="${REPO_BASEDIR}/${PACKAGE_FILE_AMD64}"
     71 readonly PACKAGE_LIST_I386="${REPO_BASEDIR}/${PACKAGE_FILE_I386}"
     72 readonly PACKAGE_LIST_ARM="${REPO_BASEDIR}/${PACKAGE_FILE_ARM}"
     73 
     74 readonly DEBIAN_DEP_LIST_AMD64="packagelist.${DIST}.amd64"
     75 readonly DEBIAN_DEP_LIST_I386="packagelist.${DIST}.i386"
     76 readonly DEBIAN_DEP_LIST_ARM="packagelist.${DIST}.arm"
     77 
     78 ######################################################################
     79 # Helper
     80 ######################################################################
     81 
     82 Banner() {
     83   echo "######################################################################"
     84   echo $*
     85   echo "######################################################################"
     86 }
     87 
     88 
     89 SubBanner() {
     90   echo "----------------------------------------------------------------------"
     91   echo $*
     92   echo "----------------------------------------------------------------------"
     93 }
     94 
     95 
     96 Usage() {
     97   egrep "^#@" "${BASH_SOURCE[0]}" | cut --bytes=3-
     98 }
     99 
    100 
    101 DownloadOrCopy() {
    102   if [ -f "$2" ] ; then
    103     echo "$2 already in place"
    104     return
    105   fi
    106 
    107   HTTP=0
    108   echo "$1" | grep -qs ^http:// && HTTP=1
    109   if [ "$HTTP" = "1" ]; then
    110     SubBanner "downloading from $1 -> $2"
    111     wget "$1" -O "${2}.partial"
    112     mv "${2}.partial" $2
    113   else
    114     SubBanner "copying from $1"
    115     cp "$1" "$2"
    116   fi
    117 }
    118 
    119 
    120 SetEnvironmentVariables() {
    121   ARCH=""
    122   echo $1 | grep -qs Amd64$ && ARCH=AMD64
    123   if [ -z "$ARCH" ]; then
    124     echo $1 | grep -qs I386$ && ARCH=I386
    125   fi
    126   if [ -z "$ARCH" ]; then
    127     echo $1 | grep -qs ARM$ && ARCH=ARM
    128   fi
    129   case "$ARCH" in
    130     ARM)
    131       INSTALL_ROOT="$INSTALL_ROOT_ARM";
    132       ;;
    133     AMD64)
    134       INSTALL_ROOT="$INSTALL_ROOT_AMD64";
    135       ;;
    136     I386)
    137       INSTALL_ROOT="$INSTALL_ROOT_I386";
    138       ;;
    139     *)
    140       echo "ERROR: Unexpected bad architecture."
    141       exit 1
    142       ;;
    143   esac
    144 }
    145 
    146 
    147 # some sanity checks to make sure this script is run from the right place
    148 # with the right tools
    149 SanityCheck() {
    150   Banner "Sanity Checks"
    151 
    152   if ! mkdir -p "${INSTALL_ROOT}" ; then
    153      echo "ERROR: ${INSTALL_ROOT} can't be created."
    154     exit 1
    155   fi
    156 
    157   CHROME_DIR=$(cd "${SCRIPT_DIR}/../../.." && pwd)
    158   BUILD_DIR=${CHROME_DIR}/out/sysroot-build/${DIST}
    159   mkdir -p ${BUILD_DIR}
    160   echo "Using build directory: ${BUILD_DIR}"
    161 
    162   for tool in ${REQUIRED_TOOLS} ; do
    163     if ! which ${tool} > /dev/null ; then
    164       echo "Required binary $tool not found."
    165       echo "Exiting."
    166       exit 1
    167     fi
    168   done
    169 }
    170 
    171 
    172 ChangeDirectory() {
    173   # Change directory to where this script is.
    174   cd ${SCRIPT_DIR}
    175 }
    176 
    177 
    178 ClearInstallDir() {
    179   Banner "Clearing dirs in ${INSTALL_ROOT}"
    180   rm -rf ${INSTALL_ROOT}/*
    181 }
    182 
    183 
    184 CreateTarBall() {
    185   local tarball=$1
    186   Banner "Creating tar ball ${tarball}"
    187   tar zcf ${tarball} -C ${INSTALL_ROOT} .
    188 }
    189 
    190 CheckBuildSysrootArgs() {
    191   if [ "$#" -ne "1" ]; then
    192     echo "ERROR: BuildSysroot commands only take 1 argument"
    193     exit 1
    194   fi
    195 
    196   if [ -z "$1" ]; then
    197     echo "ERROR: tarball name required"
    198     exit 1
    199   fi
    200 }
    201 
    202 ExtractPackageBz2() {
    203   bzcat "$1" | egrep '^(Package:|Filename:|SHA256:) ' > "$2"
    204 }
    205 
    206 GeneratePackageListAmd64() {
    207   local output_file="$1"
    208   local package_list="${BUILD_DIR}/Packages.${DIST}_amd64.bz2"
    209   local tmp_package_list="${BUILD_DIR}/Packages.${DIST}_amd64"
    210   DownloadOrCopy "${PACKAGE_LIST_AMD64}" "${package_list}"
    211   VerifyPackageListing "${PACKAGE_FILE_AMD64}" "${package_list}"
    212   ExtractPackageBz2 "$package_list" "$tmp_package_list"
    213   GeneratePackageList "$tmp_package_list" "$output_file" "${DEBIAN_PACKAGES}
    214     ${DEBIAN_PACKAGES_X86}"
    215 }
    216 
    217 GeneratePackageListI386() {
    218   local output_file="$1"
    219   local package_list="${BUILD_DIR}/Packages.${DIST}_i386.bz2"
    220   local tmp_package_list="${BUILD_DIR}/Packages.${DIST}_amd64"
    221   DownloadOrCopy "${PACKAGE_LIST_I386}" "${package_list}"
    222   VerifyPackageListing "${PACKAGE_FILE_I386}" "${package_list}"
    223   ExtractPackageBz2 "$package_list" "$tmp_package_list"
    224   GeneratePackageList "$tmp_package_list" "$output_file" "${DEBIAN_PACKAGES}
    225     ${DEBIAN_PACKAGES_X86}"
    226 }
    227 
    228 GeneratePackageListARM() {
    229   local output_file="$1"
    230   local package_list="${BUILD_DIR}/Packages.${DIST}_arm.bz2"
    231   local tmp_package_list="${BUILD_DIR}/Packages.${DIST}_arm"
    232   DownloadOrCopy "${PACKAGE_LIST_ARM}" "${package_list}"
    233   VerifyPackageListing "${PACKAGE_FILE_ARM}" "${package_list}"
    234   ExtractPackageBz2 "$package_list" "$tmp_package_list"
    235   GeneratePackageList "$tmp_package_list" "$output_file" "${DEBIAN_PACKAGES}"
    236 }
    237 
    238 StripChecksumsFromPackageList() {
    239   local package_file="$1"
    240   sed -i 's/ [a-f0-9]\{64\}$//' "$package_file"
    241 }
    242 
    243 VerifyPackageFilesMatch() {
    244   local downloaded_package_file="$1"
    245   local stored_package_file="$2"
    246   diff -u "$downloaded_package_file" "$stored_package_file"
    247   if [ "$?" -ne "0" ]; then
    248     echo "ERROR: downloaded package files does not match $2."
    249     echo "You may need to run UpdatePackageLists."
    250     exit 1
    251   fi
    252 }
    253 
    254 ######################################################################
    255 #
    256 ######################################################################
    257 
    258 HacksAndPatchesAmd64() {
    259   Banner "Misc Hacks & Patches"
    260   # these are linker scripts with absolute pathnames in them
    261   # which we rewrite here
    262   lscripts="${INSTALL_ROOT}/usr/lib/x86_64-linux-gnu/libpthread.so \
    263             ${INSTALL_ROOT}/usr/lib/x86_64-linux-gnu/libc.so"
    264 
    265   #SubBanner "Rewriting Linker Scripts"
    266   sed -i -e 's|/usr/lib/x86_64-linux-gnu/||g'  ${lscripts}
    267   sed -i -e 's|/lib/x86_64-linux-gnu/||g' ${lscripts}
    268 
    269   # This is for chrome's ./build/linux/pkg-config-wrapper
    270   # which overwrites PKG_CONFIG_PATH internally
    271   SubBanner "Package Configs Symlink"
    272   mkdir -p ${INSTALL_ROOT}/usr/share
    273   ln -s ../lib/x86_64-linux-gnu/pkgconfig ${INSTALL_ROOT}/usr/share/pkgconfig
    274 
    275   SubBanner "Adding an additional ld.conf include"
    276   LD_SO_HACK_CONF="${INSTALL_ROOT}/etc/ld.so.conf.d/zz_hack.conf"
    277   echo /usr/lib/gcc/x86_64-linux-gnu/4.6 > "$LD_SO_HACK_CONF"
    278   echo /usr/lib >> "$LD_SO_HACK_CONF"
    279 }
    280 
    281 
    282 HacksAndPatchesI386() {
    283   Banner "Misc Hacks & Patches"
    284   # these are linker scripts with absolute pathnames in them
    285   # which we rewrite here
    286   lscripts="${INSTALL_ROOT}/usr/lib/i386-linux-gnu/libpthread.so \
    287             ${INSTALL_ROOT}/usr/lib/i386-linux-gnu/libc.so"
    288 
    289   #SubBanner "Rewriting Linker Scripts"
    290   sed -i -e 's|/usr/lib/i386-linux-gnu/||g'  ${lscripts}
    291   sed -i -e 's|/lib/i386-linux-gnu/||g' ${lscripts}
    292 
    293   # This is for chrome's ./build/linux/pkg-config-wrapper
    294   # which overwrites PKG_CONFIG_PATH internally
    295   SubBanner "Package Configs Symlink"
    296   mkdir -p ${INSTALL_ROOT}/usr/share
    297   ln -s ../lib/i386-linux-gnu/pkgconfig ${INSTALL_ROOT}/usr/share/pkgconfig
    298 
    299   SubBanner "Adding an additional ld.conf include"
    300   LD_SO_HACK_CONF="${INSTALL_ROOT}/etc/ld.so.conf.d/zz_hack.conf"
    301   echo /usr/lib/gcc/i486-linux-gnu/4.6 > "$LD_SO_HACK_CONF"
    302   echo /usr/lib >> "$LD_SO_HACK_CONF"
    303 }
    304 
    305 
    306 HacksAndPatchesARM() {
    307   Banner "Misc Hacks & Patches"
    308   # these are linker scripts with absolute pathnames in them
    309   # which we rewrite here
    310   lscripts="${INSTALL_ROOT}/usr/lib/arm-linux-gnueabihf/libpthread.so \
    311             ${INSTALL_ROOT}/usr/lib/arm-linux-gnueabihf/libc.so"
    312 
    313   #SubBanner "Rewriting Linker Scripts"
    314   sed -i -e 's|/usr/lib/arm-linux-gnueabihf/||g' ${lscripts}
    315   sed -i -e 's|/lib/arm-linux-gnueabihf/||g' ${lscripts}
    316 
    317   # This is for chrome's ./build/linux/pkg-config-wrapper
    318   # which overwrites PKG_CONFIG_PATH internally
    319   SubBanner "Package Configs Symlink"
    320   mkdir -p ${INSTALL_ROOT}/usr/share
    321   ln -s ../lib/arm-linux-gnueabihf/pkgconfig ${INSTALL_ROOT}/usr/share/pkgconfig
    322 }
    323 
    324 
    325 InstallIntoSysroot() {
    326   Banner "Install Libs And Headers Into Jail"
    327 
    328   mkdir -p ${BUILD_DIR}/debian-packages
    329   mkdir -p ${INSTALL_ROOT}
    330   while (( "$#" )); do
    331     local file="$1"
    332     local package="${BUILD_DIR}/debian-packages/${file##*/}"
    333     shift
    334     local sha256sum="$1"
    335     shift
    336     if [ "${#sha256sum}" -ne "64" ]; then
    337       echo "Bad sha256sum from package list"
    338       exit 1
    339     fi
    340 
    341     Banner "Installing ${file}"
    342     DownloadOrCopy ${APT_REPO}/pool/${file} ${package}
    343     if [ ! -s "${package}" ] ; then
    344       echo
    345       echo "ERROR: bad package ${package}"
    346       exit 1
    347     fi
    348     echo "${sha256sum}  ${package}" | sha256sum --quiet -c
    349 
    350     SubBanner "Extracting to ${INSTALL_ROOT}"
    351     dpkg --fsys-tarfile ${package}\
    352       | tar -xf - --exclude=./usr/share -C ${INSTALL_ROOT}
    353   done
    354 }
    355 
    356 
    357 CleanupJailSymlinks() {
    358   Banner "Jail symlink cleanup"
    359 
    360   SAVEDPWD=$(pwd)
    361   cd ${INSTALL_ROOT}
    362   find lib lib64 usr/lib -type l -printf '%p %l\n' | while read link target; do
    363     # skip links with non-absolute paths
    364     echo "${target}" | grep -qs ^/ || continue
    365     echo "${link}: ${target}"
    366     case "${link}" in
    367       usr/lib/gcc/x86_64-linux-gnu/4.*/* | usr/lib/gcc/i486-linux-gnu/4.*/* | \
    368       usr/lib/gcc/arm-linux-gnueabihf/4.*/*)
    369         # Relativize the symlink.
    370         ln -snfv "../../../../..${target}" "${link}"
    371         ;;
    372       usr/lib/x86_64-linux-gnu/* | usr/lib/i386-linux-gnu/* | \
    373       usr/lib/arm-linux-gnueabihf/*)
    374         # Relativize the symlink.
    375         ln -snfv "../../..${target}" "${link}"
    376         ;;
    377       usr/lib/*)
    378         # Relativize the symlink.
    379         ln -snfv "../..${target}" "${link}"
    380         ;;
    381       lib64/* | lib/*)
    382         # Relativize the symlink.
    383         ln -snfv "..${target}" "${link}"
    384         ;;
    385     esac
    386   done
    387 
    388   find lib lib64 usr/lib -type l -printf '%p %l\n' | while read link target; do
    389     # Make sure we catch new bad links.
    390     if [ ! -r "${link}" ]; then
    391       echo "ERROR: FOUND BAD LINK ${link}"
    392       ls -l ${link}
    393       exit 1
    394     fi
    395   done
    396   cd "$SAVEDPWD"
    397 }
    398 
    399 #@
    400 #@ BuildSysrootAmd64 <tarball-name>
    401 #@
    402 #@    Build everything and package it
    403 BuildSysrootAmd64() {
    404   CheckBuildSysrootArgs $@
    405   ClearInstallDir
    406   local package_file="$BUILD_DIR/package_with_sha256sum_amd64"
    407   GeneratePackageListAmd64 "$package_file"
    408   local files_and_sha256sums="$(cat ${package_file})"
    409   StripChecksumsFromPackageList "$package_file"
    410   VerifyPackageFilesMatch "$package_file" "$DEBIAN_DEP_LIST_AMD64"
    411   InstallIntoSysroot ${files_and_sha256sums}
    412   CleanupJailSymlinks
    413   HacksAndPatchesAmd64
    414   CreateTarBall "$1"
    415 }
    416 
    417 #@
    418 #@ BuildSysrootI386 <tarball-name>
    419 #@
    420 #@    Build everything and package it
    421 BuildSysrootI386() {
    422   CheckBuildSysrootArgs $@
    423   ClearInstallDir
    424   local package_file="$BUILD_DIR/package_with_sha256sum_i386"
    425   GeneratePackageListI386 "$package_file"
    426   local files_and_sha256sums="$(cat ${package_file})"
    427   StripChecksumsFromPackageList "$package_file"
    428   VerifyPackageFilesMatch "$package_file" "$DEBIAN_DEP_LIST_I386"
    429   InstallIntoSysroot ${files_and_sha256sums}
    430   CleanupJailSymlinks
    431   HacksAndPatchesI386
    432   CreateTarBall "$1"
    433 }
    434 
    435 #@
    436 #@ BuildSysrootARM <tarball-name>
    437 #@
    438 #@    Build everything and package it
    439 BuildSysrootARM() {
    440   CheckBuildSysrootArgs $@
    441   ClearInstallDir
    442   local package_file="$BUILD_DIR/package_with_sha256sum_arm"
    443   GeneratePackageListARM "$package_file"
    444   local files_and_sha256sums="$(cat ${package_file})"
    445   StripChecksumsFromPackageList "$package_file"
    446   VerifyPackageFilesMatch "$package_file" "$DEBIAN_DEP_LIST_ARM"
    447   APT_REPO=${APR_REPO_ARM:=$APT_REPO}
    448   InstallIntoSysroot ${files_and_sha256sums}
    449   CleanupJailSymlinks
    450   HacksAndPatchesARM
    451   CreateTarBall "$1"
    452 }
    453 
    454 #
    455 # CheckForDebianGPGKeyring
    456 #
    457 #     Make sure the Debian GPG keys exist. Otherwise print a helpful message.
    458 #
    459 CheckForDebianGPGKeyring() {
    460   if [ ! -e "$KEYRING_FILE" ]; then
    461     echo "Debian GPG keys missing. Install the debian-archive-keyring package."
    462     exit 1
    463   fi
    464 }
    465 
    466 #
    467 # VerifyPackageListing
    468 #
    469 #     Verifies the downloaded Packages.bz2 file has the right checksums.
    470 #
    471 VerifyPackageListing() {
    472   local file_path=$1
    473   local output_file=$2
    474   local release_file="${BUILD_DIR}/${RELEASE_FILE}"
    475   local release_file_gpg="${BUILD_DIR}/${RELEASE_FILE_GPG}"
    476 
    477   CheckForDebianGPGKeyring
    478 
    479   DownloadOrCopy ${RELEASE_LIST} ${release_file}
    480   DownloadOrCopy ${RELEASE_LIST_GPG} ${release_file_gpg}
    481   echo "Verifying: ${release_file} with ${release_file_gpg}"
    482   gpgv --keyring $KEYRING_FILE ${release_file_gpg} ${release_file}
    483 
    484   echo "Verifying: ${output_file}"
    485   local checksums=$(grep ${file_path} ${release_file} | cut -d " " -f 2)
    486   local sha256sum=$(echo ${checksums} | cut -d " " -f 3)
    487 
    488   if [ "${#sha256sum}" -ne "64" ]; then
    489     echo "Bad sha256sum from ${RELEASE_LIST}"
    490     exit 1
    491   fi
    492 
    493   echo "${sha256sum}  ${output_file}" | sha256sum --quiet -c
    494 }
    495 
    496 #
    497 # GeneratePackageList
    498 #
    499 #     Looks up package names in ${BUILD_DIR}/Packages and write list of URLs
    500 #     to output file.
    501 #
    502 GeneratePackageList() {
    503   local input_file="$1"
    504   local output_file="$2"
    505   echo "Updating: ${output_file} from ${input_file}"
    506   /bin/rm -f "${output_file}"
    507   shift
    508   shift
    509   for pkg in $@ ; do
    510     local pkg_full=$(grep -A 1 " ${pkg}\$" "$input_file" | \
    511       egrep -o "pool/.*")
    512     if [ -z "${pkg_full}" ]; then
    513         echo "ERROR: missing package: $pkg"
    514         exit 1
    515     fi
    516     local pkg_nopool=$(echo "$pkg_full" | sed "s/^pool\///")
    517     local sha256sum=$(grep -A 4 " ${pkg}\$" "$input_file" | \
    518       grep ^SHA256: | sed 's/^SHA256: //')
    519     if [ "${#sha256sum}" -ne "64" ]; then
    520       echo "Bad sha256sum from Packages"
    521       exit 1
    522     fi
    523     echo $pkg_nopool $sha256sum >> "$output_file"
    524   done
    525   # sort -o does an in-place sort of this file
    526   sort "$output_file" -o "$output_file"
    527 }
    528 
    529 #@
    530 #@ UpdatePackageListsAmd64
    531 #@
    532 #@     Regenerate the package lists such that they contain an up-to-date
    533 #@     list of URLs within the Debian archive. (For amd64)
    534 UpdatePackageListsAmd64() {
    535   GeneratePackageListAmd64 "$DEBIAN_DEP_LIST_AMD64"
    536   StripChecksumsFromPackageList "$DEBIAN_DEP_LIST_AMD64"
    537 }
    538 
    539 #@
    540 #@ UpdatePackageListsI386
    541 #@
    542 #@     Regenerate the package lists such that they contain an up-to-date
    543 #@     list of URLs within the Debian archive. (For i386)
    544 UpdatePackageListsI386() {
    545   GeneratePackageListI386 "$DEBIAN_DEP_LIST_I386"
    546   StripChecksumsFromPackageList "$DEBIAN_DEP_LIST_I386"
    547 }
    548 
    549 #@
    550 #@ UpdatePackageListsARM
    551 #@
    552 #@     Regenerate the package lists such that they contain an up-to-date
    553 #@     list of URLs within the Debian archive. (For arm)
    554 UpdatePackageListsARM() {
    555   GeneratePackageListARM "$DEBIAN_DEP_LIST_ARM"
    556   StripChecksumsFromPackageList "$DEBIAN_DEP_LIST_ARM"
    557 }
    558 
    559 if [ $# -eq 0 ] ; then
    560   echo "ERROR: you must specify a mode on the commandline"
    561   echo
    562   Usage
    563   exit 1
    564 elif [ "$(type -t $1)" != "function" ]; then
    565   echo "ERROR: unknown function '$1'." >&2
    566   echo "For help, try:"
    567   echo "    $0 help"
    568   exit 1
    569 else
    570   ChangeDirectory
    571   SetEnvironmentVariables "$1"
    572   SanityCheck
    573   "$@"
    574 fi
    575