1 # Copyright 2014 The Chromium Authors. All rights reserved. 2 # Use of this source code is governed by a BSD-style license that can be 3 # found in the LICENSE file. 4 # 5 # This script should not be run directly but sourced by the other 6 # scripts (e.g. sysroot-creator-trusty.sh). Its up to the parent scripts 7 # to define certain environment variables: e.g. 8 # DIST=trusty 9 # APT_REPO=http://archive.ubuntu.com/ubuntu 10 # KEYRING_FILE=/usr/share/keyrings/ubuntu-archive-keyring.gpg 11 # DEBIAN_PACKAGES="gcc libz libssl" 12 13 #@ This script builds a Debian sysroot images for building Google Chrome. 14 #@ 15 #@ Generally this script is invoked as: 16 #@ sysroot-creator-<flavour>.sh <mode> <args>* 17 #@ Available modes are shown below. 18 #@ 19 #@ List of modes: 20 21 ###################################################################### 22 # Config 23 ###################################################################### 24 25 set -o nounset 26 set -o errexit 27 28 SCRIPT_DIR=$(cd $(dirname $0) && pwd) 29 30 if [ -z "${DIST:-}" ]; then 31 echo "error: DIST not defined" 32 exit 1 33 fi 34 35 if [ -z "${APT_REPO:-}" ]; then 36 echo "error: APT_REPO not defined" 37 exit 1 38 fi 39 40 if [ -z "${KEYRING_FILE:-}" ]; then 41 echo "error: KEYRING_FILE not defined" 42 exit 1 43 fi 44 45 if [ -z "${DEBIAN_PACKAGES:-}" ]; then 46 echo "error: DEBIAN_PACKAGES not defined" 47 exit 1 48 fi 49 50 readonly REPO_BASEDIR="${APT_REPO}/dists/${DIST}" 51 52 # This is where the staging sysroot is. 53 readonly INSTALL_ROOT_AMD64=$(pwd)/${DIST}_amd64_staging 54 readonly INSTALL_ROOT_I386=$(pwd)/${DIST}_i386_staging 55 readonly INSTALL_ROOT_ARM=$(pwd)/${DIST}_arm_staging 56 57 readonly REQUIRED_TOOLS="wget" 58 59 ###################################################################### 60 # Package Config 61 ###################################################################### 62 63 readonly RELEASE_FILE="Release" 64 readonly RELEASE_FILE_GPG="Release.gpg" 65 readonly RELEASE_LIST="${REPO_BASEDIR}/${RELEASE_FILE}" 66 readonly RELEASE_LIST_GPG="${REPO_BASEDIR}/${RELEASE_FILE_GPG}" 67 readonly PACKAGE_FILE_AMD64="main/binary-amd64/Packages.bz2" 68 readonly PACKAGE_FILE_I386="main/binary-i386/Packages.bz2" 69 readonly PACKAGE_FILE_ARM="main/binary-armhf/Packages.bz2" 70 readonly PACKAGE_LIST_AMD64="${REPO_BASEDIR}/${PACKAGE_FILE_AMD64}" 71 readonly PACKAGE_LIST_I386="${REPO_BASEDIR}/${PACKAGE_FILE_I386}" 72 readonly PACKAGE_LIST_ARM="${REPO_BASEDIR}/${PACKAGE_FILE_ARM}" 73 74 readonly DEBIAN_DEP_LIST_AMD64="packagelist.${DIST}.amd64" 75 readonly DEBIAN_DEP_LIST_I386="packagelist.${DIST}.i386" 76 readonly DEBIAN_DEP_LIST_ARM="packagelist.${DIST}.arm" 77 78 ###################################################################### 79 # Helper 80 ###################################################################### 81 82 Banner() { 83 echo "######################################################################" 84 echo $* 85 echo "######################################################################" 86 } 87 88 89 SubBanner() { 90 echo "----------------------------------------------------------------------" 91 echo $* 92 echo "----------------------------------------------------------------------" 93 } 94 95 96 Usage() { 97 egrep "^#@" "${BASH_SOURCE[0]}" | cut --bytes=3- 98 } 99 100 101 DownloadOrCopy() { 102 if [ -f "$2" ] ; then 103 echo "$2 already in place" 104 return 105 fi 106 107 HTTP=0 108 echo "$1" | grep -qs ^http:// && HTTP=1 109 if [ "$HTTP" = "1" ]; then 110 SubBanner "downloading from $1 -> $2" 111 wget "$1" -O "${2}.partial" 112 mv "${2}.partial" $2 113 else 114 SubBanner "copying from $1" 115 cp "$1" "$2" 116 fi 117 } 118 119 120 SetEnvironmentVariables() { 121 ARCH="" 122 echo $1 | grep -qs Amd64$ && ARCH=AMD64 123 if [ -z "$ARCH" ]; then 124 echo $1 | grep -qs I386$ && ARCH=I386 125 fi 126 if [ -z "$ARCH" ]; then 127 echo $1 | grep -qs ARM$ && ARCH=ARM 128 fi 129 case "$ARCH" in 130 ARM) 131 INSTALL_ROOT="$INSTALL_ROOT_ARM"; 132 ;; 133 AMD64) 134 INSTALL_ROOT="$INSTALL_ROOT_AMD64"; 135 ;; 136 I386) 137 INSTALL_ROOT="$INSTALL_ROOT_I386"; 138 ;; 139 *) 140 echo "ERROR: Unexpected bad architecture." 141 exit 1 142 ;; 143 esac 144 } 145 146 147 # some sanity checks to make sure this script is run from the right place 148 # with the right tools 149 SanityCheck() { 150 Banner "Sanity Checks" 151 152 if ! mkdir -p "${INSTALL_ROOT}" ; then 153 echo "ERROR: ${INSTALL_ROOT} can't be created." 154 exit 1 155 fi 156 157 CHROME_DIR=$(cd "${SCRIPT_DIR}/../../.." && pwd) 158 BUILD_DIR=${CHROME_DIR}/out/sysroot-build/${DIST} 159 mkdir -p ${BUILD_DIR} 160 echo "Using build directory: ${BUILD_DIR}" 161 162 for tool in ${REQUIRED_TOOLS} ; do 163 if ! which ${tool} > /dev/null ; then 164 echo "Required binary $tool not found." 165 echo "Exiting." 166 exit 1 167 fi 168 done 169 } 170 171 172 ChangeDirectory() { 173 # Change directory to where this script is. 174 cd ${SCRIPT_DIR} 175 } 176 177 178 ClearInstallDir() { 179 Banner "Clearing dirs in ${INSTALL_ROOT}" 180 rm -rf ${INSTALL_ROOT}/* 181 } 182 183 184 CreateTarBall() { 185 local tarball=$1 186 Banner "Creating tar ball ${tarball}" 187 tar zcf ${tarball} -C ${INSTALL_ROOT} . 188 } 189 190 CheckBuildSysrootArgs() { 191 if [ "$#" -ne "1" ]; then 192 echo "ERROR: BuildSysroot commands only take 1 argument" 193 exit 1 194 fi 195 196 if [ -z "$1" ]; then 197 echo "ERROR: tarball name required" 198 exit 1 199 fi 200 } 201 202 ExtractPackageBz2() { 203 bzcat "$1" | egrep '^(Package:|Filename:|SHA256:) ' > "$2" 204 } 205 206 GeneratePackageListAmd64() { 207 local output_file="$1" 208 local package_list="${BUILD_DIR}/Packages.${DIST}_amd64.bz2" 209 local tmp_package_list="${BUILD_DIR}/Packages.${DIST}_amd64" 210 DownloadOrCopy "${PACKAGE_LIST_AMD64}" "${package_list}" 211 VerifyPackageListing "${PACKAGE_FILE_AMD64}" "${package_list}" 212 ExtractPackageBz2 "$package_list" "$tmp_package_list" 213 GeneratePackageList "$tmp_package_list" "$output_file" "${DEBIAN_PACKAGES} 214 ${DEBIAN_PACKAGES_X86}" 215 } 216 217 GeneratePackageListI386() { 218 local output_file="$1" 219 local package_list="${BUILD_DIR}/Packages.${DIST}_i386.bz2" 220 local tmp_package_list="${BUILD_DIR}/Packages.${DIST}_amd64" 221 DownloadOrCopy "${PACKAGE_LIST_I386}" "${package_list}" 222 VerifyPackageListing "${PACKAGE_FILE_I386}" "${package_list}" 223 ExtractPackageBz2 "$package_list" "$tmp_package_list" 224 GeneratePackageList "$tmp_package_list" "$output_file" "${DEBIAN_PACKAGES} 225 ${DEBIAN_PACKAGES_X86}" 226 } 227 228 GeneratePackageListARM() { 229 local output_file="$1" 230 local package_list="${BUILD_DIR}/Packages.${DIST}_arm.bz2" 231 local tmp_package_list="${BUILD_DIR}/Packages.${DIST}_arm" 232 DownloadOrCopy "${PACKAGE_LIST_ARM}" "${package_list}" 233 VerifyPackageListing "${PACKAGE_FILE_ARM}" "${package_list}" 234 ExtractPackageBz2 "$package_list" "$tmp_package_list" 235 GeneratePackageList "$tmp_package_list" "$output_file" "${DEBIAN_PACKAGES}" 236 } 237 238 StripChecksumsFromPackageList() { 239 local package_file="$1" 240 sed -i 's/ [a-f0-9]\{64\}$//' "$package_file" 241 } 242 243 VerifyPackageFilesMatch() { 244 local downloaded_package_file="$1" 245 local stored_package_file="$2" 246 diff -u "$downloaded_package_file" "$stored_package_file" 247 if [ "$?" -ne "0" ]; then 248 echo "ERROR: downloaded package files does not match $2." 249 echo "You may need to run UpdatePackageLists." 250 exit 1 251 fi 252 } 253 254 ###################################################################### 255 # 256 ###################################################################### 257 258 HacksAndPatchesAmd64() { 259 Banner "Misc Hacks & Patches" 260 # these are linker scripts with absolute pathnames in them 261 # which we rewrite here 262 lscripts="${INSTALL_ROOT}/usr/lib/x86_64-linux-gnu/libpthread.so \ 263 ${INSTALL_ROOT}/usr/lib/x86_64-linux-gnu/libc.so" 264 265 #SubBanner "Rewriting Linker Scripts" 266 sed -i -e 's|/usr/lib/x86_64-linux-gnu/||g' ${lscripts} 267 sed -i -e 's|/lib/x86_64-linux-gnu/||g' ${lscripts} 268 269 # This is for chrome's ./build/linux/pkg-config-wrapper 270 # which overwrites PKG_CONFIG_PATH internally 271 SubBanner "Package Configs Symlink" 272 mkdir -p ${INSTALL_ROOT}/usr/share 273 ln -s ../lib/x86_64-linux-gnu/pkgconfig ${INSTALL_ROOT}/usr/share/pkgconfig 274 275 SubBanner "Adding an additional ld.conf include" 276 LD_SO_HACK_CONF="${INSTALL_ROOT}/etc/ld.so.conf.d/zz_hack.conf" 277 echo /usr/lib/gcc/x86_64-linux-gnu/4.6 > "$LD_SO_HACK_CONF" 278 echo /usr/lib >> "$LD_SO_HACK_CONF" 279 } 280 281 282 HacksAndPatchesI386() { 283 Banner "Misc Hacks & Patches" 284 # these are linker scripts with absolute pathnames in them 285 # which we rewrite here 286 lscripts="${INSTALL_ROOT}/usr/lib/i386-linux-gnu/libpthread.so \ 287 ${INSTALL_ROOT}/usr/lib/i386-linux-gnu/libc.so" 288 289 #SubBanner "Rewriting Linker Scripts" 290 sed -i -e 's|/usr/lib/i386-linux-gnu/||g' ${lscripts} 291 sed -i -e 's|/lib/i386-linux-gnu/||g' ${lscripts} 292 293 # This is for chrome's ./build/linux/pkg-config-wrapper 294 # which overwrites PKG_CONFIG_PATH internally 295 SubBanner "Package Configs Symlink" 296 mkdir -p ${INSTALL_ROOT}/usr/share 297 ln -s ../lib/i386-linux-gnu/pkgconfig ${INSTALL_ROOT}/usr/share/pkgconfig 298 299 SubBanner "Adding an additional ld.conf include" 300 LD_SO_HACK_CONF="${INSTALL_ROOT}/etc/ld.so.conf.d/zz_hack.conf" 301 echo /usr/lib/gcc/i486-linux-gnu/4.6 > "$LD_SO_HACK_CONF" 302 echo /usr/lib >> "$LD_SO_HACK_CONF" 303 } 304 305 306 HacksAndPatchesARM() { 307 Banner "Misc Hacks & Patches" 308 # these are linker scripts with absolute pathnames in them 309 # which we rewrite here 310 lscripts="${INSTALL_ROOT}/usr/lib/arm-linux-gnueabihf/libpthread.so \ 311 ${INSTALL_ROOT}/usr/lib/arm-linux-gnueabihf/libc.so" 312 313 #SubBanner "Rewriting Linker Scripts" 314 sed -i -e 's|/usr/lib/arm-linux-gnueabihf/||g' ${lscripts} 315 sed -i -e 's|/lib/arm-linux-gnueabihf/||g' ${lscripts} 316 317 # This is for chrome's ./build/linux/pkg-config-wrapper 318 # which overwrites PKG_CONFIG_PATH internally 319 SubBanner "Package Configs Symlink" 320 mkdir -p ${INSTALL_ROOT}/usr/share 321 ln -s ../lib/arm-linux-gnueabihf/pkgconfig ${INSTALL_ROOT}/usr/share/pkgconfig 322 } 323 324 325 InstallIntoSysroot() { 326 Banner "Install Libs And Headers Into Jail" 327 328 mkdir -p ${BUILD_DIR}/debian-packages 329 mkdir -p ${INSTALL_ROOT} 330 while (( "$#" )); do 331 local file="$1" 332 local package="${BUILD_DIR}/debian-packages/${file##*/}" 333 shift 334 local sha256sum="$1" 335 shift 336 if [ "${#sha256sum}" -ne "64" ]; then 337 echo "Bad sha256sum from package list" 338 exit 1 339 fi 340 341 Banner "Installing ${file}" 342 DownloadOrCopy ${APT_REPO}/pool/${file} ${package} 343 if [ ! -s "${package}" ] ; then 344 echo 345 echo "ERROR: bad package ${package}" 346 exit 1 347 fi 348 echo "${sha256sum} ${package}" | sha256sum --quiet -c 349 350 SubBanner "Extracting to ${INSTALL_ROOT}" 351 dpkg --fsys-tarfile ${package}\ 352 | tar -xf - --exclude=./usr/share -C ${INSTALL_ROOT} 353 done 354 } 355 356 357 CleanupJailSymlinks() { 358 Banner "Jail symlink cleanup" 359 360 SAVEDPWD=$(pwd) 361 cd ${INSTALL_ROOT} 362 find lib lib64 usr/lib -type l -printf '%p %l\n' | while read link target; do 363 # skip links with non-absolute paths 364 echo "${target}" | grep -qs ^/ || continue 365 echo "${link}: ${target}" 366 case "${link}" in 367 usr/lib/gcc/x86_64-linux-gnu/4.*/* | usr/lib/gcc/i486-linux-gnu/4.*/* | \ 368 usr/lib/gcc/arm-linux-gnueabihf/4.*/*) 369 # Relativize the symlink. 370 ln -snfv "../../../../..${target}" "${link}" 371 ;; 372 usr/lib/x86_64-linux-gnu/* | usr/lib/i386-linux-gnu/* | \ 373 usr/lib/arm-linux-gnueabihf/*) 374 # Relativize the symlink. 375 ln -snfv "../../..${target}" "${link}" 376 ;; 377 usr/lib/*) 378 # Relativize the symlink. 379 ln -snfv "../..${target}" "${link}" 380 ;; 381 lib64/* | lib/*) 382 # Relativize the symlink. 383 ln -snfv "..${target}" "${link}" 384 ;; 385 esac 386 done 387 388 find lib lib64 usr/lib -type l -printf '%p %l\n' | while read link target; do 389 # Make sure we catch new bad links. 390 if [ ! -r "${link}" ]; then 391 echo "ERROR: FOUND BAD LINK ${link}" 392 ls -l ${link} 393 exit 1 394 fi 395 done 396 cd "$SAVEDPWD" 397 } 398 399 #@ 400 #@ BuildSysrootAmd64 <tarball-name> 401 #@ 402 #@ Build everything and package it 403 BuildSysrootAmd64() { 404 CheckBuildSysrootArgs $@ 405 ClearInstallDir 406 local package_file="$BUILD_DIR/package_with_sha256sum_amd64" 407 GeneratePackageListAmd64 "$package_file" 408 local files_and_sha256sums="$(cat ${package_file})" 409 StripChecksumsFromPackageList "$package_file" 410 VerifyPackageFilesMatch "$package_file" "$DEBIAN_DEP_LIST_AMD64" 411 InstallIntoSysroot ${files_and_sha256sums} 412 CleanupJailSymlinks 413 HacksAndPatchesAmd64 414 CreateTarBall "$1" 415 } 416 417 #@ 418 #@ BuildSysrootI386 <tarball-name> 419 #@ 420 #@ Build everything and package it 421 BuildSysrootI386() { 422 CheckBuildSysrootArgs $@ 423 ClearInstallDir 424 local package_file="$BUILD_DIR/package_with_sha256sum_i386" 425 GeneratePackageListI386 "$package_file" 426 local files_and_sha256sums="$(cat ${package_file})" 427 StripChecksumsFromPackageList "$package_file" 428 VerifyPackageFilesMatch "$package_file" "$DEBIAN_DEP_LIST_I386" 429 InstallIntoSysroot ${files_and_sha256sums} 430 CleanupJailSymlinks 431 HacksAndPatchesI386 432 CreateTarBall "$1" 433 } 434 435 #@ 436 #@ BuildSysrootARM <tarball-name> 437 #@ 438 #@ Build everything and package it 439 BuildSysrootARM() { 440 CheckBuildSysrootArgs $@ 441 ClearInstallDir 442 local package_file="$BUILD_DIR/package_with_sha256sum_arm" 443 GeneratePackageListARM "$package_file" 444 local files_and_sha256sums="$(cat ${package_file})" 445 StripChecksumsFromPackageList "$package_file" 446 VerifyPackageFilesMatch "$package_file" "$DEBIAN_DEP_LIST_ARM" 447 APT_REPO=${APR_REPO_ARM:=$APT_REPO} 448 InstallIntoSysroot ${files_and_sha256sums} 449 CleanupJailSymlinks 450 HacksAndPatchesARM 451 CreateTarBall "$1" 452 } 453 454 # 455 # CheckForDebianGPGKeyring 456 # 457 # Make sure the Debian GPG keys exist. Otherwise print a helpful message. 458 # 459 CheckForDebianGPGKeyring() { 460 if [ ! -e "$KEYRING_FILE" ]; then 461 echo "Debian GPG keys missing. Install the debian-archive-keyring package." 462 exit 1 463 fi 464 } 465 466 # 467 # VerifyPackageListing 468 # 469 # Verifies the downloaded Packages.bz2 file has the right checksums. 470 # 471 VerifyPackageListing() { 472 local file_path=$1 473 local output_file=$2 474 local release_file="${BUILD_DIR}/${RELEASE_FILE}" 475 local release_file_gpg="${BUILD_DIR}/${RELEASE_FILE_GPG}" 476 477 CheckForDebianGPGKeyring 478 479 DownloadOrCopy ${RELEASE_LIST} ${release_file} 480 DownloadOrCopy ${RELEASE_LIST_GPG} ${release_file_gpg} 481 echo "Verifying: ${release_file} with ${release_file_gpg}" 482 gpgv --keyring $KEYRING_FILE ${release_file_gpg} ${release_file} 483 484 echo "Verifying: ${output_file}" 485 local checksums=$(grep ${file_path} ${release_file} | cut -d " " -f 2) 486 local sha256sum=$(echo ${checksums} | cut -d " " -f 3) 487 488 if [ "${#sha256sum}" -ne "64" ]; then 489 echo "Bad sha256sum from ${RELEASE_LIST}" 490 exit 1 491 fi 492 493 echo "${sha256sum} ${output_file}" | sha256sum --quiet -c 494 } 495 496 # 497 # GeneratePackageList 498 # 499 # Looks up package names in ${BUILD_DIR}/Packages and write list of URLs 500 # to output file. 501 # 502 GeneratePackageList() { 503 local input_file="$1" 504 local output_file="$2" 505 echo "Updating: ${output_file} from ${input_file}" 506 /bin/rm -f "${output_file}" 507 shift 508 shift 509 for pkg in $@ ; do 510 local pkg_full=$(grep -A 1 " ${pkg}\$" "$input_file" | \ 511 egrep -o "pool/.*") 512 if [ -z "${pkg_full}" ]; then 513 echo "ERROR: missing package: $pkg" 514 exit 1 515 fi 516 local pkg_nopool=$(echo "$pkg_full" | sed "s/^pool\///") 517 local sha256sum=$(grep -A 4 " ${pkg}\$" "$input_file" | \ 518 grep ^SHA256: | sed 's/^SHA256: //') 519 if [ "${#sha256sum}" -ne "64" ]; then 520 echo "Bad sha256sum from Packages" 521 exit 1 522 fi 523 echo $pkg_nopool $sha256sum >> "$output_file" 524 done 525 # sort -o does an in-place sort of this file 526 sort "$output_file" -o "$output_file" 527 } 528 529 #@ 530 #@ UpdatePackageListsAmd64 531 #@ 532 #@ Regenerate the package lists such that they contain an up-to-date 533 #@ list of URLs within the Debian archive. (For amd64) 534 UpdatePackageListsAmd64() { 535 GeneratePackageListAmd64 "$DEBIAN_DEP_LIST_AMD64" 536 StripChecksumsFromPackageList "$DEBIAN_DEP_LIST_AMD64" 537 } 538 539 #@ 540 #@ UpdatePackageListsI386 541 #@ 542 #@ Regenerate the package lists such that they contain an up-to-date 543 #@ list of URLs within the Debian archive. (For i386) 544 UpdatePackageListsI386() { 545 GeneratePackageListI386 "$DEBIAN_DEP_LIST_I386" 546 StripChecksumsFromPackageList "$DEBIAN_DEP_LIST_I386" 547 } 548 549 #@ 550 #@ UpdatePackageListsARM 551 #@ 552 #@ Regenerate the package lists such that they contain an up-to-date 553 #@ list of URLs within the Debian archive. (For arm) 554 UpdatePackageListsARM() { 555 GeneratePackageListARM "$DEBIAN_DEP_LIST_ARM" 556 StripChecksumsFromPackageList "$DEBIAN_DEP_LIST_ARM" 557 } 558 559 if [ $# -eq 0 ] ; then 560 echo "ERROR: you must specify a mode on the commandline" 561 echo 562 Usage 563 exit 1 564 elif [ "$(type -t $1)" != "function" ]; then 565 echo "ERROR: unknown function '$1'." >&2 566 echo "For help, try:" 567 echo " $0 help" 568 exit 1 569 else 570 ChangeDirectory 571 SetEnvironmentVariables "$1" 572 SanityCheck 573 "$@" 574 fi 575