1 #!/usr/bin/env bash 2 # Copyright 2016 The TensorFlow Authors. All Rights Reserved. 3 # 4 # Licensed under the Apache License, Version 2.0 (the "License"); 5 # you may not use this file except in compliance with the License. 6 # You may obtain a copy of the License at 7 # 8 # http://www.apache.org/licenses/LICENSE-2.0 9 # 10 # Unless required by applicable law or agreed to in writing, software 11 # distributed under the License is distributed on an "AS IS" BASIS, 12 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 # See the License for the specific language governing permissions and 14 # limitations under the License. 15 # ============================================================================== 16 # 17 # Usage: 18 # ci_parameterized_build.sh 19 # 20 # The script obeys the following required environment variables: 21 # TF_BUILD_CONTAINER_TYPE: (CPU | GPU | ANDROID | ANDROID_FULL) 22 # TF_BUILD_PYTHON_VERSION: (PYTHON2 | PYTHON3 | PYTHON3.5) 23 # TF_BUILD_IS_PIP: (NO_PIP | PIP | BOTH) 24 # 25 # The below environment variable is required, but will be deprecated together 26 # with TF_BUILD_MAVX and both will be replaced by TF_BUILD_OPTIONS. 27 # TF_BUILD_IS_OPT: (NO_OPT | OPT) 28 # 29 # Note: 30 # 1) Certain combinations of parameter values are regarded 31 # as invalid and will cause the script to exit with code 0. For example: 32 # NO_OPT & PIP (PIP builds should always use OPT) 33 # ANDROID & PIP (Android and PIP builds are mutually exclusive) 34 # 35 # 2) TF_BUILD_PYTHON_VERSION is set to PYTHON3, the build will use the version 36 # pointed to by "which python3" on the system, which is typically python3.4. To 37 # build for python3.5, set the environment variable to PYTHON3.5 38 # 39 # 40 # Additionally, the script follows the directions of optional environment 41 # variables: 42 # TF_BUILD_DRY_RUN: If it is set to any non-empty value that is not "0", 43 # the script will just generate and print the final 44 # command, but not actually run it. 45 # TF_BUILD_APPEND_CI_DOCKER_EXTRA_PARAMS: 46 # String appended to the content of CI_DOCKER_EXTRA_PARAMS 47 # TF_BUILD_APPEND_ARGUMENTS: 48 # Additional command line arguments for the bazel, 49 # pip.sh or android.sh command 50 # TF_BUILD_MAVX: (Soon to be deprecated, use TF_BUILD_OPTIONS instead) 51 # (unset | MAVX | MAVX2) 52 # If set to MAVX or MAVX2, will cause bazel to use the 53 # additional flag --copt=-mavx or --copt=-mavx2, to 54 # perform AVX or AVX2 builds, respectively. This requires 55 # AVX- or AVX2-compatible CPUs. 56 # TF_BUILD_BAZEL_TARGET: 57 # Used to override the default bazel build target: 58 # //tensorflow/... -//tensorflow/compiler 59 # TF_BUILD_BAZEL_CLEAN: 60 # Will perform "bazel clean", if and only if this variable 61 # is set to any non-empty and non-0 value 62 # TF_BAZEL_BUILD_ONLY: 63 # If it is set to any non-empty value that is not "0", Bazel 64 # will only build specified targets 65 # TF_GPU_COUNT: 66 # Run this many parallel tests for serial builds. 67 # For now, only can be edited for PIP builds. 68 # TF_BUILD_TEST_TUTORIALS: 69 # If set to any non-empty and non-0 value, will perform 70 # tutorials tests (Applicable only if TF_BUILD_IS_PIP is 71 # PIP or BOTH). 72 # See builds/test_tutorials.sh 73 # TF_BUILD_INTEGRATION_TESTS: 74 # If set this will perform integration tests. See 75 # builds/integration_tests.sh. 76 # TF_BUILD_RUN_BENCHMARKS: 77 # If set to any non-empty and non-0 value, will perform 78 # the benchmark tests (see *_logged_benchmark targets in 79 # tools/test/BUILD) 80 # TF_BUILD_OPTIONS: 81 # (FASTBUILD | OPT | OPTDBG | MAVX | MAVX2_FMA | MAVX_DBG | 82 # MAVX2_FMA_DBG) 83 # Use the specified configurations when building. 84 # When set, overrides TF_BUILD_IS_OPT and TF_BUILD_MAVX 85 # options, as this will replace the two. 86 # TF_BUILD_TEST_TIMEOUT: 87 # Sets the value of bazel --test_timeout, defaults to -1 88 # which uses the bazel defaults. 89 # TF_SKIP_CONTRIB_TESTS: 90 # If set to any non-empty or non-0 value, will skip running 91 # contrib tests. 92 # TF_NIGHTLY: 93 # If this run is being used to build the tf_nightly pip 94 # packages. 95 # TF_CUDA_CLANG: 96 # If set to 1, builds and runs cuda_clang configuration. 97 # Only available inside GPU containers. 98 # 99 # This script can be used by Jenkins parameterized / matrix builds. 100 101 # Helper function: Convert to lower case 102 to_lower () { 103 echo "$1" | tr '[:upper:]' '[:lower:]' 104 } 105 106 # Helper function: Strip leading and trailing whitespaces 107 str_strip () { 108 echo -e "$1" | sed -e 's/^[[:space:]]*//' -e 's/[[:space:]]*$//' 109 } 110 111 # Helper function: Exit on failure 112 die () { 113 echo $@ 114 exit 1 115 } 116 117 ########################################################## 118 # Default configuration 119 CI_BUILD_DIR="tensorflow/tools/ci_build" 120 121 # Command to call when Docker is available 122 DOCKER_MAIN_CMD="${CI_BUILD_DIR}/ci_build.sh" 123 # Command to call when Docker is unavailable 124 NO_DOCKER_MAIN_CMD="${CI_BUILD_DIR}/builds/configured" 125 126 # Additional option flags to apply when Docker is unavailable (e.g., on Mac) 127 NO_DOCKER_OPT_FLAG="--genrule_strategy=standalone" 128 129 DO_DOCKER=1 130 131 # Default values for various settings. 132 TF_BUILD_TEST_TIMEOUT=${TF_BUILD_TEST_TIMEOUT:--1} # Use bazel defaults 133 TF_GPU_COUNT=${TF_GPU_COUNT:-4} 134 135 # Helpful flags: 136 # --test_summary=detailed: Tell us more about which targets are being built 137 # --keep_going: Don't stop at the first failure; tell us all the failures 138 # --build_tests_only: Don't build targets depended on by tests if the test is 139 # disabled. Also saves some compilation time. Otherwise, 140 # tries to build everything. 141 # --test_timeout: Test timeouts in the order short,moderate,long,eternal. 142 # --test_env: Environment variables to set when running bazel tests. These are 143 # especially important when using --run_under with 144 # parallel_gpu_execute. 145 BAZEL_TEST_FLAGS=""\ 146 "--test_summary=detailed --build_tests_only --keep_going "\ 147 "--test_timeout=${TF_BUILD_TEST_TIMEOUT} "\ 148 "--test_env=TF_GPU_COUNT=${TF_GPU_COUNT}" 149 150 # Only set these environment variables if they're specified, to avoid causing 151 # problems like b/118404869, where an envvar set to the empty string has 152 # different semantics from an unset envvar. 153 if [ -n "${TF_TESTS_PER_GPU}" ]; then 154 BAZEL_TEST_FLAGS="${BAZEL_TEST_FLAGS} "\ 155 "--test_env=TF_TESTS_PER_GPU=${TF_TESTS_PER_GPU}" 156 fi 157 if [ -n "${TF_PER_DEVICE_MEMORY_LIMIT_MB}" ]; then 158 BAZEL_TEST_FLAGS="${BAZEL_TEST_FLAGS} "\ 159 "--test_env=TF_PER_DEVICE_MEMORY_LIMIT_MB=${TF_PER_DEVICE_MEMORY_LIMIT_MB}" 160 fi 161 162 BAZEL_BUILD_FLAGS="--keep_going" 163 164 # Explicitly set jdk8 since that's what's installed in our images. Note that 165 # bazel 0.16 and higher defaults to jdk9, which causes failures. See b/117634064 166 BAZEL_JAVA_FLAGS="--java_toolchain=@bazel_tools//tools/jdk:toolchain_hostjdk8" 167 168 BAZEL_CMD="bazel test ${BAZEL_TEST_FLAGS} ${BAZEL_JAVA_FLAGS}" 169 BAZEL_BUILD_ONLY_CMD="bazel build ${BAZEL_BUILD_FLAGS} ${BAZEL_JAVA_FLAGS}" 170 BAZEL_CLEAN_CMD="bazel clean" 171 172 PIP_CMD="${CI_BUILD_DIR}/builds/pip.sh" 173 PIP_TEST_TUTORIALS_FLAG="--test_tutorials" 174 PIP_INTEGRATION_TESTS_FLAG="--integration_tests" 175 ANDROID_CMD="${CI_BUILD_DIR}/builds/android.sh" 176 ANDROID_FULL_CMD="${CI_BUILD_DIR}/builds/android_full.sh" 177 178 PARALLEL_GPU_TEST_CMD='//tensorflow/tools/ci_build/gpu_build:parallel_gpu_execute' 179 180 BENCHMARK_CMD="${CI_BUILD_DIR}/builds/benchmark.sh" 181 182 EXTRA_PARAMS="" 183 BAZEL_TARGET="//tensorflow/... -//tensorflow/compiler/..." 184 185 if [[ -n "$TF_SKIP_CONTRIB_TESTS" ]]; then 186 BAZEL_TARGET="${BAZEL_TARGET} -//tensorflow/contrib/..." 187 fi 188 189 TUT_TEST_DATA_DIR="/tmp/tf_tutorial_test_data" 190 191 ########################################################## 192 193 echo "Parameterized build starts at: $(date)" 194 echo "" 195 START_TIME=$(date +'%s') 196 197 # Convert all the required environment variables to lower case 198 TF_BUILD_CONTAINER_TYPE=$(to_lower ${TF_BUILD_CONTAINER_TYPE}) 199 TF_BUILD_PYTHON_VERSION=$(to_lower ${TF_BUILD_PYTHON_VERSION}) 200 TF_BUILD_IS_OPT=$(to_lower ${TF_BUILD_IS_OPT}) 201 TF_BUILD_IS_PIP=$(to_lower ${TF_BUILD_IS_PIP}) 202 203 if [[ ! -z "${TF_BUILD_MAVX}" ]]; then 204 TF_BUILD_MAVX=$(to_lower ${TF_BUILD_MAVX}) 205 fi 206 207 208 # Print parameter values 209 echo "Required build parameters:" 210 echo " TF_BUILD_CONTAINER_TYPE=${TF_BUILD_CONTAINER_TYPE}" 211 echo " TF_BUILD_PYTHON_VERSION=${TF_BUILD_PYTHON_VERSION}" 212 echo " TF_BUILD_IS_OPT=${TF_BUILD_IS_OPT}" 213 echo " TF_BUILD_IS_PIP=${TF_BUILD_IS_PIP}" 214 echo "Optional build parameters:" 215 echo " TF_BUILD_DRY_RUN=${TF_BUILD_DRY_RUN}" 216 echo " TF_BUILD_MAVX=${TF_BUILD_MAVX}" 217 echo " TF_BUILD_APPEND_CI_DOCKER_EXTRA_PARAMS="\ 218 "${TF_BUILD_APPEND_CI_DOCKER_EXTRA_PARAMS}" 219 echo " TF_BUILD_APPEND_ARGUMENTS=${TF_BUILD_APPEND_ARGUMENTS}" 220 echo " TF_BUILD_BAZEL_TARGET=${TF_BUILD_BAZEL_TARGET}" 221 echo " TF_BUILD_BAZEL_CLEAN=${TF_BUILD_BAZEL_CLEAN}" 222 echo " TF_BUILD_TEST_TUTORIALS=${TF_BUILD_TEST_TUTORIALS}" 223 echo " TF_BUILD_INTEGRATION_TESTS=${TF_BUILD_INTEGRATION_TESTS}" 224 echo " TF_BUILD_RUN_BENCHMARKS=${TF_BUILD_RUN_BENCHMARKS}" 225 echo " TF_BUILD_OPTIONS=${TF_BUILD_OPTIONS}" 226 227 228 # Function that tries to determine CUDA capability, if deviceQuery binary 229 # is available on path 230 function get_cuda_capability_version() { 231 if [[ ! -z $(which deviceQuery) ]]; then 232 # The first listed device is used 233 deviceQuery | grep "CUDA Capability .* version" | \ 234 head -1 | awk '{print $NF}' 235 fi 236 } 237 238 # Container type, e.g., CPU, GPU 239 CTYPE=${TF_BUILD_CONTAINER_TYPE} 240 241 # Determine if the machine is a Mac 242 OPT_FLAG="--test_output=errors" 243 if [[ "$(uname -s)" == "Darwin" ]]; then 244 DO_DOCKER=0 245 246 echo "It appears this machine is a Mac. "\ 247 "We will perform this build without Docker." 248 echo "Also, the additional option flags will be applied to the build:" 249 echo " ${NO_DOCKER_OPT_FLAG}" 250 MAIN_CMD="${NO_DOCKER_MAIN_CMD} ${CTYPE}" 251 OPT_FLAG="${OPT_FLAG} ${NO_DOCKER_OPT_FLAG}" 252 fi 253 254 # In DO_DOCKER mode, appends environment variable to docker's run invocation. 255 # Otherwise, exports the corresponding variable. 256 function set_script_variable() { 257 local VAR="$1" 258 local VALUE="$2" 259 if [[ $DO_DOCKER == "1" ]]; then 260 TF_BUILD_APPEND_CI_DOCKER_EXTRA_PARAMS="${TF_BUILD_APPEND_CI_DOCKER_EXTRA_PARAMS} -e $VAR=$VALUE" 261 else 262 export $VAR="$VALUE" 263 fi 264 } 265 266 267 # Process container type 268 if [[ ${CTYPE} == cpu* ]] || [[ ${CTYPE} == "debian.jessie.cpu" ]]; then 269 : 270 elif [[ ${CTYPE} == gpu* ]]; then 271 set_script_variable TF_NEED_CUDA 1 272 273 if [[ $TF_CUDA_CLANG == "1" ]]; then 274 OPT_FLAG="${OPT_FLAG} --config=cuda_clang" 275 276 set_script_variable TF_CUDA_CLANG 1 277 # For cuda_clang we download `clang` while building. 278 set_script_variable TF_DOWNLOAD_CLANG 1 279 else 280 OPT_FLAG="${OPT_FLAG} --config=cuda" 281 fi 282 283 # Attempt to determine CUDA capability version automatically and use it if 284 # CUDA capability version is not specified by the environment variables. 285 CUDA_CAPA_VER=$(get_cuda_capability_version) 286 287 if [[ ! -z ${CUDA_CAPA_VER} ]]; then 288 AUTO_CUDA_CAPA_VER=0 289 if [[ ${DO_DOCKER} == "1" ]] && \ 290 [[ "${TF_BUILD_APPEND_CI_DOCKER_EXTRA_PARAMS}" != \ 291 *"TF_CUDA_COMPUTE_CAPABILITIES="* ]]; then 292 AUTO_CUDA_CAPA_VER=1 293 TF_BUILD_APPEND_CI_DOCKER_EXTRA_PARAMS=\ 294 "${TF_BUILD_APPEND_CI_DOCKER_EXTRA_PARAMS} -e "\ 295 "TF_CUDA_COMPUTE_CAPABILITIES=${CUDA_CAPA_VER}" 296 297 echo "Docker GPU build: TF_BUILD_APPEND_CI_DOCKER_EXTRA_PARAMS="\ 298 "\"${TF_BUILD_APPEND_CI_DOCKER_EXTRA_PARAMS}\"" 299 elif [[ ${DO_DOCKER} == "0" ]] && \ 300 [[ -z "${TF_CUDA_COMPUTE_CAPABILITIES}" ]]; then 301 AUTO_CUDA_CAPA_VER=1 302 TF_CUDA_COMPUTE_CAPABILITIES="${CUDA_CAPA_VER}" 303 304 echo "Non-Docker GPU build: TF_CUDA_COMPUTE_CAPABILITIES="\ 305 "\"${TF_CUDA_COMPUTE_CAPABILITIES}\"" 306 fi 307 308 if [[ ${AUTO_CUDA_CAPA_VER} == "1" ]]; then 309 echo "TF_CUDA_COMPUTE_CAPABILITIES is not set:" 310 echo "Using CUDA capability version from deviceQuery: ${CUDA_CAPA_VER}" 311 echo "" 312 fi 313 fi 314 elif [[ ${CTYPE} == "android" ]] || [[ ${CTYPE} == "android_full" ]]; then 315 : 316 else 317 die "Unrecognized value in TF_BUILD_CONTAINER_TYPE: "\ 318 "\"${TF_BUILD_CONTAINER_TYPE}\"" 319 fi 320 321 # Determine if this is a benchmarks job 322 RUN_BENCHMARKS=0 323 if [[ ! -z "${TF_BUILD_RUN_BENCHMARKS}" ]] && 324 [[ "${TF_BUILD_RUN_BENCHMARKS}" != "0" ]]; then 325 RUN_BENCHMARKS=1 326 fi 327 328 # Process Bazel "-c opt" flag 329 if [[ -z "${TF_BUILD_OPTIONS}" ]]; then 330 if [[ ${TF_BUILD_IS_OPT} == "no_opt" ]]; then 331 # PIP builds are done only with the -c opt flag 332 if [[ ${TF_BUILD_IS_PIP} == "pip" ]]; then 333 echo "Skipping parameter combination: ${TF_BUILD_IS_OPT} & "\ 334 "${TF_BUILD_IS_PIP}" 335 exit 0 336 fi 337 338 elif [[ ${TF_BUILD_IS_OPT} == "opt" ]]; then 339 OPT_FLAG="${OPT_FLAG} -c opt" 340 else 341 die "Unrecognized value in TF_BUILD_IS_OPT: \"${TF_BUILD_IS_OPT}\"" 342 fi 343 344 # Process MAVX option 345 if [[ ! -z "${TF_BUILD_MAVX}" ]]; then 346 if [[ "${TF_BUILD_MAVX}" == "mavx" ]]; then 347 OPT_FLAG="${OPT_FLAG} --copt=-mavx" 348 elif [[ "${TF_BUILD_MAVX}" == "mavx2" ]]; then 349 OPT_FLAG="${OPT_FLAG} --copt=-mavx2" 350 else 351 die "Unsupported value in TF_BUILD_MAVX: ${TF_BUILD_MAVX}" 352 fi 353 fi 354 else 355 case $TF_BUILD_OPTIONS in 356 FASTBUILD) 357 echo "Running FASTBUILD mode (noopt, nodbg)." 358 ;; 359 OPT) 360 OPT_FLAG="${OPT_FLAG} -c opt" 361 ;; 362 OPTDBG) 363 OPT_FLAG="${OPT_FLAG} -c opt --copt=-g" 364 ;; 365 MAVX) 366 OPT_FLAG="${OPT_FLAG} -c opt --copt=-mavx" 367 ;; 368 MAVX_DBG) 369 OPT_FLAG="${OPT_FLAG} -c opt --copt=-g --copt=-mavx" 370 ;; 371 MAVX2_FMA) 372 OPT_FLAG="${OPT_FLAG} -c opt --copt=-mavx2 --copt=-mfma" 373 ;; 374 MAVX2_FMA_DBG) 375 OPT_FLAG="${OPT_FLAG} -c opt --copt=-g --copt=-mavx2 --copt=-mfma" 376 ;; 377 esac 378 fi 379 380 # Strip whitespaces from OPT_FLAG 381 OPT_FLAG=$(str_strip "${OPT_FLAG}") 382 383 384 # 1) Filter out benchmark tests if this is not a benchmarks job; 385 # 2) Filter out tests with the "nomac" tag if the build is on Mac OS X. 386 EXTRA_ARGS=${DEFAULT_BAZEL_CONFIGS} 387 IS_MAC=0 388 if [[ "$(uname)" == "Darwin" ]]; then 389 IS_MAC=1 390 fi 391 if [[ "${TF_BUILD_APPEND_ARGUMENTS}" == *"--test_tag_filters="* ]]; then 392 ITEMS=(${TF_BUILD_APPEND_ARGUMENTS}) 393 394 for ITEM in "${ITEMS[@]}"; do 395 if [[ ${ITEM} == *"--test_tag_filters="* ]]; then 396 NEW_ITEM="${ITEM}" 397 if [[ ${NEW_ITEM} != *"benchmark-test"* ]]; then 398 NEW_ITEM="${NEW_ITEM},-benchmark-test" 399 fi 400 if [[ ${IS_MAC} == "1" ]] && [[ ${NEW_ITEM} != *"nomac"* ]]; then 401 # TODO(b/122370901): Fix nomac, no_mac inconsistency. 402 NEW_ITEM="${NEW_ITEM},-nomac,-no_mac" 403 fi 404 EXTRA_ARGS="${EXTRA_ARGS} ${NEW_ITEM}" 405 else 406 EXTRA_ARGS="${EXTRA_ARGS} ${ITEM}" 407 fi 408 done 409 else 410 EXTRA_ARGS="${EXTRA_ARGS} ${TF_BUILD_APPEND_ARGUMENTS} --test_tag_filters=-no_oss,-oss_serial,-benchmark-test" 411 if [[ ${IS_MAC} == "1" ]]; then 412 # TODO(b/122370901): Fix nomac, no_mac inconsistency. 413 EXTRA_ARGS="${EXTRA_ARGS},-nomac,-no_mac" 414 fi 415 EXTRA_ARGS="${EXTRA_ARGS} --build_tag_filters=-no_oss,-oss_serial,-benchmark-test" 416 if [[ ${IS_MAC} == "1" ]]; then 417 # TODO(b/122370901): Fix nomac, no_mac inconsistency. 418 EXTRA_ARGS="${EXTRA_ARGS},-nomac,-no_mac" 419 fi 420 fi 421 422 # For any "tool" dependencies in genrules, Bazel will build them for host 423 # instead of the target configuration. We can save some build time by setting 424 # this flag, and it only affects a few tests. 425 EXTRA_ARGS="${EXTRA_ARGS} --distinct_host_configuration=false" 426 427 if [[ ! -z "${TF_BAZEL_BUILD_ONLY}" ]] && 428 [[ "${TF_BAZEL_BUILD_ONLY}" != "0" ]];then 429 BAZEL_CMD=${BAZEL_BUILD_ONLY_CMD} 430 fi 431 432 # Process PIP install-test option 433 if [[ ${TF_BUILD_IS_PIP} == "no_pip" ]] || 434 [[ ${TF_BUILD_IS_PIP} == "both" ]]; then 435 # Process optional bazel target override 436 if [[ ! -z "${TF_BUILD_BAZEL_TARGET}" ]]; then 437 BAZEL_TARGET=${TF_BUILD_BAZEL_TARGET} 438 fi 439 440 if [[ ${CTYPE} == cpu* ]] || \ 441 [[ ${CTYPE} == "debian.jessie.cpu" ]]; then 442 # CPU only command, fully parallel. 443 NO_PIP_MAIN_CMD="${MAIN_CMD} ${BAZEL_CMD} ${OPT_FLAG} "\ 444 "${EXTRA_ARGS} -- ${BAZEL_TARGET}" 445 elif [[ ${CTYPE} == gpu* ]]; then 446 # GPU only command, run as many jobs as the GPU count only. 447 NO_PIP_MAIN_CMD="${BAZEL_CMD} ${OPT_FLAG} "\ 448 "--local_test_jobs=${TF_GPU_COUNT} "\ 449 "--run_under=${PARALLEL_GPU_TEST_CMD} "\ 450 "${EXTRA_ARGS} -- ${BAZEL_TARGET}" 451 elif [[ ${CTYPE} == "android" ]]; then 452 # Run android specific script for android build. 453 NO_PIP_MAIN_CMD="${ANDROID_CMD} ${OPT_FLAG} " 454 elif [[ ${CTYPE} == "android_full" ]]; then 455 # Run android specific script for full android build. 456 NO_PIP_MAIN_CMD="${ANDROID_FULL_CMD} ${OPT_FLAG} " 457 fi 458 459 fi 460 461 if [[ ${TF_BUILD_IS_PIP} == "pip" ]] || 462 [[ ${TF_BUILD_IS_PIP} == "both" ]]; then 463 # Android builds conflict with PIP builds 464 if [[ ${CTYPE} == "android" ]]; then 465 echo "Skipping parameter combination: ${TF_BUILD_IS_PIP} & "\ 466 "${TF_BUILD_CONTAINER_TYPE}" 467 exit 0 468 fi 469 470 PIP_MAIN_CMD="${MAIN_CMD} ${PIP_CMD} ${CTYPE} ${EXTRA_ARGS} ${OPT_FLAG}" 471 472 # Add flag for integration tests 473 if [[ ! -z "${TF_BUILD_INTEGRATION_TESTS}" ]] && 474 [[ "${TF_BUILD_INTEGRATION_TESTS}" != "0" ]]; then 475 PIP_MAIN_CMD="${PIP_MAIN_CMD} ${PIP_INTEGRATION_TESTS_FLAG}" 476 fi 477 478 # Add command for tutorial test 479 if [[ ! -z "${TF_BUILD_TEST_TUTORIALS}" ]] && 480 [[ "${TF_BUILD_TEST_TUTORIALS}" != "0" ]]; then 481 PIP_MAIN_CMD="${PIP_MAIN_CMD} ${PIP_TEST_TUTORIALS_FLAG}" 482 483 # Prepare data directory for tutorial tests 484 mkdir -p "${TUT_TEST_DATA_DIR}" || 485 die "FAILED to create data directory for tutorial tests: "\ 486 "${TUT_TEST_DATA_DIR}" 487 488 if [[ "${DO_DOCKER}" == "1" ]]; then 489 EXTRA_PARAMS="${EXTRA_PARAMS} -v ${TUT_TEST_DATA_DIR}:${TUT_TEST_DATA_DIR}" 490 fi 491 fi 492 fi 493 494 495 if [[ ${RUN_BENCHMARKS} == "1" ]]; then 496 MAIN_CMD="${BENCHMARK_CMD} ${OPT_FLAG}" 497 elif [[ ${TF_BUILD_IS_PIP} == "no_pip" ]]; then 498 MAIN_CMD="${NO_PIP_MAIN_CMD}" 499 elif [[ ${TF_BUILD_IS_PIP} == "pip" ]]; then 500 MAIN_CMD="${PIP_MAIN_CMD}" 501 elif [[ ${TF_BUILD_IS_PIP} == "both" ]]; then 502 MAIN_CMD="${NO_PIP_MAIN_CMD} && ${PIP_MAIN_CMD}" 503 else 504 die "Unrecognized value in TF_BUILD_IS_PIP: \"${TF_BUILD_IS_PIP}\"" 505 fi 506 507 # Check if this is a tf_nightly build 508 if [[ "${TF_NIGHTLY}" == "1" ]]; then 509 EXTRA_PARAMS="${EXTRA_PARAMS} -e TF_NIGHTLY=1" 510 fi 511 512 # Process Python version 513 if [[ ${TF_BUILD_PYTHON_VERSION} == "python2" ]]; then 514 : 515 elif [[ ${TF_BUILD_PYTHON_VERSION} == "python3" || \ 516 ${TF_BUILD_PYTHON_VERSION} == "python3.4" || \ 517 ${TF_BUILD_PYTHON_VERSION} == "python3.5" || \ 518 ${TF_BUILD_PYTHON_VERSION} == "python3.6" ]]; then 519 # Supply proper environment variable to select Python 3 520 if [[ "${DO_DOCKER}" == "1" ]]; then 521 EXTRA_PARAMS="${EXTRA_PARAMS} -e CI_BUILD_PYTHON=${TF_BUILD_PYTHON_VERSION}" 522 else 523 # Determine the path to python3 524 PYTHON3_PATH=$(which "${TF_BUILD_PYTHON_VERSION}" | head -1) 525 if [[ -z "${PYTHON3_PATH}" ]]; then 526 die "ERROR: Failed to locate ${TF_BUILD_PYTHON_VERSION} binary on path" 527 else 528 echo "Found ${TF_BUILD_PYTHON_VERSION} binary at: ${PYTHON3_PATH}" 529 fi 530 531 export PYTHON_BIN_PATH="${PYTHON3_PATH}" 532 fi 533 534 else 535 die "Unrecognized value in TF_BUILD_PYTHON_VERSION: "\ 536 "\"${TF_BUILD_PYTHON_VERSION}\"" 537 fi 538 539 # Append additional Docker extra parameters 540 EXTRA_PARAMS="${EXTRA_PARAMS} ${TF_BUILD_APPEND_CI_DOCKER_EXTRA_PARAMS}" 541 542 # Finally, do a dry run or call the command 543 544 # The command, which may consist of multiple parts (e.g., in the case of 545 # TF_BUILD_SERIAL_TESTS=1), are written to a bash script, which is 546 # then called. The name of the script is randomized to make concurrent 547 # builds on the node possible. 548 TMP_SCRIPT="$(mktemp)_ci_parameterized_build.sh" 549 550 if [[ "${DO_DOCKER}" == "1" ]]; then 551 # Map the tmp script into the Docker container 552 EXTRA_PARAMS="${EXTRA_PARAMS} -v ${TMP_SCRIPT}:/tmp/tf_build.sh" 553 554 if [[ ! -z "${TF_BUILD_BAZEL_CLEAN}" ]] && 555 [[ "${TF_BUILD_BAZEL_CLEAN}" != "0" ]] && 556 [[ "${TF_BUILD_IS_PIP}" != "both" ]]; then 557 # For TF_BUILD_IS_PIP == both, "bazel clean" will have already 558 # been performed before the "bazel test" step 559 EXTRA_PARAMS="${EXTRA_PARAMS} -e TF_BUILD_BAZEL_CLEAN=1" 560 fi 561 562 EXTRA_PARAMS=$(str_strip "${EXTRA_PARAMS}") 563 564 echo "Exporting CI_DOCKER_EXTRA_PARAMS: ${EXTRA_PARAMS}" 565 export CI_DOCKER_EXTRA_PARAMS="${EXTRA_PARAMS}" 566 fi 567 568 # Write to the tmp script 569 echo "#!/usr/bin/env bash" > ${TMP_SCRIPT} 570 if [[ ! -z "${TF_BUILD_BAZEL_CLEAN}" ]] && 571 [[ "${TF_BUILD_BAZEL_CLEAN}" != "0" ]]; then 572 echo ${BAZEL_CLEAN_CMD} >> ${TMP_SCRIPT} 573 fi 574 echo ${MAIN_CMD} >> ${TMP_SCRIPT} 575 576 echo "Executing final command (${TMP_SCRIPT})..." 577 echo "==========================================" 578 cat ${TMP_SCRIPT} 579 echo "==========================================" 580 echo "" 581 582 583 TMP_DIR="" 584 DOCKERFILE_FLAG="" 585 if [[ "${DO_DOCKER}" == "1" ]]; then 586 if [[ "${TF_BUILD_PYTHON_VERSION}" == "python3.5" ]] || 587 [[ "${TF_BUILD_PYTHON_VERSION}" == "python3.6" ]]; then 588 # Modify Dockerfile for Python3.5 | Python3.6 build 589 TMP_DIR=$(mktemp -d) 590 echo "Docker build will occur in temporary directory: ${TMP_DIR}" 591 592 # Copy the files required for the docker build 593 SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" 594 cp -r "${SCRIPT_DIR}/install" "${TMP_DIR}/install" || \ 595 die "ERROR: Failed to copy directory ${SCRIPT_DIR}/install" 596 597 DOCKERFILE="${SCRIPT_DIR}/Dockerfile.${TF_BUILD_CONTAINER_TYPE}" 598 cp "${DOCKERFILE}" "${TMP_DIR}/" || \ 599 die "ERROR: Failed to copy Dockerfile at ${DOCKERFILE}" 600 DOCKERFILE="${TMP_DIR}/Dockerfile.${TF_BUILD_CONTAINER_TYPE}" 601 602 # Replace a line in the Dockerfile 603 if sed -i \ 604 "s/RUN \/install\/install_pip_packages.sh/RUN \/install\/install_${TF_BUILD_PYTHON_VERSION}_pip_packages.sh/g" \ 605 "${DOCKERFILE}" 606 then 607 echo "Copied and modified Dockerfile for ${TF_BUILD_PYTHON_VERSION} build: ${DOCKERFILE}" 608 else 609 die "ERROR: Faild to copy and modify Dockerfile: ${DOCKERFILE}" 610 fi 611 612 DOCKERFILE_FLAG="--dockerfile ${DOCKERFILE}" 613 fi 614 fi 615 616 # Set a disk usage trap. 617 function debug_disk_usage { 618 echo "Finished script... disk usage report in ${TMP_DIR}" 619 du -k -d 2 ${TMP_DIR} | sort -n -r 620 } 621 # trap debug_disk_usage EXIT 622 623 chmod +x ${TMP_SCRIPT} 624 625 # Map TF_BUILD container types to containers we actually have. 626 if [[ "${CTYPE}" == "android_full" ]]; then 627 CONTAINER="android" 628 else 629 CONTAINER=${CTYPE} 630 fi 631 632 FAILURE=0 633 if [[ ! -z "${TF_BUILD_DRY_RUN}" ]] && [[ ${TF_BUILD_DRY_RUN} != "0" ]]; then 634 # Do a dry run: just print the final command 635 echo "*** This is a DRY RUN ***" 636 else 637 # Actually run the command 638 if [[ "${DO_DOCKER}" == "1" ]]; then 639 ${DOCKER_MAIN_CMD} ${CONTAINER} ${DOCKERFILE_FLAG} /tmp/tf_build.sh 640 else 641 ${TMP_SCRIPT} 642 fi 643 644 if [[ $? != "0" ]]; then 645 FAILURE=1 646 fi 647 fi 648 649 [[ ${FAILURE} == "0" ]] && RESULT="SUCCESS" || RESULT="FAILURE" 650 651 rm -f ${TMP_SCRIPT} 652 653 END_TIME=$(date +'%s') 654 echo "" 655 echo "Parameterized build ends with ${RESULT} at: $(date) "\ 656 "(Elapsed time: $((END_TIME - START_TIME)) s)" 657 658 # Dump disk usage 659 debug_disk_usage 660 661 # Clean up temporary directory if it exists 662 if [[ ! -z "${TMP_DIR}" ]]; then 663 echo "Cleaning up temporary directory: ${TMP_DIR}" 664 rm -rf "${TMP_DIR}" 665 fi 666 667 exit ${FAILURE} 668
