Home | History | Annotate | Download | only in builds
      1 #!/usr/bin/env bash
      2 # Copyright 2016 The TensorFlow Authors. All Rights Reserved.
      3 #
      4 # Licensed under the Apache License, Version 2.0 (the "License");
      5 # you may not use this file except in compliance with the License.
      6 # You may obtain a copy of the License at
      7 #
      8 #     http://www.apache.org/licenses/LICENSE-2.0
      9 #
     10 # Unless required by applicable law or agreed to in writing, software
     11 # distributed under the License is distributed on an "AS IS" BASIS,
     12 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
     13 # See the License for the specific language governing permissions and
     14 # limitations under the License.
     15 # ==============================================================================
     16 #
     17 # Build the Python PIP installation package for TensorFlow and install
     18 # the package.
     19 # The PIP installation is done using the --user flag.
     20 #
     21 # Usage:
     22 #   pip.sh CONTAINER_TYPE [--test_tutorials] [--integration_tests] [bazel flags]
     23 #
     24 # When executing the Python unit tests, the script obeys the shell
     25 # variables: TF_BUILD_BAZEL_CLEAN, TF_BUILD_INSTALL_EXTRA_PIP_PACKAGES,
     26 # NO_TEST_ON_INSTALL, PIP_TEST_ROOT, TF_NIGHTLY
     27 #
     28 # TF_BUILD_BAZEL_CLEAN, if set to any non-empty and non-0 value, directs the
     29 # script to perform bazel clean prior to main build and test steps.
     30 #
     31 # TF_BUILD_INSTALL_EXTRA_PIP_PACKAGES overrides the default extra pip packages
     32 # to be installed in virtualenv before run_pip_tests.sh is called. Multiple
     33 # pakcage names are separated with spaces.
     34 #
     35 # If NO_TEST_ON_INSTALL has any non-empty and non-0 value, the test-on-install
     36 # part will be skipped.
     37 #
     38 # If NO_TEST_USER_OPS has any non-empty and non-0 value, the testing of user-
     39 # defined ops against the installation will be skipped.
     40 #
     41 # If NO_TEST_TFDBG_BINARIES has any non-empty and non-0 value, the testing of
     42 # TensorFlow Debugger (tfdbg) binaries and examples will be skipped.
     43 #
     44 # If PIP_TEST_ROOT has a non-empty and a non-0 value, the whl files will be
     45 # placed in that directory.
     46 #
     47 # If TF_NIGHTLY has a non-empty and a non-0 value, the name of the project will
     48 # be changed to tf_nightly or tf_nightly_gpu.
     49 #
     50 # Any flags not listed in the usage above will be passed directly to Bazel.
     51 #
     52 # If the --test_tutorials flag is set, it will cause the script to run the
     53 # tutorial tests (see test_tutorials.sh) after the PIP
     54 # installation and the Python unit tests-on-install step. Likewise,
     55 # --integration_tests will cause the integration tests (integration_tests.sh)
     56 # to run.
     57 #
     58 
     59 # Helper function: Strip leading and trailing whitespaces
     60 str_strip () {
     61   echo -e "$1" | sed -e 's/^[[:space:]]*//' -e 's/[[:space:]]*$//'
     62 }
     63 
     64 # Fixed naming patterns for wheel (.whl) files given different python versions
     65 if [[ $(uname) == "Linux" ]]; then
     66   declare -A WHL_TAGS
     67   WHL_TAGS=(["2.7"]="cp27-none" ["3.4"]="cp34-cp34m" ["3.5"]="cp35-cp35m")
     68 fi
     69 
     70 
     71 INSTALL_EXTRA_PIP_PACKAGES=${TF_BUILD_INSTALL_EXTRA_PIP_PACKAGES}
     72 
     73 
     74 # Script directory
     75 SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
     76 source "${SCRIPT_DIR}/builds_common.sh"
     77 
     78 
     79 SKIP_RETURN_CODE=112
     80 
     81 
     82 # Get the command line arguments
     83 CONTAINER_TYPE=$( echo "$1" | tr '[:upper:]' '[:lower:]' )
     84 shift
     85 
     86 if [[ -n "${TF_BUILD_BAZEL_CLEAN}" ]] && \
     87    [[ "${TF_BUILD_BAZEL_CLEAN}" != "0" ]]; then
     88   echo "TF_BUILD_BAZEL_CLEAN=${TF_BUILD_BAZEL_CLEAN}: Performing 'bazel clean'"
     89   bazel clean
     90 fi
     91 
     92 DO_TEST_USER_OPS=1
     93 if [[ -n "${NO_TEST_USER_OPS}" ]] && \
     94    [[ "${NO_TEST_USER_OPS}" != "0" ]]; then
     95   echo "NO_TEST_USER_OPS=${NO_TEST_USER_OPS}: Will skip testing of user ops"
     96   DO_TEST_USER_OPS=0
     97 fi
     98 
     99 DO_TEST_TFDBG_BINARIES=1
    100 if [[ -n "${NO_TEST_TFDBG_BINARIES}" ]] && \
    101    [[ "${NO_TEST_TFDBG_BINARIES}" != "0" ]]; then
    102   echo "NO_TEST_TFDBG_BINARIES=${NO_TEST_TFDBG_BINARIES}: Will skip testing of tfdbg binaries"
    103   DO_TEST_TFDBG_BINARIES=0
    104 fi
    105 
    106 DO_TEST_TUTORIALS=0
    107 DO_INTEGRATION_TESTS=0
    108 BAZEL_FLAGS=""
    109 while true; do
    110   if [[ "${1}" == "--test_tutorials" ]]; then
    111     DO_TEST_TUTORIALS=1
    112   elif [[ "${1}" == "--integration_tests" ]]; then
    113     DO_INTEGRATION_TESTS=1
    114   else
    115     BAZEL_FLAGS="${BAZEL_FLAGS} ${1}"
    116   fi
    117 
    118   shift
    119   if [[ -z "${1}" ]]; then
    120     break
    121   fi
    122 done
    123 
    124 BAZEL_FLAGS=$(str_strip "${BAZEL_FLAGS}")
    125 
    126 echo "Using Bazel flags: ${BAZEL_FLAGS}"
    127 
    128 PIP_BUILD_TARGET="//tensorflow/tools/pip_package:build_pip_package"
    129 GPU_FLAG=""
    130 if [[ ${CONTAINER_TYPE} == "cpu" ]] || \
    131    [[ ${CONTAINER_TYPE} == "debian.jessie.cpu" ]]; then
    132   bazel build ${BAZEL_FLAGS} ${PIP_BUILD_TARGET} || \
    133       die "Build failed."
    134 elif [[ ${CONTAINER_TYPE} == "gpu" ]]; then
    135   bazel build ${BAZEL_FLAGS} ${PIP_BUILD_TARGET} || \
    136       die "Build failed."
    137   GPU_FLAG="--gpu"
    138 else
    139   die "Unrecognized container type: \"${CONTAINER_TYPE}\""
    140 fi
    141 
    142 MAC_FLAG=""
    143 if [[ $(uname) == "Darwin" ]]; then
    144   MAC_FLAG="--mac"
    145 fi
    146 
    147 
    148 # Check if in a virtualenv
    149 IN_VENV=$(python -c 'import sys; print("1" if hasattr(sys, "real_prefix") else "0")')
    150 # If still in a virtualenv, deactivate it first
    151 if [[ "$IN_VENV" == "1" ]]; then
    152   echo "It appears that we are already in a virtualenv. Deactivating..."
    153   deactivate || die "FAILED: Unable to deactivate from existing virtualenv"
    154 fi
    155 
    156 # Obtain the path to Python binary
    157 source tools/python_bin_path.sh
    158 
    159 # Assume: PYTHON_BIN_PATH is exported by the script above
    160 if [[ -z "$PYTHON_BIN_PATH" ]]; then
    161   die "PYTHON_BIN_PATH was not provided. Did you run configure?"
    162 fi
    163 
    164 # Determine the major and minor versions of Python being used (e.g., 2.7)
    165 # This info will be useful for determining the directory of the local pip
    166 # installation of Python
    167 PY_MAJOR_MINOR_VER=$(${PYTHON_BIN_PATH} -V 2>&1 | awk '{print $NF}' | cut -d. -f-2)
    168 if [[ -z "${PY_MAJOR_MINOR_VER}" ]]; then
    169   die "ERROR: Unable to determine the major.minor version of Python"
    170 fi
    171 
    172 echo "Python binary path to be used in PIP install: ${PYTHON_BIN_PATH} "\
    173 "(Major.Minor version: ${PY_MAJOR_MINOR_VER})"
    174 
    175 # Create a TF_NIGHTLY argument if this is a nightly build
    176 PROJECT_NAME="tensorflow"
    177 NIGHTLY_FLAG=""
    178 if [ -n "$TF_NIGHTLY" ]; then
    179   PROJECT_NAME="tf_nightly"
    180   NIGHTLY_FLAG="--nightly_flag"
    181 fi
    182 
    183 # Build PIP Wheel file
    184 # Set default pip file folder unless specified by env variable
    185 if [ -z "$PIP_TEST_ROOT" ]; then
    186   PIP_TEST_ROOT="pip_test"
    187 fi
    188 PIP_WHL_DIR="${PIP_TEST_ROOT}/whl"
    189 PIP_WHL_DIR=$(realpath ${PIP_WHL_DIR})  # Get absolute path
    190 rm -rf ${PIP_WHL_DIR} && mkdir -p ${PIP_WHL_DIR}
    191 bazel-bin/tensorflow/tools/pip_package/build_pip_package ${PIP_WHL_DIR} ${GPU_FLAG} ${NIGHTLY_FLAG} || \
    192     die "build_pip_package FAILED"
    193 
    194 WHL_PATH=$(ls ${PIP_WHL_DIR}/${PROJECT_NAME}*.whl)
    195 if [[ $(echo ${WHL_PATH} | wc -w) -ne 1 ]]; then
    196   die "ERROR: Failed to find exactly one built TensorFlow .whl file in "\
    197 "directory: ${PIP_WHL_DIR}"
    198 fi
    199 
    200 # Print the size of the PIP wheel file.
    201 echo
    202 echo "Size of the PIP wheel file built: $(ls -l ${WHL_PATH} | awk '{print $5}')"
    203 echo
    204 
    205 # Rename the whl file properly so it will have the python
    206 # version tags and platform tags that won't cause pip install issues.
    207 if [[ $(uname) == "Linux" ]]; then
    208   PY_TAGS=${WHL_TAGS[${PY_MAJOR_MINOR_VER}]}
    209   PLATFORM_TAG=$(to_lower "$(uname)_$(uname -m)")
    210 # MAC has bash v3, which does not have associative array
    211 elif [[ $(uname) == "Darwin" ]]; then
    212   if [[ ${PY_MAJOR_MINOR_VER} == "2.7" ]]; then
    213     PY_TAGS="py2-none"
    214   elif [[ ${PY_MAJOR_MINOR_VER} == "3.5" ]]; then
    215     PY_TAGS="py3-none"
    216   elif [[ ${PY_MAJOR_MINOR_VER} == "3.6" ]]; then
    217     PY_TAGS="py3-none"
    218   fi
    219   PLATFORM_TAG="any"
    220 fi
    221 
    222 WHL_DIR=$(dirname "${WHL_PATH}")
    223 WHL_BASE_NAME=$(basename "${WHL_PATH}")
    224 
    225 if [[ -n "${PY_TAGS}" ]]; then
    226   NEW_WHL_BASE_NAME=$(echo ${WHL_BASE_NAME} | cut -d \- -f 1)-\
    227 $(echo ${WHL_BASE_NAME} | cut -d \- -f 2)-${PY_TAGS}-${PLATFORM_TAG}.whl
    228 
    229   if [[ ! -f "${WHL_DIR}/${NEW_WHL_BASE_NAME}" ]]; then
    230     if cp "${WHL_DIR}/${WHL_BASE_NAME}" "${WHL_DIR}/${NEW_WHL_BASE_NAME}"
    231     then
    232       echo "Copied wheel file: ${WHL_BASE_NAME} --> ${NEW_WHL_BASE_NAME}"
    233     else
    234       die "ERROR: Failed to copy wheel file to ${NEW_WHL_BASE_NAME}"
    235     fi
    236   fi
    237 fi
    238 
    239 if [[ $(uname) == "Linux" ]]; then
    240   AUDITED_WHL_NAME="${WHL_DIR}/$(echo ${WHL_BASE_NAME//linux/manylinux1})"
    241 
    242   # Repair the wheels for cpu manylinux1
    243   if [[ ${CONTAINER_TYPE} == "cpu" ]]; then
    244     echo "auditwheel repairing ${WHL_PATH}"
    245     auditwheel repair -w ${WHL_DIR} ${WHL_PATH}
    246 
    247     if [[ -f ${AUDITED_WHL_NAME} ]]; then
    248       WHL_PATH=${AUDITED_WHL_NAME}
    249       echo "Repaired manylinx1 wheel file at: ${WHL_PATH}"
    250     else
    251       die "ERROR: Cannot find repaired wheel."
    252     fi
    253   # Copy and rename for gpu manylinux as we do not want auditwheel to package in libcudart.so
    254   elif [[ ${CONTAINER_TYPE} == "gpu" ]]; then
    255     WHL_PATH=${AUDITED_WHL_NAME}
    256     cp ${WHL_DIR}/${WHL_BASE_NAME} ${WHL_PATH}
    257     echo "Copied manylinx1 wheel file at ${WHL_PATH}"
    258   fi
    259 fi
    260 
    261 
    262 create_activate_virtualenv_and_install_tensorflow() {
    263   # Create and activate a virtualenv; then install tensorflow pip package in it.
    264   #
    265   # Usage:
    266   #   create_activate_virtualenv_and_install_tensorflow [--clean] \
    267   #       <VIRTUALENV_DIR> <TF_WHEEL_PATH>
    268   #
    269   # Arguments:
    270   #   --clean: Create a clean virtualenv, i.e., without --system-site-packages.
    271   #   VIRTUALENV_DIR: virtualenv directory to be created.
    272   #   TF_WHEEL_PATH: Path to the tensorflow wheel file to be installed in the
    273   #     virtualenv.
    274 
    275   VIRTUALENV_FLAGS="--system-site-packages"
    276   if [[ "$1" == "--clean" ]]; then
    277     VIRTUALENV_FLAGS=""
    278     shift
    279   fi
    280 
    281   VIRTUALENV_DIR="$1"
    282   TF_WHEEL_PATH="$2"
    283   if [[ -d "${VIRTUALENV_DIR}" ]]; then
    284     if rm -rf "${VIRTUALENV_DIR}"
    285     then
    286       echo "Removed existing virtualenv directory: ${VIRTUALENV_DIR}"
    287     else
    288       die "Failed to remove existing virtualenv directory: ${VIRTUALENV_DIR}"
    289     fi
    290   fi
    291 
    292   if mkdir -p "${VIRTUALENV_DIR}"
    293   then
    294     echo "Created virtualenv directory: ${VIRTUALENV_DIR}"
    295   else
    296     die "FAILED to create virtualenv directory: ${VIRTUALENV_DIR}"
    297   fi
    298 
    299   # Use the virtualenv from the default python version (i.e., python-virtualenv)
    300   # to create the virtualenv directory for testing. Use the -p flag to specify
    301   # the python version inside the to-be-created virtualenv directory.
    302   ${PYTHON_BIN_PATH} -m virtualenv -p "${PYTHON_BIN_PATH}" ${VIRTUALENV_FLAGS} \
    303     "${VIRTUALENV_DIR}" || \
    304     die "FAILED: Unable to create virtualenv"
    305 
    306   source "${VIRTUALENV_DIR}/bin/activate" || \
    307     die "FAILED: Unable to activate virtualenv in ${VIRTUALENV_DIR}"
    308 
    309   # Install the pip file in virtual env.
    310 
    311   # Upgrade pip so it supports tags such as cp27mu, manylinux1 etc.
    312   echo "Upgrade pip in virtualenv"
    313   pip install --upgrade pip==9.0.1
    314 
    315   # Force tensorflow reinstallation. Otherwise it may not get installed from
    316   # last build if it had the same version number as previous build.
    317   PIP_FLAGS="--upgrade --force-reinstall"
    318   pip install -v ${PIP_FLAGS} ${WHL_PATH} || \
    319     die "pip install (forcing to reinstall tensorflow) FAILED"
    320   echo "Successfully installed pip package ${TF_WHEEL_PATH}"
    321 }
    322 
    323 ################################################################################
    324 # Smoke test of tensorflow install in clean virtualenv
    325 ################################################################################
    326 do_clean_virtualenv_smoke_test() {
    327   if [[ -n "${NO_TEST_ON_INSTALL}" ]] &&
    328        [[ "${NO_TEST_ON_INSTALL}" != "0" ]]; then
    329     echo "NO_TEST_ON_INSTALL=${NO_TEST_ON_INSTALL}:"
    330     echo "  Skipping smoke test of tensorflow install in clean virtualenv"
    331     return ${SKIP_RETURN_CODE}
    332   fi
    333 
    334   CLEAN_VENV_DIR="${PIP_TEST_ROOT}/venv_clean"
    335   create_activate_virtualenv_and_install_tensorflow --clean \
    336     "${CLEAN_VENV_DIR}" "${WHL_PATH}"
    337 
    338   # cd to a temporary directory to avoid picking up Python files in the source
    339   # tree.
    340   TMP_DIR=$(mktemp -d)
    341   pushd "${TMP_DIR}"
    342   if [[ $(python -c "import tensorflow as tf; print(tf.Session().run(tf.constant(42)))") == 42 ]];
    343   then
    344     echo "Smoke test of tensorflow install in clean virtualenv PASSED."
    345   else
    346     echo "Smoke test of tensorflow install in clean virtualenv FAILED."
    347     return 1
    348   fi
    349 
    350   deactivate
    351   if [[ $? != 0 ]]; then
    352     echo "FAILED: Unable to deactivate virtualenv from ${CLEAN_VENV_DIR}"
    353     return 1
    354   fi
    355 
    356   popd
    357   rm -rf "${TMP_DIR}" "${CLEAN_VENV_DIR}"
    358 }
    359 
    360 ################################################################################
    361 # Perform installation of tensorflow in "non-clean" virtualenv and tests against
    362 # the install.
    363 ################################################################################
    364 do_virtualenv_pip_test() {
    365   # Create virtualenv directory for install test
    366   VENV_DIR="${PIP_TEST_ROOT}/venv"
    367   create_activate_virtualenv_and_install_tensorflow \
    368     "${VENV_DIR}" "${WHL_PATH}"
    369 
    370   # Install extra pip packages required by the test-on-install
    371   for PACKAGE in ${INSTALL_EXTRA_PIP_PACKAGES}; do
    372     echo "Installing extra pip package required by test-on-install: ${PACKAGE}"
    373 
    374     pip install ${PACKAGE}
    375     if [[ $? != 0 ]]; then
    376       echo "pip install ${PACKAGE} FAILED"
    377       return 1
    378     fi
    379   done
    380 
    381   if [[ -n "${NO_TEST_ON_INSTALL}" ]] &&
    382      [[ "${NO_TEST_ON_INSTALL}" != "0" ]]; then
    383     echo "NO_TEST_ON_INSTALL=${NO_TEST_ON_INSTALL}:"
    384     echo "  Skipping ALL Python unit tests on install"
    385     return ${SKIP_RETURN_CODE}
    386   else
    387     # Call run_pip_tests.sh to perform test-on-install
    388     "${SCRIPT_DIR}/run_pip_tests.sh" --virtualenv ${GPU_FLAG} ${MAC_FLAG}
    389     if [[ $? != 0 ]]; then
    390       echo "PIP tests-on-install FAILED"
    391       return 1
    392     fi
    393   fi
    394 }
    395 
    396 ################################################################################
    397 # Run tests tagged with oss_serial against the virtualenv install.
    398 ################################################################################
    399 do_virtualenv_oss_serial_pip_test() {
    400   if [[ -n "${NO_TEST_ON_INSTALL}" ]] &&
    401      [[ "${NO_TEST_ON_INSTALL}" != "0" ]]; then
    402     echo "NO_TEST_ON_INSTALL=${NO_TEST_ON_INSTALL}:"
    403     echo "  Skipping Python unit tests on install tagged with oss_serial"
    404     return ${SKIP_RETURN_CODE}
    405   else
    406     # Call run_pip_tests.sh to perform test-on-install
    407     "${SCRIPT_DIR}/run_pip_tests.sh" \
    408       --virtualenv ${GPU_FLAG} ${MAC_FLAG} --oss_serial
    409     if [[ $? != 0 ]]; then
    410       echo "PIP tests-on-install (oss_serial) FAILED"
    411       return 1
    412     fi
    413   fi
    414 }
    415 
    416 ################################################################################
    417 # Test user ops (optional).
    418 ################################################################################
    419 do_test_user_ops() {
    420   if [[ "${DO_TEST_USER_OPS}" == "1" ]]; then
    421     "${SCRIPT_DIR}/test_user_ops.sh" --virtualenv ${GPU_FLAG}
    422     if [[ $? != 0 ]]; then
    423       echo "PIP user-op tests-on-install FAILED"
    424       return 1
    425     fi
    426   else
    427     echo "Skipping user-op test-on-install due to DO_TEST_USER_OPS = ${DO_TEST_USER_OPS}"
    428     return ${SKIP_RETURN_CODE}
    429   fi
    430 }
    431 
    432 ################################################################################
    433 # Test TensorFlow Debugger (tfdbg) binaries (optional).
    434 ################################################################################
    435 do_test_tfdbg_binaries() {
    436   if [[ "${DO_TEST_TFDBG_BINARIES}" == "1" ]]; then
    437     # cd to a temporary directory to avoid picking up Python files in the source
    438     # tree.
    439     TMP_DIR=$(mktemp -d)
    440     pushd "${TMP_DIR}"
    441 
    442     "${SCRIPT_DIR}/../../../python/debug/examples/examples_test.sh" \
    443       --virtualenv
    444     if  [[ $? != 0 ]]; then
    445       echo "PIP tests-on-install of tfdbg binaries FAILED"
    446       return 1
    447     fi
    448     popd
    449   else
    450     echo "Skipping test of tfdbg binaries due to DO_TEST_TFDBG_BINARIES = ${DO_TEST_TFDBG_BINARIES}"
    451     return ${SKIP_RETURN_CODE}
    452   fi
    453 }
    454 
    455 ################################################################################
    456 # Test tutorials (optional).
    457 ################################################################################
    458 do_test_tutorials() {
    459   if [[ "${DO_TEST_TUTORIALS}" == "1" ]]; then
    460     "${SCRIPT_DIR}/test_tutorials.sh" --virtualenv
    461     if [[ $? != 0 ]]; then
    462       echo "PIP tutorial tests-on-install FAILED"
    463       return 1
    464     fi
    465   else
    466     echo "Skipping tutorial tests-on-install due to DO_TEST_TUTORIALS = ${DO_TEST_TUTORIALS}"
    467     return ${SKIP_RETURN_CODE}
    468   fi
    469 }
    470 
    471 ################################################################################
    472 # Integration test for ffmpeg (optional).
    473 ################################################################################
    474 do_ffmpeg_integration_test() {
    475   # Optional: Run integration tests
    476   if [[ "${DO_INTEGRATION_TESTS}" == "1" ]]; then
    477     "${SCRIPT_DIR}/integration_tests.sh" --virtualenv
    478     if [[ $? != 0 ]]; then
    479       echo "Integration tests on install FAILED"
    480       return 1
    481     fi
    482   else
    483     echo "Skipping ffmpeg integration due to DO_INTEGRATION_TESTS = ${DO_INTEGRATION_TESTS}"
    484     return ${SKIP_RETURN_CODE}
    485   fi
    486 }
    487 
    488 
    489 # List of all PIP test tasks and their descriptions.
    490 PIP_TASKS=("do_clean_virtualenv_smoke_test" "do_virtualenv_pip_test" "do_virtualenv_oss_serial_pip_test" "do_test_user_ops" "do_test_tfdbg_binaries" "do_test_tutorials" "do_ffmpeg_integration_test")
    491 PIP_TASKS_DESC=("Smoke test of pip install in clean virtualenv" "PIP tests in virtualenv" "PIP test in virtualenv (tag: oss_serial)" "User ops test" "TensorFlow Debugger (tfdbg) binaries test" "Tutorials test" "ffmpeg integration test")
    492 
    493 
    494 # Execute all the PIP test steps.
    495 COUNTER=0
    496 FAIL_COUNTER=0
    497 PASS_COUNTER=0
    498 SKIP_COUNTER=0
    499 while [[ ${COUNTER} -lt "${#PIP_TASKS[@]}" ]]; do
    500   INDEX=COUNTER
    501   ((INDEX++))
    502 
    503   echo
    504   printf "${COLOR_BOLD}=== PIP test step ${INDEX} of ${#PIP_TASKS[@]}: "\
    505 "${PIP_TASKS[COUNTER]} (${PIP_TASKS_DESC[COUNTER]}) ===${COLOR_NC}"
    506   echo
    507 
    508   ${PIP_TASKS[COUNTER]}
    509   RESULT=$?
    510 
    511   if [[ ${RESULT} == ${SKIP_RETURN_CODE} ]]; then
    512     ((SKIP_COUNTER++))
    513   elif [[ ${RESULT} != "0" ]]; then
    514     ((FAIL_COUNTER++))
    515   else
    516     ((PASS_COUNTER++))
    517   fi
    518 
    519   STEP_EXIT_CODES+=(${RESULT})
    520 
    521   echo ""
    522   ((COUNTER++))
    523 done
    524 
    525 deactivate || die "FAILED: Unable to deactivate virtualenv from ${VENV_DIR}"
    526 
    527 
    528 # Print summary of build results
    529 COUNTER=0
    530 echo "==== Summary of PIP test results ===="
    531 while [[ ${COUNTER} -lt "${#PIP_TASKS[@]}" ]]; do
    532   INDEX=COUNTER
    533   ((INDEX++))
    534 
    535   echo "${INDEX}. ${PIP_TASKS[COUNTER]}: ${PIP_TASKS_DESC[COUNTER]}"
    536   if [[ ${STEP_EXIT_CODES[COUNTER]} == ${SKIP_RETURN_CODE} ]]; then
    537     printf "  ${COLOR_LIGHT_GRAY}SKIP${COLOR_NC}\n"
    538   elif [[ ${STEP_EXIT_CODES[COUNTER]} == "0" ]]; then
    539     printf "  ${COLOR_GREEN}PASS${COLOR_NC}\n"
    540   else
    541     printf "  ${COLOR_RED}FAIL${COLOR_NC}\n"
    542   fi
    543 
    544   ((COUNTER++))
    545 done
    546 
    547 echo
    548 echo "${SKIP_COUNTER} skipped; ${FAIL_COUNTER} failed; ${PASS_COUNTER} passed."
    549 
    550 echo
    551 if [[ ${FAIL_COUNTER} == "0" ]]; then
    552   printf "PIP test ${COLOR_GREEN}PASSED${COLOR_NC}\n"
    553 else
    554   printf "PIP test ${COLOR_RED}FAILED${COLOR_NC}\n"
    555   exit 1
    556 fi
    557