1 #!/usr/bin/env bash 2 # Copyright 2017 The TensorFlow Authors. All Rights Reserved. 3 # 4 # Licensed under the Apache License, Version 2.0 (the "License"); 5 # you may not use this file except in compliance with the License. 6 # You may obtain a copy of the License at 7 # 8 # http://www.apache.org/licenses/LICENSE-2.0 9 # 10 # Unless required by applicable law or agreed to in writing, software 11 # distributed under the License is distributed on an "AS IS" BASIS, 12 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 # See the License for the specific language governing permissions and 14 # limitations under the License. 15 # ============================================================================== 16 # 17 # This script is intended to be run inside a docker container to provide a 18 # hermetic process. See release.sh for the expected invocation. 19 20 21 RELEASE_URL_PREFIX="https://storage.googleapis.com/tensorflow/libtensorflow" 22 23 # By default we deploy to both ossrh and bintray. These two 24 # environment variables can be set to skip either repository. 25 DEPLOY_BINTRAY="${DEPLOY_BINTRAY:-true}" 26 DEPLOY_OSSRH="${DEPLOY_OSSRH:-true}" 27 28 IS_SNAPSHOT="false" 29 if [[ "${TF_VERSION}" == *"-SNAPSHOT" ]]; then 30 IS_SNAPSHOT="true" 31 # Bintray does not allow snapshots. 32 DEPLOY_BINTRAY="false" 33 fi 34 PROTOC_RELEASE_URL="https://github.com/google/protobuf/releases/download/v3.3.0/protoc-3.3.0-linux-x86_64.zip" 35 if [[ "${DEPLOY_BINTRAY}" != "true" && "${DEPLOY_OSSRH}" != "true" ]]; then 36 echo "Must deploy to at least one of Bintray or OSSRH" >&2 37 exit 2 38 fi 39 40 set -ex 41 42 clean() { 43 # Clean up any existing artifacts 44 # (though if run inside a clean docker container, there won't be any dirty 45 # artifacts lying around) 46 mvn -q clean 47 rm -rf libtensorflow_jni/src libtensorflow_jni/target libtensorflow_jni_gpu/src libtensorflow_jni_gpu/target libtensorflow/src libtensorflow/target tensorflow-android/target 48 } 49 50 update_version_in_pom() { 51 mvn versions:set -DnewVersion="${TF_VERSION}" 52 } 53 54 # Fetch a property from pom files for a given profile. 55 # Arguments: 56 # profile - name of the selected profile. 57 # property - name of the property to be retrieved. 58 # Output: 59 # Echo property value to stdout 60 mvn_property() { 61 local profile="$1" 62 local prop="$2" 63 mvn -q --non-recursive exec:exec -P "${profile}" \ 64 -Dexec.executable='echo' \ 65 -Dexec.args="\${${prop}}" 66 } 67 68 download_libtensorflow() { 69 if [[ "${IS_SNAPSHOT}" == "true" ]]; then 70 URL="http://ci.tensorflow.org/view/Nightly/job/nightly-libtensorflow/TYPE=cpu-slave/lastSuccessfulBuild/artifact/lib_package/libtensorflow-src.jar" 71 else 72 URL="${RELEASE_URL_PREFIX}/libtensorflow-src-${TF_VERSION}.jar" 73 fi 74 curl -L "${URL}" -o /tmp/src.jar 75 cd "${DIR}/libtensorflow" 76 jar -xvf /tmp/src.jar 77 rm -rf META-INF 78 cd "${DIR}" 79 } 80 81 # Fetch the android aar artifact from the CI build system, and update 82 # its associated pom file. 83 update_tensorflow_android() { 84 TARGET_DIR="${DIR}/tensorflow-android/target" 85 mkdir -p "${TARGET_DIR}" 86 python "${DIR}/tensorflow-android/update.py" \ 87 --version "${TF_VERSION}" \ 88 --template "${DIR}/tensorflow-android/pom-android.xml.template" \ 89 --dir "${TARGET_DIR}" 90 } 91 92 download_libtensorflow_jni() { 93 NATIVE_DIR="${DIR}/libtensorflow_jni/src/main/resources/org/tensorflow/native" 94 mkdir -p "${NATIVE_DIR}" 95 cd "${NATIVE_DIR}" 96 97 mkdir linux-x86_64 98 mkdir windows-x86_64 99 mkdir darwin-x86_64 100 101 if [[ "${IS_SNAPSHOT}" == "true" ]]; then 102 # Nightly builds from http://ci.tensorflow.org/view/Nightly/job/nightly-libtensorflow/ 103 # and http://ci.tensorflow.org/view/Nightly/job/nightly-libtensorflow-windows/ 104 curl -L "http://ci.tensorflow.org/view/Nightly/job/nightly-libtensorflow/TYPE=cpu-slave/lastSuccessfulBuild/artifact/lib_package/libtensorflow_jni-cpu-linux-x86_64.tar.gz" | tar -xvz -C linux-x86_64 105 curl -L "http://ci.tensorflow.org/view/Nightly/job/nightly-libtensorflow/TYPE=mac-slave/lastSuccessfulBuild/artifact/lib_package/libtensorflow_jni-cpu-darwin-x86_64.tar.gz" | tar -xvz -C darwin-x86_64 106 curl -L "http://ci.tensorflow.org/view/Nightly/job/nightly-libtensorflow-windows/lastSuccessfulBuild/artifact/lib_package/libtensorflow_jni-cpu-windows-x86_64.zip" -o /tmp/windows.zip 107 else 108 curl -L "${RELEASE_URL_PREFIX}/libtensorflow_jni-cpu-linux-x86_64-${TF_VERSION}.tar.gz" | tar -xvz -C linux-x86_64 109 curl -L "${RELEASE_URL_PREFIX}/libtensorflow_jni-cpu-darwin-x86_64-${TF_VERSION}.tar.gz" | tar -xvz -C darwin-x86_64 110 curl -L "${RELEASE_URL_PREFIX}/libtensorflow_jni-cpu-windows-x86_64-${TF_VERSION}.zip" -o /tmp/windows.zip 111 fi 112 113 unzip /tmp/windows.zip -d windows-x86_64 114 rm -f /tmp/windows.zip 115 # Updated timestamps seem to be required to get Maven to pick up the file. 116 touch linux-x86_64/* 117 touch darwin-x86_64/* 118 touch windows-x86_64/* 119 cd "${DIR}" 120 } 121 122 download_libtensorflow_jni_gpu() { 123 NATIVE_DIR="${DIR}/libtensorflow_jni_gpu/src/main/resources/org/tensorflow/native" 124 mkdir -p "${NATIVE_DIR}" 125 cd "${NATIVE_DIR}" 126 127 mkdir linux-x86_64 128 129 if [[ "${IS_SNAPSHOT}" == "true" ]]; then 130 # Nightly builds from http://ci.tensorflow.org/view/Nightly/job/nightly-libtensorflow/ 131 # and http://ci.tensorflow.org/view/Nightly/job/nightly-libtensorflow-windows/ 132 curl -L "http://ci.tensorflow.org/view/Nightly/job/nightly-libtensorflow/TYPE=gpu-linux/lastSuccessfulBuild/artifact/lib_package/libtensorflow_jni-gpu-linux-x86_64.tar.gz" | tar -xvz -C linux-x86_64 133 else 134 curl -L "${RELEASE_URL_PREFIX}/libtensorflow_jni-gpu-linux-x86_64-${TF_VERSION}.tar.gz" | tar -xvz -C linux-x86_64 135 fi 136 137 # Updated timestamps seem to be required to get Maven to pick up the file. 138 touch linux-x86_64/* 139 cd "${DIR}" 140 } 141 142 # Ideally, the .jar for generated Java code for TensorFlow protocol buffer files 143 # would have been produced by bazel rules. However, protocol buffer library 144 # support in bazel is in flux. Once 145 # https://github.com/bazelbuild/bazel/issues/2626 has been resolved, perhaps 146 # TensorFlow can move to something like 147 # https://bazel.build/blog/2017/02/27/protocol-buffers.html 148 # for generating C++, Java and Python code for protocol buffers. 149 # 150 # At that point, perhaps the libtensorflow build scripts 151 # (tensorflow/tools/ci_build/builds/libtensorflow.sh) can build .jars for 152 # generated code and this function would not need to download protoc to generate 153 # code. 154 generate_java_protos() { 155 # Clean any previous attempts 156 rm -rf "${DIR}/proto/tmp" 157 158 # Download protoc 159 curl -L "${PROTOC_RELEASE_URL}" -o "/tmp/protoc.zip" 160 mkdir -p "${DIR}/proto/tmp/protoc" 161 unzip -d "${DIR}/proto/tmp/protoc" "/tmp/protoc.zip" 162 rm -f "/tmp/protoc.zip" 163 164 # Download the release archive of TensorFlow protos. 165 if [[ "${IS_SNAPSHOT}" == "true" ]]; then 166 URL="http://ci.tensorflow.org/view/Nightly/job/nightly-libtensorflow/TYPE=cpu-slave/lastSuccessfulBuild/artifact/lib_package/libtensorflow_proto.zip" 167 else 168 URL="${RELEASE_URL_PREFIX}/libtensorflow_proto-${TF_VERSION}.zip" 169 fi 170 curl -L "${URL}" -o /tmp/libtensorflow_proto.zip 171 mkdir -p "${DIR}/proto/tmp/src" 172 unzip -d "${DIR}/proto/tmp/src" "/tmp/libtensorflow_proto.zip" 173 rm -f "/tmp/libtensorflow_proto.zip" 174 175 # Generate Java code 176 mkdir -p "${DIR}/proto/src/main/java" 177 find "${DIR}/proto/tmp/src" -name "*.proto" | xargs \ 178 ${DIR}/proto/tmp/protoc/bin/protoc \ 179 --proto_path="${DIR}/proto/tmp/src" \ 180 --java_out="${DIR}/proto/src/main/java" 181 182 # Cleanup 183 rm -rf "${DIR}/proto/tmp" 184 } 185 186 # Deploy artifacts using a specific profile. 187 # Arguments: 188 # profile - name of selected profile. 189 # Outputs: 190 # n/a 191 deploy_profile() { 192 local profile="$1" 193 # Deploy the non-android pieces. 194 mvn deploy -P"${profile}" 195 # Determine the correct pom file property to use 196 # for the repository url. 197 local rtype 198 if [[ "${IS_SNAPSHOT}" == "true" ]]; then 199 rtype='snapshotRepository' 200 else 201 rtype='repository' 202 fi 203 local url=$(mvn_property "${profile}" "project.distributionManagement.${rtype}.url") 204 local repositoryId=$(mvn_property "${profile}" "project.distributionManagement.${rtype}.id") 205 mvn gpg:sign-and-deploy-file \ 206 -Dfile="${DIR}/tensorflow-android/target/tensorflow.aar" \ 207 -DpomFile="${DIR}/tensorflow-android/target/pom-android.xml" \ 208 -Durl="${url}" \ 209 -DrepositoryId="${repositoryId}" 210 } 211 212 # If successfully built, try to deploy. 213 # If successfully deployed, clean. 214 # If deployment fails, debug with 215 # ./release.sh ${TF_VERSION} ${SETTINGS_XML} bash 216 # To get a shell to poke around the maven artifacts with. 217 deploy_artifacts() { 218 # Deploy artifacts to ossrh if requested. 219 if [[ "${DEPLOY_OSSRH}" == "true" ]]; then 220 deploy_profile 'ossrh' 221 fi 222 # Deploy artifacts to bintray if requested. 223 if [[ "${DEPLOY_BINTRAY}" == "true" ]]; then 224 deploy_profile 'bintray' 225 fi 226 # Clean up when everything works 227 clean 228 } 229 230 if [ -z "${TF_VERSION}" ] 231 then 232 echo "Must set the TF_VERSION environment variable" 233 exit 1 234 fi 235 236 DIR="$(realpath $(dirname $0))" 237 cd "${DIR}" 238 239 # The meat of the script. 240 # Comment lines out appropriately if debugging/tinkering with the release 241 # process. 242 # gnupg2 is required for signing 243 apt-get -qq update && apt-get -qqq install -y gnupg2 244 clean 245 update_version_in_pom 246 download_libtensorflow 247 download_libtensorflow_jni 248 download_libtensorflow_jni_gpu 249 update_tensorflow_android 250 generate_java_protos 251 # Build the release artifacts 252 mvn verify 253 # Push artifacts to repository 254 deploy_artifacts 255 256 set +ex 257 if [[ "${IS_SNAPSHOT}" == "false" ]]; then 258 echo "Uploaded to the staging repository" 259 echo "After validating the release: " 260 if [[ "${DEPLOY_OSSRH}" == "true" ]]; then 261 echo "* Login to https://oss.sonatype.org/#stagingRepositories" 262 echo "* Find the 'org.tensorflow' staging release and click either 'Release' to release or 'Drop' to abort" 263 fi 264 if [[ "${DEPLOY_BINTRAY}" == "true" ]]; then 265 echo "* Login to https://bintray.com/google/tensorflow/tensorflow" 266 echo "* Either 'Publish' unpublished items to release, or 'Discard' to abort" 267 fi 268 else 269 echo "Uploaded to the snapshot repository" 270 fi 271
