1 #!/bin/bash 2 # 3 # Build and runs tests for the protobuf project. The tests as written here are 4 # used by both Jenkins and Travis, though some specialized logic is required to 5 # handle the differences between them. 6 7 on_travis() { 8 if [ "$TRAVIS" == "true" ]; then 9 "$@" 10 fi 11 } 12 13 # For when some other test needs the C++ main build, including protoc and 14 # libprotobuf. 15 internal_build_cpp() { 16 if [ -f src/protoc ]; then 17 # Already built. 18 return 19 fi 20 21 if [[ $(uname -s) == "Linux" && "$TRAVIS" == "true" ]]; then 22 # Install GCC 4.8 to replace the default GCC 4.6. We need 4.8 for more 23 # decent C++ 11 support in order to compile conformance tests. 24 sudo add-apt-repository ppa:ubuntu-toolchain-r/test -y 25 sudo apt-get update -qq 26 sudo apt-get install -qq g++-4.8 27 export CXX="g++-4.8" CC="gcc-4.8" 28 fi 29 30 ./autogen.sh 31 ./configure 32 make -j2 33 } 34 35 build_cpp() { 36 internal_build_cpp 37 make check -j2 38 cd conformance && make test_cpp && cd .. 39 40 # Verify benchmarking code can build successfully. 41 cd benchmarks && make && ./generate-datasets && cd .. 42 } 43 44 build_cpp_distcheck() { 45 ./autogen.sh 46 ./configure 47 make dist 48 49 # List all files that should be included in the distribution package. 50 git ls-files | grep "^\(java\|python\|objectivec\|csharp\|js\|ruby\|cmake\|examples\)" |\ 51 grep -v ".gitignore" | grep -v "java/compatibility_tests" > dist.lst 52 # Unzip the dist tar file. 53 DIST=`ls *.tar.gz` 54 tar -xf $DIST 55 cd ${DIST//.tar.gz} 56 # Check if every file exists in the dist tar file. 57 FILES_MISSING="" 58 for FILE in $(<../dist.lst); do 59 if ! file $FILE &>/dev/null; then 60 echo "$FILE is not found!" 61 FILES_MISSING="$FILE $FILES_MISSING" 62 fi 63 done 64 cd .. 65 if [ ! -z "$FILES_MISSING" ]; then 66 echo "Missing files in EXTRA_DIST: $FILES_MISSING" 67 exit 1 68 fi 69 70 # Do the regular dist-check for C++. 71 make distcheck -j2 72 } 73 74 build_csharp() { 75 # Just for the conformance tests. We don't currently 76 # need to really build protoc, but it's simplest to keep with the 77 # conventions of the other builds. 78 internal_build_cpp 79 NUGET=/usr/local/bin/nuget.exe 80 81 if [ "$TRAVIS" == "true" ]; then 82 # Install latest version of Mono 83 sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 3FA7E0328081BFF6A14DA29AA6A19B38D3D831EF 84 sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 1397BC53640DB551 85 echo "deb http://download.mono-project.com/repo/debian wheezy main" | sudo tee /etc/apt/sources.list.d/mono-xamarin.list 86 sudo apt-get update -qq 87 sudo apt-get install -qq mono-devel referenceassemblies-pcl nunit 88 89 # Then install the dotnet SDK as per Ubuntu 14.04 instructions on dot.net. 90 sudo sh -c 'echo "deb [arch=amd64] https://apt-mo.trafficmanager.net/repos/dotnet-release/ trusty main" > /etc/apt/sources.list.d/dotnetdev.list' 91 sudo apt-key adv --keyserver apt-mo.trafficmanager.net --recv-keys 417A0893 92 sudo apt-get update -qq 93 sudo apt-get install -qq dotnet-dev-1.0.0-preview2-003121 94 fi 95 96 # Perform "dotnet new" once to get the setup preprocessing out of the 97 # way. That spews a lot of output (including backspaces) into logs 98 # otherwise, and can cause problems. It doesn't matter if this step 99 # is performed multiple times; it's cheap after the first time anyway. 100 mkdir dotnettmp 101 (cd dotnettmp; dotnet new > /dev/null) 102 rm -rf dotnettmp 103 104 (cd csharp/src; dotnet restore) 105 csharp/buildall.sh 106 cd conformance && make test_csharp && cd .. 107 } 108 109 build_golang() { 110 # Go build needs `protoc`. 111 internal_build_cpp 112 # Add protoc to the path so that the examples build finds it. 113 export PATH="`pwd`/src:$PATH" 114 115 # Install Go and the Go protobuf compiler plugin. 116 on_travis sudo apt-get update -qq 117 on_travis sudo apt-get install -qq golang 118 119 export GOPATH="$HOME/gocode" 120 mkdir -p "$GOPATH/src/github.com/google" 121 rm -f "$GOPATH/src/github.com/google/protobuf" 122 ln -s "`pwd`" "$GOPATH/src/github.com/google/protobuf" 123 export PATH="$GOPATH/bin:$PATH" 124 go get github.com/golang/protobuf/protoc-gen-go 125 126 cd examples && make gotest && cd .. 127 } 128 129 use_java() { 130 version=$1 131 case "$version" in 132 jdk7) 133 on_travis sudo apt-get install openjdk-7-jdk 134 export PATH=/usr/lib/jvm/java-7-openjdk-amd64/bin:$PATH 135 export JAVA_HOME=/usr/lib/jvm/java-7-openjdk-amd64 136 ;; 137 oracle7) 138 if [ "$TRAVIS" == "true" ]; then 139 sudo apt-get install python-software-properties # for apt-add-repository 140 echo "oracle-java7-installer shared/accepted-oracle-license-v1-1 select true" | \ 141 sudo debconf-set-selections 142 yes | sudo apt-add-repository ppa:webupd8team/java 143 yes | sudo apt-get install oracle-java7-installer 144 fi; 145 export PATH=/usr/lib/jvm/java-7-oracle/bin:$PATH 146 export JAVA_HOME=/usr/lib/jvm/java-7-openjdk-amd64 147 ;; 148 esac 149 150 if [ "$TRAVIS" != "true" ]; then 151 MAVEN_LOCAL_REPOSITORY=/var/maven_local_repository 152 MVN="$MVN -e -X --offline -Dmaven.repo.local=$MAVEN_LOCAL_REPOSITORY" 153 fi; 154 155 which java 156 java -version 157 $MVN -version 158 } 159 160 # --batch-mode supresses download progress output that spams the logs. 161 MVN="mvn --batch-mode" 162 163 build_java() { 164 version=$1 165 dir=java_$version 166 # Java build needs `protoc`. 167 internal_build_cpp 168 cp -r java $dir 169 cd $dir && $MVN clean && $MVN test 170 cd ../.. 171 } 172 173 # The conformance tests are hard-coded to work with the $ROOT/java directory. 174 # So this can't run in parallel with two different sets of tests. 175 build_java_with_conformance_tests() { 176 # Java build needs `protoc`. 177 internal_build_cpp 178 cd java && $MVN test && $MVN install 179 cd util && $MVN package assembly:single 180 cd ../.. 181 cd conformance && make test_java && cd .. 182 } 183 184 build_javanano() { 185 # Java build needs `protoc`. 186 internal_build_cpp 187 cd javanano && $MVN test && cd .. 188 } 189 190 build_java_jdk7() { 191 use_java jdk7 192 build_java_with_conformance_tests 193 } 194 build_java_oracle7() { 195 use_java oracle7 196 build_java oracle7 197 } 198 199 build_javanano_jdk7() { 200 use_java jdk7 201 build_javanano 202 } 203 build_javanano_oracle7() { 204 use_java oracle7 205 build_javanano 206 } 207 208 internal_install_python_deps() { 209 if [ "$TRAVIS" != "true" ]; then 210 return; 211 fi 212 # Install tox (OS X doesn't have pip). 213 if [ $(uname -s) == "Darwin" ]; then 214 sudo easy_install tox 215 else 216 sudo pip install tox 217 fi 218 # Only install Python2.6/3.x on Linux. 219 if [ $(uname -s) == "Linux" ]; then 220 sudo apt-get install -y python-software-properties # for apt-add-repository 221 sudo apt-add-repository -y ppa:fkrull/deadsnakes 222 sudo apt-get update -qq 223 sudo apt-get install -y python2.6 python2.6-dev 224 sudo apt-get install -y python3.3 python3.3-dev 225 sudo apt-get install -y python3.4 python3.4-dev 226 fi 227 } 228 229 build_objectivec_ios() { 230 # Reused the build script that takes care of configuring and ensuring things 231 # are up to date. The OS X test runs the objc conformance test, so skip it 232 # here. 233 # Note: travis has xctool installed, and we've looked at using it in the past 234 # but it has ended up proving unreliable (bugs), an they are removing build 235 # support in favor of xcbuild (or just xcodebuild). 236 objectivec/DevTools/full_mac_build.sh \ 237 --core-only --skip-xcode-osx --skip-objc-conformance "$@" 238 } 239 240 build_objectivec_ios_debug() { 241 build_objectivec_ios --skip-xcode-release 242 } 243 244 build_objectivec_ios_release() { 245 build_objectivec_ios --skip-xcode-debug 246 } 247 248 build_objectivec_osx() { 249 # Reused the build script that takes care of configuring and ensuring things 250 # are up to date. 251 objectivec/DevTools/full_mac_build.sh \ 252 --core-only --skip-xcode-ios 253 } 254 255 build_objectivec_cocoapods_integration() { 256 # First, load the RVM environment in bash, needed to update ruby. 257 source ~/.rvm/scripts/rvm 258 # Update rvm to the latest version. This is needed to solve 259 # https://github.com/google/protobuf/issues/1786 and may not be needed in the 260 # future when Travis updates the default version of rvm. 261 rvm get head 262 # Update ruby to 2.2.3 as the default one crashes with segmentation faults 263 # when using pod. 264 rvm use 2.2.3 --install --binary --fuzzy 265 # Update pod to the latest version. 266 gem install cocoapods --no-ri --no-rdoc 267 objectivec/Tests/CocoaPods/run_tests.sh 268 } 269 270 build_python() { 271 internal_build_cpp 272 internal_install_python_deps 273 cd python 274 # Only test Python 2.6/3.x on Linux 275 if [ $(uname -s) == "Linux" ]; then 276 envlist=py\{26,27,33,34\}-python 277 else 278 envlist=py27-python 279 fi 280 tox -e $envlist 281 cd .. 282 } 283 284 build_python_cpp() { 285 internal_build_cpp 286 internal_install_python_deps 287 export LD_LIBRARY_PATH=../src/.libs # for Linux 288 export DYLD_LIBRARY_PATH=../src/.libs # for OS X 289 cd python 290 # Only test Python 2.6/3.x on Linux 291 if [ $(uname -s) == "Linux" ]; then 292 # py26 is currently disabled due to json_format 293 envlist=py\{27,33,34\}-cpp 294 else 295 envlist=py27-cpp 296 fi 297 tox -e $envlist 298 cd .. 299 } 300 301 build_ruby21() { 302 internal_build_cpp # For conformance tests. 303 cd ruby && bash travis-test.sh ruby-2.1 && cd .. 304 } 305 build_ruby22() { 306 internal_build_cpp # For conformance tests. 307 cd ruby && bash travis-test.sh ruby-2.2 && cd .. 308 } 309 build_jruby() { 310 internal_build_cpp # For conformance tests. 311 # TODO(xiaofeng): Upgrade to jruby-9.x. There are some broken jests to be 312 # fixed. 313 cd ruby && bash travis-test.sh jruby-1.7 && cd .. 314 } 315 build_ruby_all() { 316 build_ruby21 317 build_ruby22 318 build_jruby 319 } 320 321 build_javascript() { 322 internal_build_cpp 323 cd js && npm install && npm test && cd .. 324 } 325 326 # Note: travis currently does not support testing more than one language so the 327 # .travis.yml cheats and claims to only be cpp. If they add multiple language 328 # support, this should probably get updated to install steps and/or 329 # rvm/gemfile/jdk/etc. entries rather than manually doing the work. 330 331 # .travis.yml uses matrix.exclude to block the cases where app-get can't be 332 # use to install things. 333 334 # -------- main -------- 335 336 if [ "$#" -ne 1 ]; then 337 echo " 338 Usage: $0 { cpp | 339 cpp_distcheck | 340 csharp | 341 java_jdk7 | 342 java_oracle7 | 343 javanano_jdk7 | 344 javanano_oracle7 | 345 objectivec_ios | 346 objectivec_ios_debug | 347 objectivec_ios_release | 348 objectivec_osx | 349 objectivec_cocoapods_integration | 350 python | 351 python_cpp | 352 ruby21 | 353 ruby22 | 354 jruby | 355 ruby_all) 356 " 357 exit 1 358 fi 359 360 set -e # exit immediately on error 361 set -x # display all commands 362 eval "build_$1" 363
