Home | History | Annotate | Download | only in test
      1 #!/bin/sh
      2 # //===--------------------------- testit ---------------------------------===//
      3 # //
      4 # //                     The LLVM Compiler Infrastructure
      5 # //
      6 # // This file is distributed under the University of Illinois Open Source
      7 # // License. See LICENSE.TXT for details.
      8 # //
      9 # //===--------------------------------------------------------------------===//
     10 
     11 currentpath=`pwd`
     12 origpath=$currentpath
     13 currentdir=`basename $currentpath`
     14 while [ $currentdir != "test" ]; do
     15 	if [ $currentdir = "/" ]
     16 	then
     17 		echo "current directory must be in or under \"test\"."
     18 		exit 1
     19 	fi
     20 	cd ..
     21 	currentpath=`pwd`
     22 	currentdir=`basename $currentpath`
     23 done
     24 
     25 cd ..
     26 LIBCXX_ROOT=`pwd`
     27 cd $origpath
     28 
     29 VERBOSE=1
     30 
     31 run () {
     32   if [ "$VERBOSE" -gt 1 ]; then
     33     echo "COMMAND: $@"
     34   fi
     35   case $VERBOSE in
     36     0|1)
     37       # Hide command output and errors.
     38       "$@" >/dev/null 2>&1
     39       ;;
     40     2)
     41       # Only hide command output
     42       "$@" >/dev/null
     43       ;;
     44     *)
     45       # Show command output and errors.
     46       "$@"
     47       ;;
     48   esac
     49 }
     50 
     51 run2 () {
     52   if [ "$VERBOSE" -gt 2 ]; then
     53     echo "COMMAND: $@"
     54   fi
     55   case $VERBOSE in
     56     0|1)
     57       # Hide command output and errors.
     58       "$@" >/dev/null 2>&1
     59       ;;
     60     2)
     61       # Only hide command output
     62       "$@" >/dev/null
     63       ;;
     64     *)
     65       # Show command output and errors.
     66       "$@"
     67       ;;
     68   esac
     69 }
     70 
     71 # The list of valid target abis supported by this script.
     72 VALID_ABIS="armeabi armeabi-v7a x86 mips"
     73 
     74 HOST_OS=`uname -s`
     75 if [ "$HOST_OS" = "Darwin" ]; then
     76     # darwin doesn't have readelf.  hard-code a known one
     77     READELF=../../../../../toolchains/arm-linux-androideabi-4.8/prebuilt/darwin-x86/bin/arm-linux-androideabi-readelf
     78 else
     79     READELF=readelf
     80 fi
     81 
     82 DO_HELP=
     83 DO_STATIC=
     84 TARGET_ABI=
     85 TARGET_ARCH=
     86 if [ -z "$TARGET_TEST_SUBDIR" ]; then
     87     TARGET_TEST_SUBDIR="libcxx"
     88 fi
     89 TARGET_PATH=/data/local/tmp/$TARGET_TEST_SUBDIR
     90 CXX=
     91 WITH_COMPILER_RT=
     92 LIBCXX_NEEDS_COMPILER_RT=
     93 for OPT; do
     94   case $OPT in
     95     --help|-?)
     96       DO_HELP=true
     97       ;;
     98     --abi=*)
     99       TARGET_ABI=${OPT##--abi=}
    100       ;;
    101     --static)
    102       DO_STATIC=true
    103       ;;
    104     --shared)
    105       DO_STATIC=
    106       ;;
    107     --cxx=*)
    108       CXX=${OPT##--cxx=}
    109       ;;
    110     --verbose)
    111       VERBOSE=$(( $VERBOSE + 1 ))
    112       ;;
    113     --with-compiler-rt)
    114       WITH_COMPILER_RT=yes
    115       ;;
    116     -*)
    117       echo "Unknown option: $OPT. See --help."
    118       exit 1
    119       ;;
    120     *)
    121       echo "This script doesn't take parameters. See --help."
    122       exit 1
    123       ;;
    124   esac
    125 done
    126 
    127 if [ "$DO_HELP" ]; then
    128   echo \
    129 "Usage: $(basename $0) [options]
    130 
    131 This script is used to run the libc++ test suite for Android.
    132 You will need the following things:
    133 
    134   - The prebuild libc++ libraries in your NDK install.
    135   - A prebuilt Android toolchain in your path.
    136   - The 'adb' tool in your path.
    137   - An Android device connected to ADB.
    138 
    139 The toolchain and device must match your target ABI. For example, if
    140 you use --abi=armeabi-v7a, your device must run ARMv7-A Android binaries,
    141 and arm-linux-androideabi-g++ will be used to compile all tests, unless
    142 you use --cxx=<command> to override it.
    143 
    144 Valid options:
    145   --help|-?        Display this message.
    146   --abi=<name>     Specify target ABI. Use --abi=list for list.
    147   --static         Link against static libc++ library.
    148   --cxx=<program>  Override C++ compiler/linker.
    149   --verbose        Increase verbosity.
    150 "
    151   exit 0
    152 fi
    153 
    154 # Check target ABI.
    155 if [ "$TARGET_ABI" = "list" ]; then
    156   echo "List of valid target ABIs:"
    157   for ABI in $VALID_ABIS; do
    158     printf " %s" $ABI
    159   done
    160   printf "\n"
    161   exit 0
    162 fi
    163 
    164 if [ -z "$TARGET_ABI" ]; then
    165   echo "ERROR: Please specify a target ABI (--abi=<name>)."
    166   exit 1
    167 fi
    168 
    169 FOUND_ABI=
    170 for ABI in $VALID_ABIS; do
    171   if [ "$ABI" = "$TARGET_ABI" ]; then
    172     FOUND_ABI=true
    173     break
    174   fi
    175 done
    176 
    177 if [ -z "$FOUND_ABI" ]; then
    178   echo "ERROR: Invalid abi '$TARGET_ABI'. Must be one of: $VALID_ABIS"
    179   exit 1
    180 fi
    181 
    182 LIBCXX_LIBS=$(cd $LIBCXX_ROOT/.. && pwd)/libs/$TARGET_ABI
    183 for LIB in libc++_static.a libc++_shared.so; do
    184   if [ ! -f "$LIBCXX_LIBS/$LIB" ]; then
    185     echo "ERROR: Missing prebuilt library: $LIBCXX_LIBS/$LIB"
    186     echo "Please run: build/tools/build-cxx-stl.sh --stl=libc++"
    187     exit 1
    188   fi
    189 done
    190 
    191 LIBCOMPILER_RT_LIBS=$(cd "$LIBCXX_ROOT"/../../../android/compiler-rt && pwd)/libs/$TARGET_ABI
    192 $READELF -d "$LIBCXX_LIBS/libc++_shared.so" | grep NEEDED | grep -q compiler_rt
    193 if [ $? = 0 ]; then
    194   LIBCXX_NEEDS_COMPILER_RT=yes
    195 fi
    196 
    197 if [ "$WITH_COMPILER_RT" = "yes" -o "$LIBCXX_NEEDS_COMPILER_RT" = "yes" ]; then
    198   for LIB in libcompiler_rt_static.a libcompiler_rt_shared.so; do
    199     if [ ! -f "$LIBCOMPILER_RT_LIBS/$LIB" ]; then
    200       echo "ERROR: Missing prebuilt library: $LIBCOMPILER_RT_LIBS/$LIB"
    201       echo "Please run: build/tools/build-compiler-rt.sh --ndk-dir=$NDK \
    202 --src-dir=/tmp/ndk-$USER/src/llvm-3.3/compiler-rt --llvm-version=3.3"
    203       exit 1
    204     fi
    205   done
    206 fi
    207 
    208 # Check or detect C++ toolchain.
    209 TOOLCHAIN_CFLAGS=
    210 TOOLCHAIN_LDFLAGS=
    211 if [ -z "$TOOLCHAIN_PREFIX" ]; then
    212   # Compute
    213   case $TARGET_ABI in
    214     armeabi)
    215       TOOLCHAIN_PREFIX=arm-linux-androideabi
    216       TOOLCHAIN_CFLAGS="-mthumb"
    217       TOOLCHAIN_LDFLAGS="-mthumb"
    218       ;;
    219     armeabi-v7a)
    220       TOOLCHAIN_PREFIX=arm-linux-androideabi
    221       TOOLCHAIN_CFLAGS="-march=armv7-a -mthumb -mfpu=vfpv3-d16"
    222       TOOLCHAIN_LDFLAGS="-march=armv7-a -mthumb -Wl,--fix-cortex-a8"
    223       ;;
    224     armeabi-v7a-hard)
    225       TOOLCHAIN_PREFIX=arm-linux-androideabi
    226       TOOLCHAIN_CFLAGS="-march=armv7-a -mthumb -mfpu=vfpv3-d16 -mhard-float -D_NDK_MATH_NO_SOFTFP=1"
    227       TOOLCHAIN_LDFLAGS="-march=armv7-a -mthumb -Wl,--fix-cortex-a8 -Wl,--no-warn-mismatch -lm_hard"
    228       ;;
    229     x86)
    230       TOOLCHAIN_PREFIX=i686-linux-android
    231       ;;
    232     mips)
    233       TOOLCHAIN_PREFIX=mipsel-linux-android
    234       ;;
    235     *)
    236       echo "ERROR: Unknown ABI '$ABI'"
    237       exit 1
    238       ;;
    239   esac
    240   if [ -z "$CXX" ]; then
    241       CXX=$TOOLCHAIN_PREFIX-g++
    242   fi
    243 fi
    244 
    245 REAL_CXX=$(which "$CXX" 2>/dev/null)
    246 if [ -z "$REAL_CXX" ]; then
    247   echo "ERROR: Missing C++ compiler: $CXX"
    248   exit 1
    249 fi
    250 CC=$CXX
    251 
    252 if [ -z "$OPTIONS" ]
    253 then
    254   # Note that some tests use assert() to check condition but -O2 defines assert() to nothing,
    255   # unless we specify -UNDEBUG to bring assert() back.
    256   OPTIONS="--std=c++11 -O2 -g -funwind-tables -UNDEBUG"
    257   if [ "$CXX" != "${CXX%clang++*}" ] ; then
    258     OPTIONS="$OPTIONS" # clang3.5 doesn't accept the following: -mllvm -arm-enable-ehabi-descriptors -mllvm -arm-enable-ehabi"
    259   else
    260     # Some libc++ implementation (eg. list) break TBAA!
    261     OPTIONS="$OPTIONS -fno-strict-aliasing"
    262   fi
    263 fi
    264 
    265 OPTIONS="$OPTIONS $TOOLCHAIN_CFLAGS $TOOLCHAIN_LDFLAGS"
    266 OPTIONS="$OPTIONS -I$LIBCXX_ROOT/test/support"
    267 # llvm-libc++/libcxx/test/lit.cfg line #278 defineds the following for testing only on Linux
    268 OPTIONS="$OPTIONS -D__STDC_FORMAT_MACROS -D__STDC_LIMIT_MACROS -D__STDC_CONSTANT_MACROS"
    269 
    270 if [ -z "$ADB" ]
    271 then
    272   ADB=adb
    273 fi
    274 
    275 # Run a shell command through ADB, return its status.
    276 # Variable ERR contains output if $RET is non-zero
    277 adb_shell () {
    278   # We need a temporary file to store the output of our command
    279   local CMD_OUT RET
    280   ERR=
    281   CMD_OUT=$(mktemp /tmp/testit_android-cmdout-XXXXXX)
    282   # Run the command, while storing the standard output to CMD_OUT
    283   # and appending the exit code as the last line.
    284   if [ "$VERBOSE" -gt 2 ]; then
    285     echo "COMMAND: $ADB shell $@"
    286   fi
    287   $ADB shell "$@ ; echo \$?" | sed -e 's![[:cntrl:]]!!g' > $CMD_OUT 2>&1
    288   # Get last line in log, which contains the exit code from the command
    289   RET=$(sed -e '$!d' $CMD_OUT)
    290   # Get output, which corresponds to everything except the last line
    291   OUT=$(sed -e '$d' $CMD_OUT)
    292   if [ "$RET" != "0" ]; then
    293     ERR=$OUT
    294   fi
    295   if [ "$VERBOSE" -gt 2 ]; then
    296     printf "%s" "$OUT"
    297   fi
    298   rm -f $CMD_OUT
    299   return $RET
    300 }
    301 
    302 # Push a given file through ADB.
    303 # $1: File path
    304 adb_push () {
    305   local FILE=$1
    306   local FILE_BASENAME=$(basename "$FILE")
    307   run2 $ADB push $FILE $TARGET_PATH/$FILE_BASENAME 2>/dev/null
    308 }
    309 
    310 # Run a given executable through ADB.
    311 # $1: Executable path
    312 adb_run () {
    313   local EXECUTABLE=$1
    314   local EXECUTABLE_BASENAME=$(basename "$EXECUTABLE")
    315   run2 $ADB push $EXECUTABLE $TARGET_PATH/$EXECUTABLE_BASENAME 2>/dev/null
    316   if [ "$?" != 0 ]; then
    317     return 1;
    318   fi
    319   # Retry up to 10 times if fail is due to "Text file busy"
    320   for i in 1 2 3 4 5 6 7 8 9 10; do
    321     adb_shell "LD_LIBRARY_PATH=$TARGET_PATH; cd $TARGET_PATH; ./$EXECUTABLE_BASENAME"
    322     if [ "$?" = "0" ]; then
    323       return 0
    324     fi
    325     if ! $(echo $ERR | grep -iq "Text file busy"); then
    326       if [ "$i" != "1" ]; then
    327         # Dump error message to help diagnostics
    328         echo "ERR=$ERR"
    329       fi
    330       break;
    331     fi
    332     echo "Text file busy.  Re-try $i"
    333     sleep 1
    334     run2 $ADB push $EXECUTABLE $TARGET_PATH/$EXECUTABLE_BASENAME 2>/dev/null
    335     sleep 2  # try again
    336   done
    337   return 1
    338 }
    339 
    340 adb_shell "rm -rf $TARGET_PATH"
    341 adb_shell "mkdir -p $TARGET_PATH"
    342 
    343 if [ "$DO_STATIC" ]; then
    344   # Statically link to ensure the executable can be run easily through ADB
    345   # Note that in standalone toolchain libc++_static is renamed to libstdc++, and
    346   # compiler adds -lstdc++ implicitly
    347   if [ "$WITH_COMPILER_RT" = "yes" ]; then
    348     LIBS="-nodefaultlibs -lstdc++ -latomic -ldl -lm -lc -lcompiler_rt_static"
    349   else
    350     LIBS="-latomic"
    351   fi
    352 else
    353   run2 $ADB push $LIBCXX_LIBS/libc++_shared.so $TARGET_PATH 2>/dev/null
    354   if [ $? != 0 ]; then
    355     echo "ERROR: Can't push shared libc++ to target device!"
    356     exit 1
    357   fi
    358 
    359   if [ "$WITH_COMPILER_RT" = "yes" -o "$LIBCXX_NEEDS_COMPILER_RT" = "yes" ]; then
    360     run2 $ADB push $LIBCOMPILER_RT_LIBS/libcompiler_rt_shared.so $TARGET_PATH 2>/dev/null
    361     if [ $? != 0 ]; then
    362       echo "ERROR: Can't push shared libcompiler_rt to target device!"
    363       exit 1
    364     fi
    365   fi
    366 
    367   if [ "$WITH_COMPILER_RT" = "yes" ]; then
    368     LIBS="-nodefaultlibs -lc++_shared -latomic -ldl -lm -lc -lcompiler_rt_shared"
    369   else
    370     LIBS="-lc++_shared -latomic"
    371   fi
    372 fi
    373 
    374 case $TRIPLE in
    375   *-*-mingw* | *-*-cygwin* | *-*-win*)
    376     TEST_EXE=test.exe
    377     ;;
    378   *)
    379     TEST_EXE=a.out
    380     ;;
    381 esac
    382 
    383 TEST_EXE=/tmp/testit_android-$USER-$$-$TEST_EXE
    384 
    385 FAIL=0
    386 PASS=0
    387 UNIMPLEMENTED=0
    388 IMPLEMENTED_FAIL=0
    389 IMPLEMENTED_PASS=0
    390 
    391 # Run tests in current directory, recursively
    392 #
    393 # Note that file path containing EQ are symlink to the existing tests whose path contain '=',
    394 # to workaround an issue in ndk-build which doesn't handle LOCAL_SRC_FILES with '='.
    395 # See tests/device/test-libc++-static-full/jni/Android.mk  We need to filter out path containing
    396 # EQ such that we don't run same tests twice
    397 #
    398 # An alternative is to do "find . -type f", but this doesn't work in NDK windows package
    399 # where zip turns symlink into physical file it points to.
    400 #
    401 # We also sort the test to make the test report comparable to previous test
    402 #
    403 
    404 afunc() {
    405 	fail=0
    406 	pass=0
    407 	if (ls *.fail.cpp > /dev/null 2>&1)
    408 	then
    409 		for FILE in $(ls *.fail.cpp | tr ' ' '\n' | grep -v EQ | sort); do
    410 			if run $CC $OPTIONS $HEADER_INCLUDE $SOURCE_LIB $FILE $LIBS -o $TEST_EXE > /dev/null 2>&1
    411 			then
    412 				rm $TEST_EXE
    413 				echo "$FILE should not compile"
    414 				fail=$(($fail+1))
    415 			else
    416 				pass=$(($pass+1))
    417 			fi
    418 		done
    419 	fi
    420 
    421 	if (ls *.pass.cpp > /dev/null 2>&1)
    422 	then
    423 		if (ls *.dat > /dev/null 2>&1)
    424 		then
    425 			adb_shell "rm -f $TARGET_PATH/*.dat"
    426 			for FILE in $(ls *.dat | tr ' ' '\n' | grep -v EQ | sort); do
    427 	                      if [ "$VERBOSE" -gt 1 ]; then
    428 	                          echo "Pushing data: " $FILE
    429 			      fi
    430 			      adb_push $FILE
    431 			      if [ $? != 0 ]; then
    432 				  echo "Failed to push file $FILE"
    433 	                      fi
    434 			done
    435 		fi
    436 		for FILE in $(ls *.pass.cpp | tr ' ' '\n' | grep -v EQ | sort); do
    437                       if [ "$VERBOSE" -gt 1 ]; then
    438                           echo "Running test: " $FILE
    439                       fi
    440                         COMMAND="( cd $(pwd) && $CC $OPTIONS $HEADER_INCLUDE $SOURCE_LIB $FILE $LIBS )"
    441 			if run $CC $OPTIONS $HEADER_INCLUDE $SOURCE_LIB $FILE $LIBS -o $TEST_EXE
    442 			then
    443 				if adb_run $TEST_EXE
    444 				then
    445 					rm $TEST_EXE
    446 					pass=$(($pass+1))
    447 				else
    448 					echo "`pwd`/$FILE failed at run time"
    449 					echo "Compile line was: $COMMAND # run-time"
    450 					fail=$(($fail+1))
    451 					rm $TEST_EXE
    452 				fi
    453 			else
    454 				echo "`pwd`/$FILE failed to compile"
    455 				echo "Compile line was: $COMMAND # compile-time"
    456 				fail=$(($fail+1))
    457 			fi
    458 		done
    459 	fi
    460 
    461 	if [ $fail -gt 0 ]
    462 	then
    463 		echo "failed $fail tests in `pwd`"
    464 		IMPLEMENTED_FAIL=$(($IMPLEMENTED_FAIL+1))
    465 	fi
    466 	if [ $pass -gt 0 ]
    467 	then
    468 		echo "passed $pass tests in `pwd`"
    469 		if [ $fail -eq 0 ]
    470 		then
    471 			IMPLEMENTED_PASS=$((IMPLEMENTED_PASS+1))
    472 		fi
    473 	fi
    474 	if [ $fail -eq 0 -a $pass -eq 0 ]
    475 	then
    476 		echo "not implemented:  `pwd`"
    477 		UNIMPLEMENTED=$(($UNIMPLEMENTED+1))
    478 	fi
    479 
    480 	FAIL=$(($FAIL+$fail))
    481 	PASS=$(($PASS+$pass))
    482 
    483 	for FILE in $(ls | tr ' ' '\n' | grep -v EQ | sort)
    484 	do
    485 		if [ -d "$FILE" ];
    486 		then
    487 			cd $FILE
    488 			afunc
    489 			cd ..
    490 		fi
    491 	done
    492 }
    493 
    494 afunc
    495 
    496 echo "****************************************************"
    497 echo "Results for `pwd`:"
    498 echo "using `$CC --version`"
    499 echo "with $OPTIONS $HEADER_INCLUDE $SOURCE_LIB"
    500 echo "----------------------------------------------------"
    501 echo "sections without tests   : $UNIMPLEMENTED"
    502 echo "sections with failures   : $IMPLEMENTED_FAIL"
    503 echo "sections without failures: $IMPLEMENTED_PASS"
    504 echo "                       +   ----"
    505 echo "total number of sections : $(($UNIMPLEMENTED+$IMPLEMENTED_FAIL+$IMPLEMENTED_PASS))"
    506 echo "----------------------------------------------------"
    507 echo "number of tests failed   : $FAIL"
    508 echo "number of tests passed   : $PASS"
    509 echo "                       +   ----"
    510 echo "total number of tests    : $(($FAIL+$PASS))"
    511 echo "****************************************************"
    512 
    513 exit $FAIL
    514