1 #!/bin/bash 2 3 # Tests for our tools. 4 # 5 # TODO: currently, this only passes on Linux (which is the platform that 6 # the housekeeper bot runs on, e.g. 7 # http://70.32.156.51:10117/builders/Skia_PerCommit_House_Keeping/builds/1415/steps/RunToolSelfTests/logs/stdio ) 8 # See https://code.google.com/p/skia/issues/detail?id=677 9 # ('make tools/tests/run.sh work cross-platform') 10 # Ideally, these tests should pass on all development platforms... 11 # otherwise, how can developers be expected to test them before committing a 12 # change? 13 14 # cd into .../trunk so all the paths will work 15 cd $(dirname $0)/../.. 16 17 # TODO: make it look in Release and/or Debug 18 SKDIFF_BINARY=out/Debug/skdiff 19 20 # Suffixes of the raw bench data files we want to process. 21 BENCHDATA_FILE_SUFFIXES_YES_INDIVIDUAL_TILES=\ 22 "data_skp_scale_1.3061_config_8888_mode_tile_256_256_timeIndividualTiles_bbh_rtree "\ 23 "data_skp_scale_1.3061_config_8888_mode_tile_256_256_timeIndividualTiles" 24 BENCHDATA_FILE_SUFFIXES_NO_INDIVIDUAL_TILES=\ 25 "data_skp_multi_4_scale_1.3061_config_8888_mode_tile_256_256 "\ 26 "data_skp_scale_1.3061_config_8888_mode_record" 27 28 # Compare contents of all files within directories $1 and $2, 29 # EXCEPT for any dotfiles. 30 # If there are any differences, a description is written to stdout and 31 # we exit with a nonzero return value. 32 # Otherwise, we write nothing to stdout and return. 33 function compare_directories { 34 if [ $# != 2 ]; then 35 echo "compare_directories requires exactly 2 parameters, got $#" 36 exit 1 37 fi 38 diff --recursive --exclude=.* $1 $2 39 if [ $? != 0 ]; then 40 echo "failed in: compare_directories $1 $2" 41 exit 1 42 fi 43 } 44 45 # Run skdiff with arguments in $1 (plus implicit final argument causing skdiff 46 # to write its output, if any, to directory $2/output-actual). 47 # Then compare its results against those in $2/output-expected. 48 function skdiff_test { 49 if [ $# != 2 ]; then 50 echo "skdiff_test requires exactly 2 parameters, got $#" 51 exit 1 52 fi 53 SKDIFF_ARGS="$1" 54 ACTUAL_OUTPUT_DIR="$2/output-actual" 55 EXPECTED_OUTPUT_DIR="$2/output-expected" 56 57 rm -rf $ACTUAL_OUTPUT_DIR 58 mkdir -p $ACTUAL_OUTPUT_DIR 59 COMMAND="$SKDIFF_BINARY $SKDIFF_ARGS $ACTUAL_OUTPUT_DIR" 60 echo "$COMMAND" >$ACTUAL_OUTPUT_DIR/command_line 61 $COMMAND &>$ACTUAL_OUTPUT_DIR/stdout 62 echo $? >$ACTUAL_OUTPUT_DIR/return_value 63 64 compare_directories $EXPECTED_OUTPUT_DIR $ACTUAL_OUTPUT_DIR 65 } 66 67 # Download a subset of the raw bench data for platform $1 at revision $2. 68 # (For the subset, download all files matching any of the suffixes in 69 # whitespace-separated list $3.) 70 # If any of those files already exist locally, we assume that they are 71 # correct and up to date, and we don't download them again. 72 function download_bench_rawdata { 73 if [ $# != 3 ]; then 74 echo "download_bench_rawdata requires exactly 3 parameters, got $#" 75 exit 1 76 fi 77 PLATFORM="$1" 78 REV="$2" 79 FILE_SUFFIXES="$3" 80 81 PLATFORM_DIR="tools/tests/benchalerts/$PLATFORM" 82 RAW_BENCH_DATA_DIR="$PLATFORM_DIR/raw-bench-data" 83 mkdir -p $RAW_BENCH_DATA_DIR 84 85 for FILE_SUFFIX in $FILE_SUFFIXES; do 86 FILE=bench_${REV}_${FILE_SUFFIX} 87 DESTFILE=$RAW_BENCH_DATA_DIR/$FILE 88 if [ ! -f $DESTFILE ]; 89 then 90 URL=http://chromium-skia-gm.commondatastorage.googleapis.com/perfdata/${PLATFORM}/${FILE} 91 echo Downloading $URL ... 92 curl $URL --output $DESTFILE 93 fi 94 done 95 } 96 97 # Run check_bench_regressions.py across the data from platform $1, 98 # writing its output to output-actual and comparing those results against 99 # output-expected. 100 function benchalert_test { 101 if [ $# != 2 ]; then 102 echo "benchalert_test requires exactly 2 parameter, got $#" 103 exit 1 104 fi 105 PLATFORM="$1" 106 REVISION="$2" 107 108 PLATFORM_DIR="tools/tests/benchalerts/$PLATFORM" 109 RAW_BENCH_DATA_DIR="$PLATFORM_DIR/raw-bench-data" 110 ACTUAL_OUTPUT_DIR="$PLATFORM_DIR/output-actual" 111 EXPECTED_OUTPUT_DIR="$PLATFORM_DIR/output-expected" 112 113 # Run check_bench_regressions.py . 114 rm -rf $ACTUAL_OUTPUT_DIR 115 mkdir -p $ACTUAL_OUTPUT_DIR 116 COMMAND="python bench/check_bench_regressions.py -a 25th -b $PLATFORM -d $RAW_BENCH_DATA_DIR -e $PLATFORM_DIR/expectations.txt -r $REVISION" 117 echo "$COMMAND" >$ACTUAL_OUTPUT_DIR/command_line 118 START_TIMESTAMP=$(date +%s) 119 $COMMAND 2>$ACTUAL_OUTPUT_DIR/stderr 120 echo $? >$ACTUAL_OUTPUT_DIR/return_value 121 END_TIMESTAMP=$(date +%s) 122 123 SECONDS_RUN=$(expr $END_TIMESTAMP - $START_TIMESTAMP) 124 echo "check_bench_regressions.py took $SECONDS_RUN seconds to complete" 125 126 compare_directories $EXPECTED_OUTPUT_DIR $ACTUAL_OUTPUT_DIR 127 } 128 129 # Test rebaseline.py's JSON-format expectations rebaselining capability. 130 # 131 # Copy expected-results.json files from $1 into a dir where they can be modified. 132 # Run rebaseline.py with arguments in $2, recording its output. 133 # Then compare the output (and modified expected-results.json files) to the 134 # content of $2/output-expected. 135 function rebaseline_test { 136 if [ $# != 3 ]; then 137 echo "rebaseline_test requires exactly 3 parameters, got $#" 138 exit 1 139 fi 140 COPY_EXPECTATIONS_FROM_DIR="$1" 141 ARGS="$2" 142 ACTUAL_OUTPUT_DIR="$3/output-actual" 143 EXPECTED_OUTPUT_DIR="$3/output-expected" 144 145 rm -rf $ACTUAL_OUTPUT_DIR 146 mkdir -p $ACTUAL_OUTPUT_DIR 147 EXPECTATIONS_TO_MODIFY_DIR="$ACTUAL_OUTPUT_DIR/gm-expectations" 148 BUILDERS=$(ls $COPY_EXPECTATIONS_FROM_DIR) 149 for BUILDER in $BUILDERS; do 150 mkdir -p $EXPECTATIONS_TO_MODIFY_DIR/$BUILDER 151 cp $COPY_EXPECTATIONS_FROM_DIR/$BUILDER/expected-results.json \ 152 $EXPECTATIONS_TO_MODIFY_DIR/$BUILDER 153 done 154 COMMAND="python tools/rebaseline.py --expectations-root $EXPECTATIONS_TO_MODIFY_DIR $ARGS" 155 echo "$COMMAND" >$ACTUAL_OUTPUT_DIR/command_line 156 $COMMAND &>$ACTUAL_OUTPUT_DIR/stdout 157 echo $? >$ACTUAL_OUTPUT_DIR/return_value 158 159 compare_directories $EXPECTED_OUTPUT_DIR $ACTUAL_OUTPUT_DIR 160 } 161 162 # Run jsondiff.py with arguments in $1, recording its output. 163 # Then compare that output to the content of $2/output-expected. 164 function jsondiff_test { 165 if [ $# != 2 ]; then 166 echo "jsondiff_test requires exactly 2 parameters, got $#" 167 exit 1 168 fi 169 ARGS="$1" 170 ACTUAL_OUTPUT_DIR="$2/output-actual" 171 EXPECTED_OUTPUT_DIR="$2/output-expected" 172 173 rm -rf $ACTUAL_OUTPUT_DIR 174 mkdir -p $ACTUAL_OUTPUT_DIR 175 COMMAND="python tools/jsondiff.py $ARGS" 176 echo "$COMMAND" >$ACTUAL_OUTPUT_DIR/command_line 177 $COMMAND &>$ACTUAL_OUTPUT_DIR/stdout 178 echo $? >$ACTUAL_OUTPUT_DIR/return_value 179 180 compare_directories $EXPECTED_OUTPUT_DIR $ACTUAL_OUTPUT_DIR 181 } 182 183 184 185 # 186 # Run skdiff tests... 187 # 188 189 SKDIFF_TESTDIR=tools/tests/skdiff 190 191 # Run skdiff over a variety of file pair types: identical bits, identical pixels, missing from 192 # baseDir, etc. 193 skdiff_test "$SKDIFF_TESTDIR/baseDir $SKDIFF_TESTDIR/comparisonDir" "$SKDIFF_TESTDIR/test1" 194 195 # Run skdiff over the same set of files, but with arguments as used by our buildbots: 196 # - return the number of mismatching file pairs (but ignore any files missing from either 197 # baseDir or comparisonDir) 198 # - list filenames with each result type to stdout 199 # - don't generate HTML output files 200 skdiff_test "--failonresult DifferentPixels --failonresult DifferentSizes --failonresult Unknown --failonstatus CouldNotDecode,CouldNotRead any --failonstatus any CouldNotDecode,CouldNotRead --listfilenames --nodiffs $SKDIFF_TESTDIR/baseDir $SKDIFF_TESTDIR/comparisonDir" "$SKDIFF_TESTDIR/test2" 201 202 # Run skdiff over just the files that have identical bits. 203 skdiff_test "--nodiffs --match identical-bits $SKDIFF_TESTDIR/baseDir $SKDIFF_TESTDIR/comparisonDir" "$SKDIFF_TESTDIR/identical-bits" 204 205 # Run skdiff over just the files that have identical bits or identical pixels. 206 skdiff_test "--nodiffs --match identical-bits --match identical-pixels $SKDIFF_TESTDIR/baseDir $SKDIFF_TESTDIR/comparisonDir" "$SKDIFF_TESTDIR/identical-bits-or-pixels" 207 208 # 209 # Run bench alerts tests... 210 # 211 212 # Parse a collection of bench data 213 PLATFORM=Perf-Android-Nexus7-Tegra3-Arm7-Release 214 REVISION=69c9e1a7261a3c8361e2b2c109d6340862149e34 215 download_bench_rawdata $PLATFORM $REVISION "$BENCHDATA_FILE_SUFFIXES_NO_INDIVIDUAL_TILES" 216 download_bench_rawdata $PLATFORM $REVISION "$BENCHDATA_FILE_SUFFIXES_YES_INDIVIDUAL_TILES" 217 benchalert_test $PLATFORM $REVISION 218 219 # 220 # Run self test for skimage ... 221 # 222 223 COMMAND="python tools/tests/skimage_self_test.py" 224 echo "$COMMAND" 225 $COMMAND 226 ret=$? 227 if [ $ret -ne 0 ]; then 228 echo "skimage self tests failed." 229 exit 1 230 fi 231 232 # 233 # Test rebaseline.py ... 234 # 235 236 REBASELINE_INPUT=tools/tests/rebaseline/input 237 REBASELINE_OUTPUT=tools/tests/rebaseline/output 238 rebaseline_test "$REBASELINE_INPUT/json1" "--actuals-base-url $REBASELINE_INPUT/json1 --builders Test-Android-GalaxyNexus-SGX540-Arm7-Debug Test-Win7-ShuttleA-HD2000-x86-Release" "$REBASELINE_OUTPUT/using-json1-expectations" 239 rebaseline_test "$REBASELINE_INPUT/json1" "--actuals-base-url $REBASELINE_INPUT/json1 --bugs 1234 5678 --builders Test-Android-GalaxyNexus-SGX540-Arm7-Debug Test-Win7-ShuttleA-HD2000-x86-Release --notes notes_content --unreviewed" "$REBASELINE_OUTPUT/marked-unreviewed" 240 rebaseline_test "$REBASELINE_INPUT/json1" "--actuals-base-url $REBASELINE_INPUT/json1 --add-new --builders Test-Android-GalaxyNexus-SGX540-Arm7-Debug Test-Mac10.6-MacMini4.1-GeForce320M-x86-Release Test-Win7-ShuttleA-HD2000-x86-Release" "$REBASELINE_OUTPUT/add-new" 241 242 # 243 # Test jsondiff.py ... 244 # 245 246 JSONDIFF_INPUT=tools/tests/jsondiff/input 247 JSONDIFF_OUTPUT=tools/tests/jsondiff/output 248 jsondiff_test "$JSONDIFF_INPUT/old.json $JSONDIFF_INPUT/new.json" "$JSONDIFF_OUTPUT/old-vs-new" 249 250 251 echo "All tests passed." 252
