1 #!/bin/bash 2 3 # Self-tests for gm, based on tools/tests/run.sh 4 # 5 # These tests are run by the Skia_PerCommit_House_Keeping bot at every commit, 6 # so make sure that they still pass when you make changes to gm! 7 # 8 # To generate new baselines when gm behavior changes, run gm/tests/rebaseline.sh 9 # 10 # TODO: because this is written as a shell script (instead of, say, Python) 11 # it only runs on Linux and Mac. 12 # See https://code.google.com/p/skia/issues/detail?id=677 13 # ('make tools/tests/run.sh work cross-platform') 14 # Ideally, these tests should pass on all development platforms... 15 # otherwise, how can developers be expected to test them before committing a 16 # change? 17 18 # cd into .../trunk so all the paths will work 19 cd $(dirname $0)/../.. 20 21 # TODO(epoger): make it look in Release and/or Debug 22 GM_BINARY=out/Debug/gm 23 24 OUTPUT_ACTUAL_SUBDIR=output-actual 25 OUTPUT_EXPECTED_SUBDIR=output-expected 26 CONFIGS="--config 8888 565" 27 28 ENCOUNTERED_ANY_ERRORS=0 29 30 # Compare contents of all files within directories $1 and $2, 31 # EXCEPT for any dotfiles. 32 # If there are any differences, a description is written to stdout and 33 # we exit with a nonzero return value. 34 # Otherwise, we write nothing to stdout and return. 35 function compare_directories { 36 if [ $# != 2 ]; then 37 echo "compare_directories requires exactly 2 parameters, got $#" 38 exit 1 39 fi 40 diff -r --exclude=.* $1 $2 41 if [ $? != 0 ]; then 42 echo "failed in: compare_directories $1 $2" 43 ENCOUNTERED_ANY_ERRORS=1 44 fi 45 } 46 47 # Run a command, and validate that it succeeds (returns 0). 48 function assert_passes { 49 COMMAND="$1" 50 OUTPUT=$($COMMAND 2>&1) 51 if [ $? != 0 ]; then 52 echo "This command was supposed to pass, but failed: [$COMMAND]" 53 echo $OUTPUT 54 ENCOUNTERED_ANY_ERRORS=1 55 fi 56 } 57 58 # Run a command, and validate that it fails (returns nonzero). 59 function assert_fails { 60 COMMAND="$1" 61 OUTPUT=$($COMMAND 2>&1) 62 if [ $? == 0 ]; then 63 echo "This command was supposed to fail, but passed: [$COMMAND]" 64 echo $OUTPUT 65 ENCOUNTERED_ANY_ERRORS=1 66 fi 67 } 68 69 # Run gm... 70 # - with the arguments in $1 71 # - writing stdout into $2/$OUTPUT_ACTUAL_SUBDIR/stdout 72 # - writing json summary into $2/$OUTPUT_ACTUAL_SUBDIR/json-summary.txt 73 # - writing return value into $2/$OUTPUT_ACTUAL_SUBDIR/return_value 74 # Then compare all of those against $2/$OUTPUT_EXPECTED_SUBDIR . 75 function gm_test { 76 if [ $# != 2 ]; then 77 echo "gm_test requires exactly 2 parameters, got $#" 78 exit 1 79 fi 80 GM_ARGS="$1" 81 ACTUAL_OUTPUT_DIR="$2/$OUTPUT_ACTUAL_SUBDIR" 82 EXPECTED_OUTPUT_DIR="$2/$OUTPUT_EXPECTED_SUBDIR" 83 JSON_SUMMARY_FILE="$ACTUAL_OUTPUT_DIR/json-summary.txt" 84 85 rm -rf $ACTUAL_OUTPUT_DIR 86 mkdir -p $ACTUAL_OUTPUT_DIR 87 88 COMMAND="$GM_BINARY $GM_ARGS --writeJsonSummaryPath $JSON_SUMMARY_FILE --writePath $ACTUAL_OUTPUT_DIR/writePath --mismatchPath $ACTUAL_OUTPUT_DIR/mismatchPath --missingExpectationsPath $ACTUAL_OUTPUT_DIR/missingExpectationsPath" 89 90 echo "$COMMAND" >$ACTUAL_OUTPUT_DIR/command_line 91 $COMMAND >$ACTUAL_OUTPUT_DIR/stdout 2>$ACTUAL_OUTPUT_DIR/stderr 92 echo $? >$ACTUAL_OUTPUT_DIR/return_value 93 94 # Only compare selected lines in the stdout, to ignore any spurious lines 95 # as noted in http://code.google.com/p/skia/issues/detail?id=1068 . 96 # 97 # TODO(epoger): This is still hacky... we need to rewrite this script in 98 # Python soon, and make stuff like this more maintainable. 99 grep ^GM: $ACTUAL_OUTPUT_DIR/stdout >$ACTUAL_OUTPUT_DIR/stdout-tmp 100 mv $ACTUAL_OUTPUT_DIR/stdout-tmp $ACTUAL_OUTPUT_DIR/stdout 101 grep ^GM: $ACTUAL_OUTPUT_DIR/stderr >$ACTUAL_OUTPUT_DIR/stderr-tmp 102 mv $ACTUAL_OUTPUT_DIR/stderr-tmp $ACTUAL_OUTPUT_DIR/stderr 103 104 # Replace image file contents with just the filename, for two reasons: 105 # 1. Image file encoding may vary by platform 106 # 2. https://code.google.com/p/chromium/issues/detail?id=169600 107 # ('gcl/upload.py fail to upload binary files to rietveld') 108 for IMAGEFILE in $(find $ACTUAL_OUTPUT_DIR -name *.png); do 109 echo "[contents of $IMAGEFILE]" >$IMAGEFILE 110 done 111 for IMAGEFILE in $(find $ACTUAL_OUTPUT_DIR -name *.pdf); do 112 echo "[contents of $IMAGEFILE]" >$IMAGEFILE 113 done 114 115 # Add a file to any empty subdirectories. 116 for DIR in $(find $ACTUAL_OUTPUT_DIR -mindepth 1 -type d); do 117 echo "Created additional file to make sure directory isn't empty, because self-test cannot handle empty directories." >$DIR/bogusfile 118 done 119 120 compare_directories $EXPECTED_OUTPUT_DIR $ACTUAL_OUTPUT_DIR 121 } 122 123 # Create input dir (at path $1) with expectations (both image and json) 124 # that gm will match or mismatch as appropriate. 125 # 126 # We used to check these files into SVN, but then we needed to rebaseline them 127 # when our drawing changed at all... so, as proposed in 128 # http://code.google.com/p/skia/issues/detail?id=1068 , we generate them 129 # new each time. 130 function create_inputs_dir { 131 if [ $# != 1 ]; then 132 echo "create_inputs_dir requires exactly 1 parameter, got $#" 133 exit 1 134 fi 135 INPUTS_DIR="$1" 136 IMAGES_DIR=$INPUTS_DIR/images 137 JSON_DIR=$INPUTS_DIR/json 138 mkdir -p $IMAGES_DIR $JSON_DIR 139 140 THIS_IMAGE_DIR=$IMAGES_DIR/identical-bytes 141 mkdir -p $THIS_IMAGE_DIR 142 # Run GM to write out the images actually generated. 143 $GM_BINARY --hierarchy --match selftest1 $CONFIGS -w $THIS_IMAGE_DIR 144 # Run GM again to read in those images and write them out as a JSON summary. 145 $GM_BINARY --hierarchy --match selftest1 $CONFIGS -r $THIS_IMAGE_DIR \ 146 --writeJsonSummaryPath $JSON_DIR/identical-bytes.json 147 148 THIS_IMAGE_DIR=$IMAGES_DIR/identical-pixels 149 mkdir -p $THIS_IMAGE_DIR 150 $GM_BINARY --hierarchy --match selftest1 $CONFIGS -w $THIS_IMAGE_DIR 151 echo "more bytes that do not change the image pixels" \ 152 >> $THIS_IMAGE_DIR/8888/selftest1.png 153 echo "more bytes that do not change the image pixels" \ 154 >> $THIS_IMAGE_DIR/565/selftest1.png 155 $GM_BINARY --hierarchy --match selftest1 $CONFIGS -r $THIS_IMAGE_DIR \ 156 --writeJsonSummaryPath $JSON_DIR/identical-pixels.json 157 158 THIS_IMAGE_DIR=$IMAGES_DIR/different-pixels 159 mkdir -p $THIS_IMAGE_DIR 160 $GM_BINARY --hierarchy --match selftest2 $CONFIGS -w $THIS_IMAGE_DIR 161 mv $THIS_IMAGE_DIR/8888/selftest2.png $THIS_IMAGE_DIR/8888/selftest1.png 162 mv $THIS_IMAGE_DIR/565/selftest2.png $THIS_IMAGE_DIR/565/selftest1.png 163 $GM_BINARY --hierarchy --match selftest1 $CONFIGS -r $THIS_IMAGE_DIR \ 164 --writeJsonSummaryPath $JSON_DIR/different-pixels.json 165 166 # Create another JSON expectations file which is identical to 167 # different-pixels.json, but in which the *first* ignore-failure is changed 168 # from false to true. 169 OLD='"ignore-failure" : false' 170 NEW='"ignore-failure" : true' 171 sed -e "0,/$OLD/{s/$OLD/$NEW/}" $JSON_DIR/different-pixels.json \ 172 >$JSON_DIR/different-pixels-ignore-some-failures.json 173 174 THIS_IMAGE_DIR=$IMAGES_DIR/different-pixels-no-hierarchy 175 mkdir -p $THIS_IMAGE_DIR 176 $GM_BINARY --match selftest2 $CONFIGS -w $THIS_IMAGE_DIR 177 mv $THIS_IMAGE_DIR/selftest2_8888.png $THIS_IMAGE_DIR/selftest1_8888.png 178 mv $THIS_IMAGE_DIR/selftest2_565.png $THIS_IMAGE_DIR/selftest1_565.png 179 $GM_BINARY --match selftest1 $CONFIGS -r $THIS_IMAGE_DIR \ 180 --writeJsonSummaryPath $JSON_DIR/different-pixels-no-hierarchy.json 181 182 mkdir -p $IMAGES_DIR/empty-dir 183 184 echo "# Comment line" >$GM_IGNORE_FAILURES_FILE 185 echo "" >>$GM_IGNORE_FAILURES_FILE 186 echo "# ignore any test runs whose filename contains '8888/selfte'" >>$GM_IGNORE_FAILURES_FILE 187 echo "# (in other words, config is 8888 and test name starts with 'selfte')" >>$GM_IGNORE_FAILURES_FILE 188 echo "8888/selfte" >>$GM_IGNORE_FAILURES_FILE 189 } 190 191 GM_TESTDIR=gm/tests 192 GM_INPUTS=$GM_TESTDIR/inputs 193 GM_OUTPUTS=$GM_TESTDIR/outputs 194 GM_TEMPFILES=$GM_TESTDIR/tempfiles 195 GM_IGNORE_FAILURES_FILE=$GM_INPUTS/ignored-tests.txt 196 197 create_inputs_dir $GM_INPUTS 198 199 # Compare generated image against an input image file with identical bytes. 200 gm_test "--verbose --hierarchy --match selftest1 $CONFIGS -r $GM_INPUTS/images/identical-bytes" "$GM_OUTPUTS/compared-against-identical-bytes-images" 201 gm_test "--verbose --hierarchy --match selftest1 $CONFIGS -r $GM_INPUTS/json/identical-bytes.json" "$GM_OUTPUTS/compared-against-identical-bytes-json" 202 203 # Compare generated image against an input image file with identical pixels but different PNG encoding. 204 gm_test "--verbose --hierarchy --match selftest1 $CONFIGS -r $GM_INPUTS/images/identical-pixels" "$GM_OUTPUTS/compared-against-identical-pixels-images" 205 gm_test "--verbose --hierarchy --match selftest1 $CONFIGS -r $GM_INPUTS/json/identical-pixels.json" "$GM_OUTPUTS/compared-against-identical-pixels-json" 206 207 # Compare generated image against an input image file with different pixels. 208 gm_test "--verbose --hierarchy --match selftest1 $CONFIGS -r $GM_INPUTS/images/different-pixels" "$GM_OUTPUTS/compared-against-different-pixels-images" 209 gm_test "--verbose --hierarchy --match selftest1 $CONFIGS -r $GM_INPUTS/json/different-pixels.json" "$GM_OUTPUTS/compared-against-different-pixels-json" 210 211 # Exercise --ignoreFailuresFile flag. 212 gm_test "--verbose --hierarchy --match selftest1 --ignoreFailuresFile $GM_IGNORE_FAILURES_FILE $CONFIGS -r $GM_INPUTS/json/different-pixels.json" "$GM_OUTPUTS/ignoring-one-test" 213 214 # Compare different pixels, but with a SUBSET of the expectations marked as 215 # ignore-failure. 216 gm_test "--verbose --hierarchy --match selftest1 $CONFIGS -r $GM_INPUTS/json/different-pixels-ignore-some-failures.json" "$GM_OUTPUTS/ignoring-some-failures" 217 218 # Compare generated image against an empty "expected image" dir. 219 # Even the tests that have been marked as ignore-failure should show up as 220 # no-comparison. 221 gm_test "--verbose --hierarchy --match selftest1 --ignoreFailuresFile $GM_IGNORE_FAILURES_FILE $CONFIGS -r $GM_INPUTS/images/empty-dir" "$GM_OUTPUTS/compared-against-empty-dir" 222 223 # Compare generated image against a nonexistent "expected image" dir. 224 gm_test "--verbose --hierarchy --match selftest1 $CONFIGS -r ../path/to/nowhere" "$GM_OUTPUTS/compared-against-nonexistent-dir" 225 226 # Compare generated image against an empty "expected image" dir, but NOT in verbose mode. 227 gm_test "--hierarchy --match selftest1 $CONFIGS -r $GM_INPUTS/images/empty-dir" "$GM_OUTPUTS/nonverbose" 228 229 # Add pdf to the list of configs. 230 gm_test "--verbose --hierarchy --match selftest1 $CONFIGS pdf -r $GM_INPUTS/json/identical-bytes.json" "$GM_OUTPUTS/add-config-pdf" 231 232 # Test what happens if run without -r (no expected-results.json to compare 233 # against). 234 gm_test "--verbose --hierarchy --match selftest1 $CONFIGS" "$GM_OUTPUTS/no-readpath" 235 236 # Test what happens if a subset of the renderModes fail (e.g. pipe) 237 gm_test "--pipe --simulatePipePlaybackFailure --verbose --hierarchy --match selftest1 $CONFIGS -r $GM_INPUTS/json/identical-pixels.json" "$GM_OUTPUTS/pipe-playback-failure" 238 239 # Confirm that IntentionallySkipped tests are recorded as such. 240 gm_test "--verbose --hierarchy --match selftest1 selftest2 $CONFIGS" "$GM_OUTPUTS/intentionally-skipped-tests" 241 242 # Ignore some error types (including ExpectationsMismatch) 243 gm_test "--ignoreErrorTypes ExpectationsMismatch NoGpuContext --verbose --hierarchy --match selftest1 $CONFIGS -r $GM_INPUTS/json/different-pixels.json" "$GM_OUTPUTS/ignore-expectations-mismatch" 244 245 # Test non-hierarchical mode. 246 gm_test "--verbose --match selftest1 $CONFIGS -r $GM_INPUTS/json/different-pixels-no-hierarchy.json" "$GM_OUTPUTS/no-hierarchy" 247 248 # Try writing out actual images using checksum-based filenames, like we do when 249 # uploading to Google Storage. 250 gm_test "--verbose --writeChecksumBasedFilenames --match selftest1 $CONFIGS -r $GM_INPUTS/json/different-pixels-no-hierarchy.json" "$GM_OUTPUTS/checksum-based-filenames" 251 252 # Exercise display_json_results.py 253 PASSING_CASES="compared-against-identical-bytes-json compared-against-identical-pixels-json" 254 FAILING_CASES="compared-against-different-pixels-json" 255 for CASE in $PASSING_CASES; do 256 assert_passes "python gm/display_json_results.py $GM_OUTPUTS/$CASE/$OUTPUT_EXPECTED_SUBDIR/json-summary.txt" 257 done 258 for CASE in $FAILING_CASES; do 259 assert_fails "python gm/display_json_results.py $GM_OUTPUTS/$CASE/$OUTPUT_EXPECTED_SUBDIR/json-summary.txt" 260 done 261 262 if [ $ENCOUNTERED_ANY_ERRORS == 0 ]; then 263 echo "All tests passed." 264 exit 0 265 else 266 exit 1 267 fi 268
