1 #!/usr/bin/env python 2 # Copyright (c) 2013 The Chromium Authors. All rights reserved. 3 # Use of this source code is governed by a BSD-style license that can be 4 # found in the LICENSE file. 5 6 # Self-test for skimage. 7 8 import filecmp 9 import os 10 import subprocess 11 import sys 12 import tempfile 13 14 class BinaryNotFoundException(Exception): 15 def __str__ (self): 16 return ("Could not find binary!\n" 17 "Did you forget to build the tools project?\n" 18 "Self tests failed") 19 20 # Find a path to the binary to use. Iterates through a list of possible 21 # locations the binary may be. 22 def PickBinaryPath(base_dir): 23 POSSIBLE_BINARY_PATHS = [ 24 'out/Debug/skimage', 25 'out/Release/skimage', 26 'xcodebuild/Debug/skimage', 27 'xcodebuild/Release/skimage', 28 ] 29 for binary in POSSIBLE_BINARY_PATHS: 30 binary_full_path = os.path.join(base_dir, binary) 31 if (os.path.exists(binary_full_path)): 32 return binary_full_path 33 raise BinaryNotFoundException 34 35 # Quit early if two files have different content. 36 def DieIfFilesMismatch(expected, actual): 37 if not filecmp.cmp(expected, actual): 38 print 'Error: file mismatch! expected=%s , actual=%s' % ( 39 expected, actual) 40 exit(1) 41 42 def test_invalid_file(file_dir, skimage_binary): 43 """ Test the return value of skimage when an invalid file is decoded. 44 If there is no expectation file, or the file expects a particular 45 result, skimage should return nonzero indicating failure. 46 If the file has no expectation, or ignore-failure is set to true, 47 skimage should return zero indicating success. """ 48 invalid_file = os.path.join(file_dir, "skimage", "input", "bad-images", 49 "invalid.png") 50 # No expectations file: 51 args = [skimage_binary, "--readPath", invalid_file] 52 result = subprocess.call(args) 53 if 0 == result: 54 print "'%s' should have reported failure!" % " ".join(args) 55 exit(1) 56 57 # Directory holding all expectations files 58 expectations_dir = os.path.join(file_dir, "skimage", "input", "bad-images") 59 60 # Expectations file expecting a valid decode: 61 incorrect_expectations = os.path.join(expectations_dir, 62 "incorrect-results.json") 63 args = [skimage_binary, "--readPath", invalid_file, 64 "--readExpectationsPath", incorrect_expectations] 65 result = subprocess.call(args) 66 if 0 == result: 67 print "'%s' should have reported failure!" % " ".join(args) 68 exit(1) 69 70 # Empty expectations: 71 empty_expectations = os.path.join(expectations_dir, "empty-results.json") 72 output = subprocess.check_output([skimage_binary, "--readPath", invalid_file, 73 "--readExpectationsPath", 74 empty_expectations], 75 stderr=subprocess.STDOUT) 76 if not "Missing" in output: 77 # Another test (in main()) tests to ensure that "Missing" does not appear 78 # in the output. That test could be passed if the output changed so 79 # "Missing" never appears. This ensures that an error is not missed if 80 # that happens. 81 print "skimage output changed! This may cause other self tests to fail!" 82 exit(1) 83 84 # Ignore failure: 85 ignore_expectations = os.path.join(expectations_dir, "ignore-results.json") 86 output = subprocess.check_output([skimage_binary, "--readPath", invalid_file, 87 "--readExpectationsPath", 88 ignore_expectations], 89 stderr=subprocess.STDOUT) 90 if not "failures" in output: 91 # Another test (in main()) tests to ensure that "failures" does not 92 # appear in the output. That test could be passed if the output changed 93 # so "failures" never appears. This ensures that an error is not missed 94 # if that happens. 95 print "skimage output changed! This may cause other self tests to fail!" 96 exit(1) 97 98 def test_incorrect_expectations(file_dir, skimage_binary): 99 """ Test that comparing to incorrect expectations fails, unless 100 ignore-failures is set to true. """ 101 valid_file = os.path.join(file_dir, "skimage", "input", 102 "images-with-known-hashes", 103 "1209453360120438698.png") 104 expectations_dir = os.path.join(file_dir, "skimage", "input", 105 "images-with-known-hashes") 106 107 incorrect_results = os.path.join(expectations_dir, 108 "incorrect-results.json") 109 args = [skimage_binary, "--readPath", valid_file, "--readExpectationsPath", 110 incorrect_results] 111 result = subprocess.call(args) 112 if 0 == result: 113 print "'%s' should have reported failure!" % " ".join(args) 114 exit(1) 115 116 ignore_results = os.path.join(expectations_dir, "ignore-failures.json") 117 subprocess.check_call([skimage_binary, "--readPath", valid_file, 118 "--readExpectationsPath", ignore_results]) 119 120 def main(): 121 # Use the directory of this file as the out directory 122 file_dir = os.path.abspath(os.path.dirname(__file__)) 123 124 trunk_dir = os.path.normpath(os.path.join(file_dir, os.pardir, os.pardir)) 125 126 # Find the binary 127 skimage_binary = PickBinaryPath(trunk_dir) 128 print "Running " + skimage_binary 129 130 # Generate an expectations file from known images. 131 images_dir = os.path.join(file_dir, "skimage", "input", 132 "images-with-known-hashes") 133 expectations_path = os.path.join(file_dir, "skimage", "output-actual", 134 "create-expectations", "expectations.json") 135 subprocess.check_call([skimage_binary, "--readPath", images_dir, 136 "--createExpectationsPath", expectations_path]) 137 138 # Make sure the expectations file was generated correctly. 139 golden_expectations = os.path.join(file_dir, "skimage", "output-expected", 140 "create-expectations", 141 "expectations.json") 142 DieIfFilesMismatch(expected=golden_expectations, actual=expectations_path) 143 144 # Tell skimage to read back the expectations file it just wrote, and 145 # confirm that the images in images_dir match it. 146 output = subprocess.check_output([skimage_binary, "--readPath", images_dir, 147 "--readExpectationsPath", 148 expectations_path], 149 stderr=subprocess.STDOUT) 150 151 # Although skimage succeeded, it would have reported success if the file 152 # was missing from the expectations file. Consider this a failure, since 153 # the expectations file was created from this same image. (It will print 154 # "Missing" in this case before listing the missing expectations). 155 if "Missing" in output: 156 print "Expectations file was missing expectations!" 157 print output 158 exit(1) 159 160 # Again, skimage would succeed if there were known failures (and print 161 # "failures"), but there should be no failures, since the file just 162 # created did not include failures to ignore. 163 if "failures" in output: 164 print "Image failed!" 165 print output 166 exit(1) 167 168 169 test_incorrect_expectations(file_dir=file_dir, 170 skimage_binary=skimage_binary) 171 172 # Generate an expectations file from an empty directory. 173 empty_dir = tempfile.mkdtemp() 174 expectations_path = os.path.join(file_dir, "skimage", "output-actual", 175 "empty-dir", "expectations.json") 176 subprocess.check_call([skimage_binary, "--readPath", empty_dir, 177 "--createExpectationsPath", expectations_path]) 178 golden_expectations = os.path.join(file_dir, "skimage", "output-expected", 179 "empty-dir", "expectations.json") 180 DieIfFilesMismatch(expected=golden_expectations, actual=expectations_path) 181 os.rmdir(empty_dir) 182 183 # Generate an expectations file from a nonexistent directory. 184 expectations_path = os.path.join(file_dir, "skimage", "output-actual", 185 "nonexistent-dir", "expectations.json") 186 subprocess.check_call([skimage_binary, "--readPath", "/nonexistent/dir", 187 "--createExpectationsPath", expectations_path]) 188 golden_expectations = os.path.join(file_dir, "skimage", "output-expected", 189 "nonexistent-dir", "expectations.json") 190 DieIfFilesMismatch(expected=golden_expectations, actual=expectations_path) 191 192 test_invalid_file(file_dir=file_dir, skimage_binary=skimage_binary) 193 194 # Done with all tests. 195 print "Self tests succeeded!" 196 197 if __name__ == "__main__": 198 main() 199
