1 #!/usr/bin/env python 2 # Copyright (c) 2013 The Chromium Authors. All rights reserved. 3 # Use of this source code is governed by a BSD-style license that can be 4 # found in the LICENSE file. 5 6 # Self-test for skimage. 7 8 import filecmp 9 import os 10 import subprocess 11 import sys 12 import tempfile 13 14 class BinaryNotFoundException(Exception): 15 def __str__ (self): 16 return ("Could not find binary!\n" 17 "Did you forget to build the tools project?\n" 18 "Self tests failed") 19 20 # Find a path to the binary to use. Iterates through a list of possible 21 # locations the binary may be. 22 def PickBinaryPath(base_dir): 23 POSSIBLE_BINARY_PATHS = [ 24 'out/Debug/skimage', 25 'out/Release/skimage', 26 'xcodebuild/Debug/skimage', 27 'xcodebuild/Release/skimage', 28 ] 29 for binary in POSSIBLE_BINARY_PATHS: 30 binary_full_path = os.path.join(base_dir, binary) 31 if (os.path.exists(binary_full_path)): 32 return binary_full_path 33 raise BinaryNotFoundException 34 35 # Quit early if two files have different content. 36 def DieIfFilesMismatch(expected, actual): 37 if not filecmp.cmp(expected, actual): 38 raise Exception("Error: file mismatch! expected=%s , actual=%s" % ( 39 expected, actual)) 40 41 def test_invalid_file(file_dir, skimage_binary): 42 """ Test the return value of skimage when an invalid file is decoded. 43 If there is no expectation file, or the file expects a particular 44 result, skimage should return nonzero indicating failure. 45 If the file has no expectation, or ignore-failure is set to true, 46 skimage should return zero indicating success. """ 47 invalid_file = os.path.join(file_dir, "skimage", "input", "bad-images", 48 "invalid.png") 49 # No expectations file: 50 args = [skimage_binary, "--readPath", invalid_file] 51 result = subprocess.call(args) 52 if 0 == result: 53 raise Exception("'%s' should have reported failure!" % " ".join(args)) 54 55 # Directory holding all expectations files 56 expectations_dir = os.path.join(file_dir, "skimage", "input", "bad-images") 57 58 # Expectations file expecting a valid decode: 59 incorrect_expectations = os.path.join(expectations_dir, 60 "incorrect-results.json") 61 args = [skimage_binary, "--readPath", invalid_file, 62 "--readExpectationsPath", incorrect_expectations] 63 result = subprocess.call(args) 64 if 0 == result: 65 raise Exception("'%s' should have reported failure!" % " ".join(args)) 66 67 # Empty expectations: 68 empty_expectations = os.path.join(expectations_dir, "empty-results.json") 69 output = subprocess.check_output([skimage_binary, "--readPath", invalid_file, 70 "--readExpectationsPath", 71 empty_expectations], 72 stderr=subprocess.STDOUT) 73 if not "Missing" in output: 74 # Another test (in main()) tests to ensure that "Missing" does not appear 75 # in the output. That test could be passed if the output changed so 76 # "Missing" never appears. This ensures that an error is not missed if 77 # that happens. 78 raise Exception( 79 "skimage output changed! This may cause other self tests to fail!") 80 81 # Ignore failure: 82 ignore_expectations = os.path.join(expectations_dir, "ignore-results.json") 83 output = subprocess.check_output([skimage_binary, "--readPath", invalid_file, 84 "--readExpectationsPath", 85 ignore_expectations], 86 stderr=subprocess.STDOUT) 87 if not "failures" in output: 88 # Another test (in main()) tests to ensure that "failures" does not 89 # appear in the output. That test could be passed if the output changed 90 # so "failures" never appears. This ensures that an error is not missed 91 # if that happens. 92 raise Exception( 93 "skimage output changed! This may cause other self tests to fail!") 94 95 def test_incorrect_expectations(file_dir, skimage_binary): 96 """ Test that comparing to incorrect expectations fails, unless 97 ignore-failures is set to true. """ 98 valid_file = os.path.join(file_dir, "skimage", "input", 99 "images-with-known-hashes", 100 "1209453360120438698.png") 101 expectations_dir = os.path.join(file_dir, "skimage", "input", 102 "images-with-known-hashes") 103 104 incorrect_results = os.path.join(expectations_dir, 105 "incorrect-results.json") 106 args = [skimage_binary, "--readPath", valid_file, "--readExpectationsPath", 107 incorrect_results] 108 result = subprocess.call(args) 109 if 0 == result: 110 raise Exception("'%s' should have reported failure!" % " ".join(args)) 111 112 ignore_results = os.path.join(expectations_dir, "ignore-failures.json") 113 subprocess.check_call([skimage_binary, "--readPath", valid_file, 114 "--readExpectationsPath", ignore_results]) 115 116 def main(): 117 # Use the directory of this file as the out directory 118 file_dir = os.path.abspath(os.path.dirname(__file__)) 119 120 trunk_dir = os.path.normpath(os.path.join(file_dir, os.pardir, os.pardir)) 121 122 # Find the binary 123 skimage_binary = PickBinaryPath(trunk_dir) 124 print "Running " + skimage_binary 125 126 # Generate an expectations file from known images. 127 images_dir = os.path.join(file_dir, "skimage", "input", 128 "images-with-known-hashes") 129 expectations_path = os.path.join(file_dir, "skimage", "output-actual", 130 "create-expectations", "expectations.json") 131 subprocess.check_call([skimage_binary, "--readPath", images_dir, 132 "--createExpectationsPath", expectations_path]) 133 134 # Make sure the expectations file was generated correctly. 135 golden_expectations = os.path.join(file_dir, "skimage", "output-expected", 136 "create-expectations", 137 "expectations.json") 138 DieIfFilesMismatch(expected=golden_expectations, actual=expectations_path) 139 140 # Tell skimage to read back the expectations file it just wrote, and 141 # confirm that the images in images_dir match it. 142 output = subprocess.check_output([skimage_binary, "--readPath", images_dir, 143 "--readExpectationsPath", 144 expectations_path], 145 stderr=subprocess.STDOUT) 146 147 # Although skimage succeeded, it would have reported success if the file 148 # was missing from the expectations file. Consider this a failure, since 149 # the expectations file was created from this same image. (It will print 150 # "Missing" in this case before listing the missing expectations). 151 if "Missing" in output: 152 raise Exception("Expectations file was missing expectations: %s" % output) 153 154 # Again, skimage would succeed if there were known failures (and print 155 # "failures"), but there should be no failures, since the file just 156 # created did not include failures to ignore. 157 if "failures" in output: 158 raise Exception("Image failed: %s" % output) 159 160 161 test_incorrect_expectations(file_dir=file_dir, 162 skimage_binary=skimage_binary) 163 164 # Generate an expectations file from an empty directory. 165 empty_dir = tempfile.mkdtemp() 166 expectations_path = os.path.join(file_dir, "skimage", "output-actual", 167 "empty-dir", "expectations.json") 168 subprocess.check_call([skimage_binary, "--readPath", empty_dir, 169 "--createExpectationsPath", expectations_path]) 170 golden_expectations = os.path.join(file_dir, "skimage", "output-expected", 171 "empty-dir", "expectations.json") 172 DieIfFilesMismatch(expected=golden_expectations, actual=expectations_path) 173 os.rmdir(empty_dir) 174 175 # Generate an expectations file from a nonexistent directory. 176 expectations_path = os.path.join(file_dir, "skimage", "output-actual", 177 "nonexistent-dir", "expectations.json") 178 subprocess.check_call([skimage_binary, "--readPath", "/nonexistent/dir", 179 "--createExpectationsPath", expectations_path]) 180 golden_expectations = os.path.join(file_dir, "skimage", "output-expected", 181 "nonexistent-dir", "expectations.json") 182 DieIfFilesMismatch(expected=golden_expectations, actual=expectations_path) 183 184 test_invalid_file(file_dir=file_dir, skimage_binary=skimage_binary) 185 186 # Done with all tests. 187 print "Self tests succeeded!" 188 189 if __name__ == "__main__": 190 main() 191
