1 #!/usr/bin/env python 2 # Copyright (C) 2010 Google Inc. All rights reserved. 3 # 4 # Redistribution and use in source and binary forms, with or without 5 # modification, are permitted provided that the following conditions are 6 # met: 7 # 8 # * Redistributions of source code must retain the above copyright 9 # notice, this list of conditions and the following disclaimer. 10 # * Redistributions in binary form must reproduce the above 11 # copyright notice, this list of conditions and the following disclaimer 12 # in the documentation and/or other materials provided with the 13 # distribution. 14 # * Neither the name of Google Inc. nor the names of its 15 # contributors may be used to endorse or promote products derived from 16 # this software without specific prior written permission. 17 # 18 # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 19 # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 20 # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 21 # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 22 # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 23 # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 24 # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 25 # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 26 # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 27 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 28 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 29 30 """Classes for failures that occur during tests.""" 31 32 import os 33 import test_expectations 34 35 36 def determine_result_type(failure_list): 37 """Takes a set of test_failures and returns which result type best fits 38 the list of failures. "Best fits" means we use the worst type of failure. 39 40 Returns: 41 one of the test_expectations result types - PASS, TEXT, CRASH, etc.""" 42 43 if not failure_list or len(failure_list) == 0: 44 return test_expectations.PASS 45 46 failure_types = [type(f) for f in failure_list] 47 if FailureCrash in failure_types: 48 return test_expectations.CRASH 49 elif FailureTimeout in failure_types: 50 return test_expectations.TIMEOUT 51 elif (FailureMissingResult in failure_types or 52 FailureMissingImage in failure_types or 53 FailureMissingImageHash in failure_types): 54 return test_expectations.MISSING 55 else: 56 is_text_failure = FailureTextMismatch in failure_types 57 is_image_failure = (FailureImageHashIncorrect in failure_types or 58 FailureImageHashMismatch in failure_types) 59 if is_text_failure and is_image_failure: 60 return test_expectations.IMAGE_PLUS_TEXT 61 elif is_text_failure: 62 return test_expectations.TEXT 63 elif is_image_failure: 64 return test_expectations.IMAGE 65 else: 66 raise ValueError("unclassifiable set of failures: " 67 + str(failure_types)) 68 69 70 class TestFailure(object): 71 """Abstract base class that defines the failure interface.""" 72 73 @staticmethod 74 def message(): 75 """Returns a string describing the failure in more detail.""" 76 raise NotImplemented 77 78 def result_html_output(self, filename): 79 """Returns an HTML string to be included on the results.html page.""" 80 raise NotImplemented 81 82 def should_kill_test_shell(self): 83 """Returns True if we should kill the test shell before the next 84 test.""" 85 return False 86 87 def relative_output_filename(self, filename, modifier): 88 """Returns a relative filename inside the output dir that contains 89 modifier. 90 91 For example, if filename is fast\dom\foo.html and modifier is 92 "-expected.txt", the return value is fast\dom\foo-expected.txt 93 94 Args: 95 filename: relative filename to test file 96 modifier: a string to replace the extension of filename with 97 98 Return: 99 The relative windows path to the output filename 100 """ 101 return os.path.splitext(filename)[0] + modifier 102 103 104 class FailureWithType(TestFailure): 105 """Base class that produces standard HTML output based on the test type. 106 107 Subclasses may commonly choose to override the ResultHtmlOutput, but still 108 use the standard OutputLinks. 109 """ 110 111 def __init__(self, test_type): 112 TestFailure.__init__(self) 113 # TODO(ojan): This class no longer needs to know the test_type. 114 self._test_type = test_type 115 116 # Filename suffixes used by ResultHtmlOutput. 117 OUT_FILENAMES = [] 118 119 def output_links(self, filename, out_names): 120 """Returns a string holding all applicable output file links. 121 122 Args: 123 filename: the test filename, used to construct the result file names 124 out_names: list of filename suffixes for the files. If three or more 125 suffixes are in the list, they should be [actual, expected, diff, 126 wdiff]. Two suffixes should be [actual, expected], and a 127 single item is the [actual] filename suffix. 128 If out_names is empty, returns the empty string. 129 """ 130 links = [''] 131 uris = [self.relative_output_filename(filename, fn) for 132 fn in out_names] 133 if len(uris) > 1: 134 links.append("<a href='%s'>expected</a>" % uris[1]) 135 if len(uris) > 0: 136 links.append("<a href='%s'>actual</a>" % uris[0]) 137 if len(uris) > 2: 138 links.append("<a href='%s'>diff</a>" % uris[2]) 139 if len(uris) > 3: 140 links.append("<a href='%s'>wdiff</a>" % uris[3]) 141 return ' '.join(links) 142 143 def result_html_output(self, filename): 144 return self.message() + self.output_links(filename, self.OUT_FILENAMES) 145 146 147 class FailureTimeout(TestFailure): 148 """Test timed out. We also want to restart the test shell if this 149 happens.""" 150 151 @staticmethod 152 def message(): 153 return "Test timed out" 154 155 def result_html_output(self, filename): 156 return "<strong>%s</strong>" % self.message() 157 158 def should_kill_test_shell(self): 159 return True 160 161 162 class FailureCrash(TestFailure): 163 """Test shell crashed.""" 164 165 @staticmethod 166 def message(): 167 return "Test shell crashed" 168 169 def result_html_output(self, filename): 170 # TODO(tc): create a link to the minidump file 171 stack = self.relative_output_filename(filename, "-stack.txt") 172 return "<strong>%s</strong> <a href=%s>stack</a>" % (self.message(), 173 stack) 174 175 def should_kill_test_shell(self): 176 return True 177 178 179 class FailureMissingResult(FailureWithType): 180 """Expected result was missing.""" 181 OUT_FILENAMES = ["-actual.txt"] 182 183 @staticmethod 184 def message(): 185 return "No expected results found" 186 187 def result_html_output(self, filename): 188 return ("<strong>%s</strong>" % self.message() + 189 self.output_links(filename, self.OUT_FILENAMES)) 190 191 192 class FailureTextMismatch(FailureWithType): 193 """Text diff output failed.""" 194 # Filename suffixes used by ResultHtmlOutput. 195 OUT_FILENAMES = ["-actual.txt", "-expected.txt", "-diff.txt"] 196 OUT_FILENAMES_WDIFF = ["-actual.txt", "-expected.txt", "-diff.txt", 197 "-wdiff.html"] 198 199 def __init__(self, test_type, has_wdiff): 200 FailureWithType.__init__(self, test_type) 201 if has_wdiff: 202 self.OUT_FILENAMES = self.OUT_FILENAMES_WDIFF 203 204 @staticmethod 205 def message(): 206 return "Text diff mismatch" 207 208 209 class FailureMissingImageHash(FailureWithType): 210 """Actual result hash was missing.""" 211 # Chrome doesn't know to display a .checksum file as text, so don't bother 212 # putting in a link to the actual result. 213 OUT_FILENAMES = [] 214 215 @staticmethod 216 def message(): 217 return "No expected image hash found" 218 219 def result_html_output(self, filename): 220 return "<strong>%s</strong>" % self.message() 221 222 223 class FailureMissingImage(FailureWithType): 224 """Actual result image was missing.""" 225 OUT_FILENAMES = ["-actual.png"] 226 227 @staticmethod 228 def message(): 229 return "No expected image found" 230 231 def result_html_output(self, filename): 232 return ("<strong>%s</strong>" % self.message() + 233 self.output_links(filename, self.OUT_FILENAMES)) 234 235 236 class FailureImageHashMismatch(FailureWithType): 237 """Image hashes didn't match.""" 238 OUT_FILENAMES = ["-actual.png", "-expected.png", "-diff.png"] 239 240 @staticmethod 241 def message(): 242 # We call this a simple image mismatch to avoid confusion, since 243 # we link to the PNGs rather than the checksums. 244 return "Image mismatch" 245 246 247 class FailureFuzzyFailure(FailureWithType): 248 """Image hashes didn't match.""" 249 OUT_FILENAMES = ["-actual.png", "-expected.png"] 250 251 @staticmethod 252 def message(): 253 return "Fuzzy image match also failed" 254 255 256 class FailureImageHashIncorrect(FailureWithType): 257 """Actual result hash is incorrect.""" 258 # Chrome doesn't know to display a .checksum file as text, so don't bother 259 # putting in a link to the actual result. 260 OUT_FILENAMES = [] 261 262 @staticmethod 263 def message(): 264 return "Images match, expected image hash incorrect. " 265 266 def result_html_output(self, filename): 267 return "<strong>%s</strong>" % self.message() 268
