1 #!/usr/bin/env python 2 # Copyright (C) 2010 Google Inc. All rights reserved. 3 # 4 # Redistribution and use in source and binary forms, with or without 5 # modification, are permitted provided that the following conditions are 6 # met: 7 # 8 # * Redistributions of source code must retain the above copyright 9 # notice, this list of conditions and the following disclaimer. 10 # * Redistributions in binary form must reproduce the above 11 # copyright notice, this list of conditions and the following disclaimer 12 # in the documentation and/or other materials provided with the 13 # distribution. 14 # * Neither the name of Google Inc. nor the names of its 15 # contributors may be used to endorse or promote products derived from 16 # this software without specific prior written permission. 17 # 18 # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 19 # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 20 # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 21 # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 22 # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 23 # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 24 # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 25 # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 26 # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 27 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 28 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 29 30 """Classes for failures that occur during tests.""" 31 32 import test_expectations 33 34 import cPickle 35 36 37 # FIXME: This is backwards. Each TestFailure subclass should know what 38 # test_expectation type it corresponds too. Then this method just 39 # collects them all from the failure list and returns the worst one. 40 def determine_result_type(failure_list): 41 """Takes a set of test_failures and returns which result type best fits 42 the list of failures. "Best fits" means we use the worst type of failure. 43 44 Returns: 45 one of the test_expectations result types - PASS, TEXT, CRASH, etc.""" 46 47 if not failure_list or len(failure_list) == 0: 48 return test_expectations.PASS 49 50 failure_types = [type(f) for f in failure_list] 51 if FailureCrash in failure_types: 52 return test_expectations.CRASH 53 elif FailureTimeout in failure_types: 54 return test_expectations.TIMEOUT 55 elif (FailureMissingResult in failure_types or 56 FailureMissingImage in failure_types or 57 FailureMissingImageHash in failure_types or 58 FailureMissingAudio in failure_types): 59 return test_expectations.MISSING 60 else: 61 is_text_failure = FailureTextMismatch in failure_types 62 is_image_failure = (FailureImageHashIncorrect in failure_types or 63 FailureImageHashMismatch in failure_types) 64 is_reftest_failure = (FailureReftestMismatch in failure_types or 65 FailureReftestMismatchDidNotOccur in failure_types) 66 is_audio_failure = (FailureAudioMismatch in failure_types) 67 if is_text_failure and is_image_failure: 68 return test_expectations.IMAGE_PLUS_TEXT 69 elif is_text_failure: 70 return test_expectations.TEXT 71 elif is_image_failure or is_reftest_failure: 72 return test_expectations.IMAGE 73 elif is_audio_failure: 74 return test_expectations.AUDIO 75 else: 76 raise ValueError("unclassifiable set of failures: " 77 + str(failure_types)) 78 79 80 class TestFailure(object): 81 """Abstract base class that defines the failure interface.""" 82 83 @staticmethod 84 def loads(s): 85 """Creates a TestFailure object from the specified string.""" 86 return cPickle.loads(s) 87 88 @staticmethod 89 def message(): 90 """Returns a string describing the failure in more detail.""" 91 raise NotImplementedError 92 93 def __eq__(self, other): 94 return self.__class__.__name__ == other.__class__.__name__ 95 96 def __ne__(self, other): 97 return self.__class__.__name__ != other.__class__.__name__ 98 99 def __hash__(self): 100 return hash(self.__class__.__name__) 101 102 def dumps(self): 103 """Returns the string/JSON representation of a TestFailure.""" 104 return cPickle.dumps(self) 105 106 def should_kill_dump_render_tree(self): 107 """Returns True if we should kill DumpRenderTree before the next 108 test.""" 109 return False 110 111 112 class FailureTimeout(TestFailure): 113 """Test timed out. We also want to restart DumpRenderTree if this 114 happens.""" 115 def __init__(self, is_reftest=False): 116 self.is_reftest = is_reftest 117 118 @staticmethod 119 def message(): 120 return "Test timed out" 121 122 def should_kill_dump_render_tree(self): 123 return True 124 125 126 class FailureCrash(TestFailure): 127 """DumpRenderTree crashed.""" 128 def __init__(self, is_reftest=False): 129 self.is_reftest = is_reftest 130 131 @staticmethod 132 def message(): 133 return "DumpRenderTree crashed" 134 135 def should_kill_dump_render_tree(self): 136 return True 137 138 139 class FailureMissingResult(TestFailure): 140 """Expected result was missing.""" 141 142 @staticmethod 143 def message(): 144 return "No expected results found" 145 146 147 class FailureTextMismatch(TestFailure): 148 """Text diff output failed.""" 149 150 @staticmethod 151 def message(): 152 return "Text diff mismatch" 153 154 155 class FailureMissingImageHash(TestFailure): 156 """Actual result hash was missing.""" 157 # Chrome doesn't know to display a .checksum file as text, so don't bother 158 # putting in a link to the actual result. 159 160 @staticmethod 161 def message(): 162 return "No expected image hash found" 163 164 165 class FailureMissingImage(TestFailure): 166 """Actual result image was missing.""" 167 168 @staticmethod 169 def message(): 170 return "No expected image found" 171 172 173 class FailureImageHashMismatch(TestFailure): 174 """Image hashes didn't match.""" 175 176 @staticmethod 177 def message(): 178 # We call this a simple image mismatch to avoid confusion, since 179 # we link to the PNGs rather than the checksums. 180 return "Image mismatch" 181 182 183 class FailureImageHashIncorrect(TestFailure): 184 """Actual result hash is incorrect.""" 185 # Chrome doesn't know to display a .checksum file as text, so don't bother 186 # putting in a link to the actual result. 187 188 @staticmethod 189 def message(): 190 return "Images match, expected image hash incorrect. " 191 192 193 class FailureReftestMismatch(TestFailure): 194 """The result didn't match the reference rendering.""" 195 196 @staticmethod 197 def message(): 198 return "Mismatch with reference" 199 200 201 class FailureReftestMismatchDidNotOccur(TestFailure): 202 """Unexpected match between the result and the reference rendering.""" 203 204 @staticmethod 205 def message(): 206 return "Mismatch with the reference did not occur" 207 208 209 class FailureMissingAudio(TestFailure): 210 """Actual result image was missing.""" 211 212 @staticmethod 213 def message(): 214 return "No expected audio found" 215 216 217 class FailureAudioMismatch(TestFailure): 218 """Audio files didn't match.""" 219 220 @staticmethod 221 def message(): 222 return "Audio mismatch" 223 224 225 # Convenient collection of all failure classes for anything that might 226 # need to enumerate over them all. 227 ALL_FAILURE_CLASSES = (FailureTimeout, FailureCrash, FailureMissingResult, 228 FailureTextMismatch, FailureMissingImageHash, 229 FailureMissingImage, FailureImageHashMismatch, 230 FailureImageHashIncorrect, FailureReftestMismatch, 231 FailureReftestMismatchDidNotOccur) 232
