1 # Copyright (C) 2010 Google Inc. All rights reserved. 2 # 3 # Redistribution and use in source and binary forms, with or without 4 # modification, are permitted provided that the following conditions are 5 # met: 6 # 7 # * Redistributions of source code must retain the above copyright 8 # notice, this list of conditions and the following disclaimer. 9 # * Redistributions in binary form must reproduce the above 10 # copyright notice, this list of conditions and the following disclaimer 11 # in the documentation and/or other materials provided with the 12 # distribution. 13 # * Neither the name of Google Inc. nor the names of its 14 # contributors may be used to endorse or promote products derived from 15 # this software without specific prior written permission. 16 # 17 # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 18 # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 19 # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 20 # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 21 # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 22 # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 23 # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 24 # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 25 # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 26 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 27 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 28 29 import cPickle 30 31 from webkitpy.layout_tests.models import test_expectations 32 33 34 def is_reftest_failure(failure_list): 35 failure_types = [type(f) for f in failure_list] 36 return set((FailureReftestMismatch, FailureReftestMismatchDidNotOccur, FailureReftestNoImagesGenerated)).intersection(failure_types) 37 38 # FIXME: This is backwards. Each TestFailure subclass should know what 39 # test_expectation type it corresponds too. Then this method just 40 # collects them all from the failure list and returns the worst one. 41 def determine_result_type(failure_list): 42 """Takes a set of test_failures and returns which result type best fits 43 the list of failures. "Best fits" means we use the worst type of failure. 44 45 Returns: 46 one of the test_expectations result types - PASS, FAIL, CRASH, etc.""" 47 48 if not failure_list or len(failure_list) == 0: 49 return test_expectations.PASS 50 51 failure_types = [type(f) for f in failure_list] 52 if FailureCrash in failure_types: 53 return test_expectations.CRASH 54 elif FailureLeak in failure_types: 55 return test_expectations.LEAK 56 elif FailureTimeout in failure_types: 57 return test_expectations.TIMEOUT 58 elif FailureEarlyExit in failure_types: 59 return test_expectations.SKIP 60 elif (FailureMissingResult in failure_types or 61 FailureMissingImage in failure_types or 62 FailureMissingImageHash in failure_types or 63 FailureMissingAudio in failure_types): 64 return test_expectations.MISSING 65 else: 66 is_text_failure = (FailureTextMismatch in failure_types or 67 FailureTestHarnessAssertion in failure_types) 68 is_image_failure = (FailureImageHashIncorrect in failure_types or 69 FailureImageHashMismatch in failure_types) 70 is_audio_failure = (FailureAudioMismatch in failure_types) 71 if is_text_failure and is_image_failure: 72 return test_expectations.IMAGE_PLUS_TEXT 73 elif is_text_failure: 74 return test_expectations.TEXT 75 elif is_image_failure or is_reftest_failure(failure_list): 76 return test_expectations.IMAGE 77 elif is_audio_failure: 78 return test_expectations.AUDIO 79 else: 80 raise ValueError("unclassifiable set of failures: " 81 + str(failure_types)) 82 83 84 class TestFailure(object): 85 """Abstract base class that defines the failure interface.""" 86 87 @staticmethod 88 def loads(s): 89 """Creates a TestFailure object from the specified string.""" 90 return cPickle.loads(s) 91 92 def message(self): 93 """Returns a string describing the failure in more detail.""" 94 raise NotImplementedError 95 96 def __eq__(self, other): 97 return self.__class__.__name__ == other.__class__.__name__ 98 99 def __ne__(self, other): 100 return self.__class__.__name__ != other.__class__.__name__ 101 102 def __hash__(self): 103 return hash(self.__class__.__name__) 104 105 def dumps(self): 106 """Returns the string/JSON representation of a TestFailure.""" 107 return cPickle.dumps(self) 108 109 def driver_needs_restart(self): 110 """Returns True if we should kill the driver before the next test.""" 111 return False 112 113 114 class FailureTimeout(TestFailure): 115 def __init__(self, is_reftest=False): 116 super(FailureTimeout, self).__init__() 117 self.is_reftest = is_reftest 118 119 def message(self): 120 return "test timed out" 121 122 def driver_needs_restart(self): 123 return True 124 125 126 class FailureCrash(TestFailure): 127 def __init__(self, is_reftest=False, process_name='content_shell', pid=None, has_log=False): 128 super(FailureCrash, self).__init__() 129 self.process_name = process_name 130 self.pid = pid 131 self.is_reftest = is_reftest 132 self.has_log = has_log 133 134 def message(self): 135 if self.pid: 136 return "%s crashed [pid=%d]" % (self.process_name, self.pid) 137 return self.process_name + " crashed" 138 139 def driver_needs_restart(self): 140 return True 141 142 143 class FailureLeak(TestFailure): 144 def __init__(self, is_reftest=False, log=''): 145 super(FailureLeak, self).__init__() 146 self.is_reftest = is_reftest 147 self.log = log 148 149 def message(self): 150 return "leak detected: %s" % (self.log) 151 152 153 class FailureMissingResult(TestFailure): 154 def message(self): 155 return "-expected.txt was missing" 156 157 158 class FailureTestHarnessAssertion(TestFailure): 159 def message(self): 160 return "asserts failed" 161 162 163 class FailureTextMismatch(TestFailure): 164 def message(self): 165 return "text diff" 166 167 168 class FailureMissingImageHash(TestFailure): 169 def message(self): 170 return "-expected.png was missing an embedded checksum" 171 172 173 class FailureMissingImage(TestFailure): 174 def message(self): 175 return "-expected.png was missing" 176 177 178 class FailureImageHashMismatch(TestFailure): 179 def message(self): 180 return "image diff" 181 182 183 class FailureImageHashIncorrect(TestFailure): 184 def message(self): 185 return "-expected.png embedded checksum is incorrect" 186 187 188 class FailureReftestMismatch(TestFailure): 189 def __init__(self, reference_filename=None): 190 super(FailureReftestMismatch, self).__init__() 191 self.reference_filename = reference_filename 192 193 def message(self): 194 return "reference mismatch" 195 196 197 class FailureReftestMismatchDidNotOccur(TestFailure): 198 def __init__(self, reference_filename=None): 199 super(FailureReftestMismatchDidNotOccur, self).__init__() 200 self.reference_filename = reference_filename 201 202 def message(self): 203 return "reference mismatch didn't happen" 204 205 206 class FailureReftestNoImagesGenerated(TestFailure): 207 def __init__(self, reference_filename=None): 208 super(FailureReftestNoImagesGenerated, self).__init__() 209 self.reference_filename = reference_filename 210 211 def message(self): 212 return "reference didn't generate pixel results." 213 214 215 class FailureMissingAudio(TestFailure): 216 def message(self): 217 return "expected audio result was missing" 218 219 220 class FailureAudioMismatch(TestFailure): 221 def message(self): 222 return "audio mismatch" 223 224 225 class FailureEarlyExit(TestFailure): 226 def message(self): 227 return "skipped due to early exit" 228 229 230 # Convenient collection of all failure classes for anything that might 231 # need to enumerate over them all. 232 ALL_FAILURE_CLASSES = (FailureTimeout, FailureCrash, FailureMissingResult, 233 FailureTestHarnessAssertion, 234 FailureTextMismatch, FailureMissingImageHash, 235 FailureMissingImage, FailureImageHashMismatch, 236 FailureImageHashIncorrect, FailureReftestMismatch, 237 FailureReftestMismatchDidNotOccur, FailureReftestNoImagesGenerated, 238 FailureMissingAudio, FailureAudioMismatch, 239 FailureEarlyExit) 240