1 # Copyright (c) 2010, Google Inc. All rights reserved. 2 # 3 # Redistribution and use in source and binary forms, with or without 4 # modification, are permitted provided that the following conditions are 5 # met: 6 # 7 # * Redistributions of source code must retain the above copyright 8 # notice, this list of conditions and the following disclaimer. 9 # * Redistributions in binary form must reproduce the above 10 # copyright notice, this list of conditions and the following disclaimer 11 # in the documentation and/or other materials provided with the 12 # distribution. 13 # * Neither the name of Google Inc. nor the names of its 14 # contributors may be used to endorse or promote products derived from 15 # this software without specific prior written permission. 16 # 17 # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 18 # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 19 # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 20 # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 21 # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 22 # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 23 # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 24 # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 25 # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 26 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 27 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 28 # 29 # A module for parsing results.html files generated by old-run-webkit-tests 30 # This class is one big hack and only needs to exist until we transition to new-run-webkit-tests. 31 32 from webkitpy.common.system.deprecated_logging import log 33 from webkitpy.thirdparty.BeautifulSoup import BeautifulSoup, SoupStrainer 34 from webkitpy.layout_tests.layout_package import test_results 35 from webkitpy.layout_tests.layout_package import test_failures 36 37 38 # FIXME: This should be unified with all the layout test results code in the layout_tests package 39 # This doesn't belong in common.net, but we don't have a better place for it yet. 40 def path_for_layout_test(test_name): 41 return "LayoutTests/%s" % test_name 42 43 44 # FIXME: This should be unified with all the layout test results code in the layout_tests package 45 # This doesn't belong in common.net, but we don't have a better place for it yet. 46 class LayoutTestResults(object): 47 """This class knows how to parse old-run-webkit-tests results.html files.""" 48 49 stderr_key = u'Tests that had stderr output:' 50 fail_key = u'Tests where results did not match expected results:' 51 timeout_key = u'Tests that timed out:' 52 crash_key = u'Tests that caused the DumpRenderTree tool to crash:' 53 missing_key = u'Tests that had no expected results (probably new):' 54 webprocess_crash_key = u'Tests that caused the Web process to crash:' 55 56 expected_keys = [ 57 stderr_key, 58 fail_key, 59 crash_key, 60 webprocess_crash_key, 61 timeout_key, 62 missing_key, 63 ] 64 65 @classmethod 66 def _failures_from_fail_row(self, row): 67 # Look at all anchors in this row, and guess what type 68 # of new-run-webkit-test failures they equate to. 69 failures = set() 70 test_name = None 71 for anchor in row.findAll("a"): 72 anchor_text = unicode(anchor.string) 73 if not test_name: 74 test_name = anchor_text 75 continue 76 if anchor_text in ["expected image", "image diffs"] or '%' in anchor_text: 77 failures.add(test_failures.FailureImageHashMismatch()) 78 elif anchor_text in ["expected", "actual", "diff", "pretty diff"]: 79 failures.add(test_failures.FailureTextMismatch()) 80 else: 81 log("Unhandled link text in results.html parsing: %s. Please file a bug against webkitpy." % anchor_text) 82 # FIXME: Its possible the row contained no links due to ORWT brokeness. 83 # We should probably assume some type of failure anyway. 84 return failures 85 86 @classmethod 87 def _failures_from_row(cls, row, table_title): 88 if table_title == cls.fail_key: 89 return cls._failures_from_fail_row(row) 90 if table_title == cls.crash_key: 91 return [test_failures.FailureCrash()] 92 if table_title == cls.webprocess_crash_key: 93 return [test_failures.FailureCrash()] 94 if table_title == cls.timeout_key: 95 return [test_failures.FailureTimeout()] 96 if table_title == cls.missing_key: 97 return [test_failures.FailureMissingResult(), test_failures.FailureMissingImageHash(), test_failures.FailureMissingImage()] 98 return None 99 100 @classmethod 101 def _test_result_from_row(cls, row, table_title): 102 test_name = unicode(row.find("a").string) 103 failures = cls._failures_from_row(row, table_title) 104 # TestResult is a class designed to work with new-run-webkit-tests. 105 # old-run-webkit-tests does not save quite enough information in results.html for us to parse. 106 # FIXME: It's unclear if test_name should include LayoutTests or not. 107 return test_results.TestResult(test_name, failures) 108 109 @classmethod 110 def _parse_results_table(cls, table): 111 table_title = unicode(table.findPreviousSibling("p").string) 112 if table_title not in cls.expected_keys: 113 # This Exception should only ever be hit if run-webkit-tests changes its results.html format. 114 raise Exception("Unhandled title: %s" % table_title) 115 # Ignore stderr failures. Everyone ignores them anyway. 116 if table_title == cls.stderr_key: 117 return [] 118 # FIXME: We might end with two TestResults object for the same test if it appears in more than one row. 119 return [cls._test_result_from_row(row, table_title) for row in table.findAll("tr")] 120 121 @classmethod 122 def _parse_results_html(cls, page): 123 tables = BeautifulSoup(page).findAll("table") 124 return sum([cls._parse_results_table(table) for table in tables], []) 125 126 @classmethod 127 def results_from_string(cls, string): 128 if not string: 129 return None 130 test_results = cls._parse_results_html(string) 131 if not test_results: 132 return None 133 return cls(test_results) 134 135 def __init__(self, test_results): 136 self._test_results = test_results 137 self._failure_limit_count = None 138 139 # FIXME: run-webkit-tests should store the --exit-after-N-failures value 140 # (or some indication of early exit) somewhere in the results.html/results.json 141 # file. Until it does, callers should set the limit to 142 # --exit-after-N-failures value used in that run. Consumers of LayoutTestResults 143 # may use that value to know if absence from the failure list means PASS. 144 # https://bugs.webkit.org/show_bug.cgi?id=58481 145 def set_failure_limit_count(self, limit): 146 self._failure_limit_count = limit 147 148 def failure_limit_count(self): 149 return self._failure_limit_count 150 151 def test_results(self): 152 return self._test_results 153 154 def results_matching_failure_types(self, failure_types): 155 return [result for result in self._test_results if result.has_failure_matching_types(failure_types)] 156 157 def tests_matching_failure_types(self, failure_types): 158 return [result.filename for result in self.results_matching_failure_types(failure_types)] 159 160 def failing_test_results(self): 161 # These should match the "fail", "crash", and "timeout" keys. 162 failure_types = [test_failures.FailureTextMismatch, test_failures.FailureImageHashMismatch, test_failures.FailureCrash, test_failures.FailureTimeout] 163 return self.results_matching_failure_types(failure_types) 164 165 def failing_tests(self): 166 return [result.filename for result in self.failing_test_results()] 167
