1 # Copyright (C) 2010 Google Inc. All rights reserved. 2 # 3 # Redistribution and use in source and binary forms, with or without 4 # modification, are permitted provided that the following conditions are 5 # met: 6 # 7 # * Redistributions of source code must retain the above copyright 8 # notice, this list of conditions and the following disclaimer. 9 # * Redistributions in binary form must reproduce the above 10 # copyright notice, this list of conditions and the following disclaimer 11 # in the documentation and/or other materials provided with the 12 # distribution. 13 # * Neither the name of Google Inc. nor the names of its 14 # contributors may be used to endorse or promote products derived from 15 # this software without specific prior written permission. 16 # 17 # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 18 # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 19 # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 20 # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 21 # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 22 # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 23 # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 24 # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 25 # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 26 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 27 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 28 29 import unittest 30 31 from webkitpy.common.host_mock import MockHost 32 from webkitpy.layout_tests.models import test_expectations 33 from webkitpy.layout_tests.models import test_failures 34 from webkitpy.layout_tests.models import test_results 35 from webkitpy.layout_tests.models import test_run_results 36 37 38 def get_result(test_name, result_type=test_expectations.PASS, run_time=0): 39 failures = [] 40 if result_type == test_expectations.TIMEOUT: 41 failures = [test_failures.FailureTimeout()] 42 elif result_type == test_expectations.AUDIO: 43 failures = [test_failures.FailureAudioMismatch()] 44 elif result_type == test_expectations.CRASH: 45 failures = [test_failures.FailureCrash()] 46 elif result_type == test_expectations.LEAK: 47 failures = [test_failures.FailureLeak()] 48 return test_results.TestResult(test_name, failures=failures, test_run_time=run_time) 49 50 51 def run_results(port, extra_skipped_tests=[]): 52 tests = ['passes/text.html', 'failures/expected/timeout.html', 'failures/expected/crash.html', 'failures/expected/leak.html', 'failures/expected/keyboard.html', 53 'failures/expected/audio.html', 'passes/skipped/skip.html'] 54 expectations = test_expectations.TestExpectations(port, tests) 55 if extra_skipped_tests: 56 expectations.add_extra_skipped_tests(extra_skipped_tests) 57 return test_run_results.TestRunResults(expectations, len(tests)) 58 59 60 def summarized_results(port, expected, passing, flaky, only_include_failing=False, extra_skipped_tests=[]): 61 test_is_slow = False 62 63 initial_results = run_results(port, extra_skipped_tests) 64 if expected: 65 initial_results.add(get_result('passes/text.html', test_expectations.PASS), expected, test_is_slow) 66 initial_results.add(get_result('failures/expected/audio.html', test_expectations.AUDIO), expected, test_is_slow) 67 initial_results.add(get_result('failures/expected/timeout.html', test_expectations.TIMEOUT), expected, test_is_slow) 68 initial_results.add(get_result('failures/expected/crash.html', test_expectations.CRASH), expected, test_is_slow) 69 initial_results.add(get_result('failures/expected/leak.html', test_expectations.LEAK), expected, test_is_slow) 70 elif passing: 71 skipped_result = get_result('passes/skipped/skip.html') 72 skipped_result.type = test_expectations.SKIP 73 initial_results.add(skipped_result, expected, test_is_slow) 74 75 initial_results.add(get_result('passes/text.html', run_time=1), expected, test_is_slow) 76 initial_results.add(get_result('failures/expected/audio.html'), expected, test_is_slow) 77 initial_results.add(get_result('failures/expected/timeout.html'), expected, test_is_slow) 78 initial_results.add(get_result('failures/expected/crash.html'), expected, test_is_slow) 79 initial_results.add(get_result('failures/expected/leak.html'), expected, test_is_slow) 80 else: 81 initial_results.add(get_result('passes/text.html', test_expectations.TIMEOUT, run_time=1), expected, test_is_slow) 82 initial_results.add(get_result('failures/expected/audio.html', test_expectations.AUDIO, run_time=0.049), expected, test_is_slow) 83 initial_results.add(get_result('failures/expected/timeout.html', test_expectations.CRASH, run_time=0.05), expected, test_is_slow) 84 initial_results.add(get_result('failures/expected/crash.html', test_expectations.TIMEOUT), expected, test_is_slow) 85 initial_results.add(get_result('failures/expected/leak.html', test_expectations.TIMEOUT), expected, test_is_slow) 86 87 # we only list keyboard.html here, since normally this is WontFix 88 initial_results.add(get_result('failures/expected/keyboard.html', test_expectations.SKIP), expected, test_is_slow) 89 90 if flaky: 91 retry_results = run_results(port, extra_skipped_tests) 92 retry_results.add(get_result('passes/text.html'), True, test_is_slow) 93 retry_results.add(get_result('failures/expected/timeout.html'), True, test_is_slow) 94 retry_results.add(get_result('failures/expected/crash.html'), True, test_is_slow) 95 retry_results.add(get_result('failures/expected/leak.html'), True, test_is_slow) 96 else: 97 retry_results = None 98 99 return test_run_results.summarize_results(port, initial_results.expectations, initial_results, retry_results, enabled_pixel_tests_in_retry=False, only_include_failing=only_include_failing) 100 101 102 class InterpretTestFailuresTest(unittest.TestCase): 103 def setUp(self): 104 host = MockHost() 105 self.port = host.port_factory.get(port_name='test') 106 107 def test_interpret_test_failures(self): 108 test_dict = test_run_results._interpret_test_failures([test_failures.FailureReftestMismatchDidNotOccur(self.port.abspath_for_test('foo/reftest-expected-mismatch.html'))]) 109 self.assertEqual(len(test_dict), 0) 110 111 test_dict = test_run_results._interpret_test_failures([test_failures.FailureMissingAudio()]) 112 self.assertIn('is_missing_audio', test_dict) 113 114 test_dict = test_run_results._interpret_test_failures([test_failures.FailureMissingResult()]) 115 self.assertIn('is_missing_text', test_dict) 116 117 test_dict = test_run_results._interpret_test_failures([test_failures.FailureMissingImage()]) 118 self.assertIn('is_missing_image', test_dict) 119 120 test_dict = test_run_results._interpret_test_failures([test_failures.FailureMissingImageHash()]) 121 self.assertIn('is_missing_image', test_dict) 122 123 124 class SummarizedResultsTest(unittest.TestCase): 125 def setUp(self): 126 host = MockHost(initialize_scm_by_default=False) 127 self.port = host.port_factory.get(port_name='test') 128 129 def test_no_svn_revision(self): 130 summary = summarized_results(self.port, expected=False, passing=False, flaky=False) 131 self.assertNotIn('revision', summary) 132 133 def test_num_failures_by_type(self): 134 summary = summarized_results(self.port, expected=False, passing=False, flaky=False) 135 self.assertEquals(summary['num_failures_by_type'], {'CRASH': 1, 'MISSING': 0, 'TEXT': 0, 'IMAGE': 0, 'NEEDSREBASELINE': 0, 'NEEDSMANUALREBASELINE': 0, 'PASS': 0, 'REBASELINE': 0, 'SKIP': 0, 'SLOW': 0, 'TIMEOUT': 3, 'IMAGE+TEXT': 0, 'LEAK': 0, 'FAIL': 0, 'AUDIO': 1, 'WONTFIX': 1}) 136 137 summary = summarized_results(self.port, expected=True, passing=False, flaky=False) 138 self.assertEquals(summary['num_failures_by_type'], {'CRASH': 1, 'MISSING': 0, 'TEXT': 0, 'IMAGE': 0, 'NEEDSREBASELINE': 0, 'NEEDSMANUALREBASELINE': 0, 'PASS': 1, 'REBASELINE': 0, 'SKIP': 0, 'SLOW': 0, 'TIMEOUT': 1, 'IMAGE+TEXT': 0, 'LEAK': 1, 'FAIL': 0, 'AUDIO': 1, 'WONTFIX': 0}) 139 140 summary = summarized_results(self.port, expected=False, passing=True, flaky=False) 141 self.assertEquals(summary['num_failures_by_type'], {'CRASH': 0, 'MISSING': 0, 'TEXT': 0, 'IMAGE': 0, 'NEEDSREBASELINE': 0, 'NEEDSMANUALREBASELINE': 0, 'PASS': 5, 'REBASELINE': 0, 'SKIP': 1, 'SLOW': 0, 'TIMEOUT': 0, 'IMAGE+TEXT': 0, 'LEAK': 0, 'FAIL': 0, 'AUDIO': 0, 'WONTFIX': 0}) 142 143 def test_svn_revision(self): 144 self.port._options.builder_name = 'dummy builder' 145 summary = summarized_results(self.port, expected=False, passing=False, flaky=False) 146 self.assertNotEquals(summary['blink_revision'], '') 147 148 def test_bug_entry(self): 149 self.port._options.builder_name = 'dummy builder' 150 summary = summarized_results(self.port, expected=False, passing=True, flaky=False) 151 self.assertEquals(summary['tests']['passes']['skipped']['skip.html']['bugs'], ['Bug(test)']) 152 153 def test_extra_skipped_tests(self): 154 self.port._options.builder_name = 'dummy builder' 155 summary = summarized_results(self.port, expected=False, passing=True, flaky=False, extra_skipped_tests=['passes/text.html']) 156 self.assertEquals(summary['tests']['passes']['text.html']['expected'], 'NOTRUN') 157 158 def test_summarized_results_wontfix(self): 159 self.port._options.builder_name = 'dummy builder' 160 summary = summarized_results(self.port, expected=False, passing=False, flaky=False) 161 self.assertEquals(summary['tests']['failures']['expected']['keyboard.html']['expected'], 'WONTFIX') 162 self.assertTrue(summary['tests']['passes']['text.html']['is_unexpected']) 163 164 def test_summarized_results_expected_pass(self): 165 self.port._options.builder_name = 'dummy builder' 166 summary = summarized_results(self.port, expected=False, passing=True, flaky=False) 167 self.assertTrue(summary['tests']['passes']['text.html']) 168 self.assertTrue('is_unexpected' not in summary['tests']['passes']['text.html']) 169 170 def test_summarized_results_expected_only_include_failing(self): 171 self.port._options.builder_name = 'dummy builder' 172 summary = summarized_results(self.port, expected=True, passing=False, flaky=False, only_include_failing=True) 173 self.assertNotIn('passes', summary['tests']) 174 self.assertTrue(summary['tests']['failures']['expected']['audio.html']) 175 self.assertTrue(summary['tests']['failures']['expected']['timeout.html']) 176 self.assertTrue(summary['tests']['failures']['expected']['crash.html']) 177 self.assertTrue(summary['tests']['failures']['expected']['leak.html']) 178 179 def test_summarized_results_skipped(self): 180 self.port._options.builder_name = 'dummy builder' 181 summary = summarized_results(self.port, expected=False, passing=True, flaky=False) 182 self.assertEquals(summary['tests']['passes']['skipped']['skip.html']['expected'], 'SKIP') 183 184 def test_summarized_results_only_inlude_failing(self): 185 self.port._options.builder_name = 'dummy builder' 186 summary = summarized_results(self.port, expected=False, passing=True, flaky=False, only_include_failing=True) 187 self.assertTrue('passes' not in summary['tests']) 188 189 def test_rounded_run_times(self): 190 summary = summarized_results(self.port, expected=False, passing=False, flaky=False) 191 self.assertEquals(summary['tests']['passes']['text.html']['time'], 1) 192 self.assertTrue('time' not in summary['tests']['failures']['expected']['audio.html']) 193 self.assertEquals(summary['tests']['failures']['expected']['timeout.html']['time'], 0.1) 194 self.assertTrue('time' not in summary['tests']['failures']['expected']['crash.html']) 195 self.assertTrue('time' not in summary['tests']['failures']['expected']['leak.html']) 196 197 def test_timeout_then_unexpected_pass(self): 198 tests = ['failures/expected/image.html'] 199 expectations = test_expectations.TestExpectations(self.port, tests) 200 initial_results = test_run_results.TestRunResults(expectations, len(tests)) 201 initial_results.add(get_result('failures/expected/image.html', test_expectations.TIMEOUT, run_time=1), False, False) 202 retry_results = test_run_results.TestRunResults(expectations, len(tests)) 203 retry_results.add(get_result('failures/expected/image.html', test_expectations.PASS, run_time=0.1), False, False) 204 summary = test_run_results.summarize_results(self.port, expectations, initial_results, retry_results, enabled_pixel_tests_in_retry=True, only_include_failing=True) 205 self.assertEquals(summary['num_regressions'], 0) 206 self.assertEquals(summary['num_passes'], 1) 207
