1 #!/usr/bin/env python 2 # Copyright (C) 2010 Google Inc. All rights reserved. 3 # Copyright (C) 2010 Gabor Rapcsanyi (rgabor (at] inf.u-szeged.hu), University of Szeged 4 # 5 # Redistribution and use in source and binary forms, with or without 6 # modification, are permitted provided that the following conditions are 7 # met: 8 # 9 # * Redistributions of source code must retain the above copyright 10 # notice, this list of conditions and the following disclaimer. 11 # * Redistributions in binary form must reproduce the above 12 # copyright notice, this list of conditions and the following disclaimer 13 # in the documentation and/or other materials provided with the 14 # distribution. 15 # * Neither the name of Google Inc. nor the names of its 16 # contributors may be used to endorse or promote products derived from 17 # this software without specific prior written permission. 18 # 19 # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 20 # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 21 # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 22 # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 23 # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 24 # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 25 # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 26 # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 27 # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 28 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 29 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 30 31 """Run layout tests.""" 32 33 import logging 34 35 import test_expectations 36 37 _log = logging.getLogger("webkitpy.layout_tests.run_webkit_tests") 38 39 TestExpectationsFile = test_expectations.TestExpectationsFile 40 41 42 class ResultSummary(object): 43 """A class for partitioning the test results we get into buckets. 44 45 This class is basically a glorified struct and it's private to this file 46 so we don't bother with any information hiding.""" 47 48 def __init__(self, expectations, test_files): 49 self.total = len(test_files) 50 self.remaining = self.total 51 self.expectations = expectations 52 self.expected = 0 53 self.unexpected = 0 54 self.unexpected_failures = 0 55 self.unexpected_crashes_or_timeouts = 0 56 self.tests_by_expectation = {} 57 self.tests_by_timeline = {} 58 self.results = {} 59 self.unexpected_results = {} 60 self.failures = {} 61 self.tests_by_expectation[test_expectations.SKIP] = set() 62 for expectation in TestExpectationsFile.EXPECTATIONS.values(): 63 self.tests_by_expectation[expectation] = set() 64 for timeline in TestExpectationsFile.TIMELINES.values(): 65 self.tests_by_timeline[timeline] = ( 66 expectations.get_tests_with_timeline(timeline)) 67 68 def add(self, result, expected): 69 """Add a TestResult into the appropriate bin. 70 71 Args: 72 result: TestResult 73 expected: whether the result was what we expected it to be. 74 """ 75 76 self.tests_by_expectation[result.type].add(result.filename) 77 self.results[result.filename] = result 78 self.remaining -= 1 79 if len(result.failures): 80 self.failures[result.filename] = result.failures 81 if expected: 82 self.expected += 1 83 else: 84 self.unexpected_results[result.filename] = result 85 self.unexpected += 1 86 if len(result.failures): 87 self.unexpected_failures += 1 88 if result.type == test_expectations.CRASH or result.type == test_expectations.TIMEOUT: 89 self.unexpected_crashes_or_timeouts += 1 90
