1 # Copyright (C) 2013 Google Inc. All rights reserved. 2 # 3 # Redistribution and use in source and binary forms, with or without 4 # modification, are permitted provided that the following conditions are 5 # met: 6 # 7 # * Redistributions of source code must retain the above copyright 8 # notice, this list of conditions and the following disclaimer. 9 # * Redistributions in binary form must reproduce the above 10 # copyright notice, this list of conditions and the following disclaimer 11 # in the documentation and/or other materials provided with the 12 # distribution. 13 # * Neither the name of Google Inc. nor the names of its 14 # contributors may be used to endorse or promote products derived from 15 # this software without specific prior written permission. 16 # 17 # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 18 # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 19 # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 20 # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 21 # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 22 # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 23 # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 24 # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 25 # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 26 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 27 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 28 29 import StringIO 30 import json 31 import math 32 import unittest2 as unittest 33 34 from webkitpy.common.host_mock import MockHost 35 from webkitpy.common.system.outputcapture import OutputCapture 36 from webkitpy.layout_tests.port.driver import DriverOutput 37 from webkitpy.layout_tests.port.test import TestDriver 38 from webkitpy.layout_tests.port.test import TestPort 39 from webkitpy.performance_tests.perftest import ChromiumStylePerfTest 40 from webkitpy.performance_tests.perftest import PerfTest 41 from webkitpy.performance_tests.perftest import PerfTestMetric 42 from webkitpy.performance_tests.perftest import PerfTestFactory 43 from webkitpy.performance_tests.perftest import SingleProcessPerfTest 44 45 46 class MockPort(TestPort): 47 def __init__(self, custom_run_test=None): 48 super(MockPort, self).__init__(host=MockHost(), custom_run_test=custom_run_test) 49 50 51 class TestPerfTestMetric(unittest.TestCase): 52 def test_init_set_missing_unit(self): 53 self.assertEqual(PerfTestMetric('Time', iterations=[1, 2, 3, 4, 5]).unit(), 'ms') 54 self.assertEqual(PerfTestMetric('Malloc', iterations=[1, 2, 3, 4, 5]).unit(), 'bytes') 55 self.assertEqual(PerfTestMetric('JSHeap', iterations=[1, 2, 3, 4, 5]).unit(), 'bytes') 56 57 def test_init_set_time_metric(self): 58 self.assertEqual(PerfTestMetric('Time', 'ms').name(), 'Time') 59 self.assertEqual(PerfTestMetric('Time', 'fps').name(), 'FrameRate') 60 self.assertEqual(PerfTestMetric('Time', 'runs/s').name(), 'Runs') 61 62 def test_has_values(self): 63 self.assertFalse(PerfTestMetric('Time').has_values()) 64 self.assertTrue(PerfTestMetric('Time', iterations=[1]).has_values()) 65 66 def test_append(self): 67 metric = PerfTestMetric('Time') 68 metric2 = PerfTestMetric('Time') 69 self.assertFalse(metric.has_values()) 70 self.assertFalse(metric2.has_values()) 71 72 metric.append_group([1]) 73 self.assertTrue(metric.has_values()) 74 self.assertFalse(metric2.has_values()) 75 self.assertEqual(metric.grouped_iteration_values(), [[1]]) 76 self.assertEqual(metric.flattened_iteration_values(), [1]) 77 78 metric.append_group([2]) 79 self.assertEqual(metric.grouped_iteration_values(), [[1], [2]]) 80 self.assertEqual(metric.flattened_iteration_values(), [1, 2]) 81 82 metric2.append_group([3]) 83 self.assertTrue(metric2.has_values()) 84 self.assertEqual(metric.flattened_iteration_values(), [1, 2]) 85 self.assertEqual(metric2.flattened_iteration_values(), [3]) 86 87 metric.append_group([4, 5]) 88 self.assertEqual(metric.grouped_iteration_values(), [[1], [2], [4, 5]]) 89 self.assertEqual(metric.flattened_iteration_values(), [1, 2, 4, 5]) 90 91 92 class TestPerfTest(unittest.TestCase): 93 def _assert_results_are_correct(self, test, output): 94 test.run_single = lambda driver, path, time_out_ms: output 95 self.assertTrue(test._run_with_driver(None, None)) 96 self.assertEqual(test._metrics.keys(), ['Time']) 97 self.assertEqual(test._metrics['Time'].flattened_iteration_values(), [1080, 1120, 1095, 1101, 1104]) 98 99 def test_parse_output(self): 100 output = DriverOutput(""" 101 Running 20 times 102 Ignoring warm-up run (1115) 103 104 Time: 105 values 1080, 1120, 1095, 1101, 1104 ms 106 avg 1100 ms 107 median 1101 ms 108 stdev 14.50862 ms 109 min 1080 ms 110 max 1120 ms 111 """, image=None, image_hash=None, audio=None) 112 output_capture = OutputCapture() 113 output_capture.capture_output() 114 try: 115 test = PerfTest(MockPort(), 'some-test', '/path/some-dir/some-test') 116 self._assert_results_are_correct(test, output) 117 finally: 118 actual_stdout, actual_stderr, actual_logs = output_capture.restore_output() 119 self.assertEqual(actual_stdout, '') 120 self.assertEqual(actual_stderr, '') 121 self.assertEqual(actual_logs, '') 122 123 def test_parse_output_with_failing_line(self): 124 output = DriverOutput(""" 125 Running 20 times 126 Ignoring warm-up run (1115) 127 128 some-unrecognizable-line 129 130 Time: 131 values 1080, 1120, 1095, 1101, 1104 ms 132 avg 1100 ms 133 median 1101 ms 134 stdev 14.50862 ms 135 min 1080 ms 136 max 1120 ms 137 """, image=None, image_hash=None, audio=None) 138 output_capture = OutputCapture() 139 output_capture.capture_output() 140 try: 141 test = PerfTest(MockPort(), 'some-test', '/path/some-dir/some-test') 142 test.run_single = lambda driver, path, time_out_ms: output 143 self.assertFalse(test._run_with_driver(None, None)) 144 finally: 145 actual_stdout, actual_stderr, actual_logs = output_capture.restore_output() 146 self.assertEqual(actual_stdout, '') 147 self.assertEqual(actual_stderr, '') 148 self.assertEqual(actual_logs, 'ERROR: some-unrecognizable-line\n') 149 150 def test_parse_output_with_description(self): 151 output = DriverOutput(""" 152 Description: this is a test description. 153 154 Running 20 times 155 Ignoring warm-up run (1115) 156 157 Time: 158 values 1080, 1120, 1095, 1101, 1104 ms 159 avg 1100 ms 160 median 1101 ms 161 stdev 14.50862 ms 162 min 1080 ms 163 max 1120 ms""", image=None, image_hash=None, audio=None) 164 test = PerfTest(MockPort(), 'some-test', '/path/some-dir/some-test') 165 self._assert_results_are_correct(test, output) 166 self.assertEqual(test.description(), 'this is a test description.') 167 168 def test_ignored_stderr_lines(self): 169 test = PerfTest(MockPort(), 'some-test', '/path/some-dir/some-test') 170 output_with_lines_to_ignore = DriverOutput('', image=None, image_hash=None, audio=None, error=""" 171 Unknown option: --foo-bar 172 Should not be ignored 173 [WARNING:proxy_service.cc] bad moon a-rising 174 [WARNING:chrome.cc] Something went wrong 175 [INFO:SkFontHost_android.cpp(1158)] Use Test Config File Main /data/local/tmp/drt/android_main_fonts.xml, Fallback /data/local/tmp/drt/android_fallback_fonts.xml, Font Dir /data/local/tmp/drt/fonts/ 176 [ERROR:main.cc] The sky has fallen""") 177 test._filter_output(output_with_lines_to_ignore) 178 self.assertEqual(output_with_lines_to_ignore.error, 179 "Should not be ignored\n" 180 "[WARNING:chrome.cc] Something went wrong\n" 181 "[ERROR:main.cc] The sky has fallen") 182 183 def test_parse_output_with_subtests(self): 184 output = DriverOutput(""" 185 Running 20 times 186 some test: [1, 2, 3, 4, 5] 187 other test = else: [6, 7, 8, 9, 10] 188 Ignoring warm-up run (1115) 189 190 Time: 191 values 1080, 1120, 1095, 1101, 1104 ms 192 avg 1100 ms 193 median 1101 ms 194 stdev 14.50862 ms 195 min 1080 ms 196 max 1120 ms 197 """, image=None, image_hash=None, audio=None) 198 output_capture = OutputCapture() 199 output_capture.capture_output() 200 try: 201 test = PerfTest(MockPort(), 'some-test', '/path/some-dir/some-test') 202 self._assert_results_are_correct(test, output) 203 finally: 204 actual_stdout, actual_stderr, actual_logs = output_capture.restore_output() 205 self.assertEqual(actual_stdout, '') 206 self.assertEqual(actual_stderr, '') 207 self.assertEqual(actual_logs, '') 208 209 210 class TestSingleProcessPerfTest(unittest.TestCase): 211 def test_use_only_one_process(self): 212 called = [0] 213 214 def run_single(driver, path, time_out_ms): 215 called[0] += 1 216 return DriverOutput(""" 217 Running 20 times 218 Ignoring warm-up run (1115) 219 220 Time: 221 values 1080, 1120, 1095, 1101, 1104 ms 222 avg 1100 ms 223 median 1101 ms 224 stdev 14.50862 ms 225 min 1080 ms 226 max 1120 ms""", image=None, image_hash=None, audio=None) 227 228 test = SingleProcessPerfTest(MockPort(), 'some-test', '/path/some-dir/some-test') 229 test.run_single = run_single 230 self.assertTrue(test.run(0)) 231 self.assertEqual(called[0], 1) 232 233 234 class TestPerfTestFactory(unittest.TestCase): 235 def test_regular_test(self): 236 test = PerfTestFactory.create_perf_test(MockPort(), 'some-dir/some-test', '/path/some-dir/some-test') 237 self.assertEqual(test.__class__, PerfTest) 238 239 def test_inspector_test(self): 240 test = PerfTestFactory.create_perf_test(MockPort(), 'inspector/some-test', '/path/inspector/some-test') 241 self.assertEqual(test.__class__, ChromiumStylePerfTest) 242
