1 # Copyright (C) 2012 Google Inc. All rights reserved. 2 # 3 # Redistribution and use in source and binary forms, with or without 4 # modification, are permitted provided that the following conditions are 5 # met: 6 # 7 # * Redistributions of source code must retain the above copyright 8 # notice, this list of conditions and the following disclaimer. 9 # * Redistributions in binary form must reproduce the above 10 # copyright notice, this list of conditions and the following disclaimer 11 # in the documentation and/or other materials provided with the 12 # distribution. 13 # * Neither the name of Google Inc. nor the names of its 14 # contributors may be used to endorse or promote products derived from 15 # this software without specific prior written permission. 16 # 17 # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 18 # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 19 # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 20 # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 21 # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 22 # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 23 # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 24 # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 25 # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 26 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 27 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 28 29 """Unit tests for run_perf_tests.""" 30 31 import StringIO 32 import json 33 import re 34 import unittest2 as unittest 35 36 from webkitpy.common.host_mock import MockHost 37 from webkitpy.common.system.outputcapture import OutputCapture 38 from webkitpy.layout_tests.port.test import TestPort 39 from webkitpy.performance_tests.perftest import DEFAULT_TEST_RUNNER_COUNT 40 from webkitpy.performance_tests.perftestsrunner import PerfTestsRunner 41 42 43 class MainTest(unittest.TestCase): 44 def create_runner(self, args=[]): 45 options, parsed_args = PerfTestsRunner._parse_args(args) 46 test_port = TestPort(host=MockHost(), options=options) 47 runner = PerfTestsRunner(args=args, port=test_port) 48 runner._host.filesystem.maybe_make_directory(runner._base_path, 'inspector') 49 runner._host.filesystem.maybe_make_directory(runner._base_path, 'Bindings') 50 runner._host.filesystem.maybe_make_directory(runner._base_path, 'Parser') 51 return runner, test_port 52 53 def _add_file(self, runner, dirname, filename, content=True): 54 dirname = runner._host.filesystem.join(runner._base_path, dirname) if dirname else runner._base_path 55 runner._host.filesystem.maybe_make_directory(dirname) 56 runner._host.filesystem.files[runner._host.filesystem.join(dirname, filename)] = content 57 58 def test_collect_tests(self): 59 runner, port = self.create_runner() 60 self._add_file(runner, 'inspector', 'a_file.html', 'a content') 61 tests = runner._collect_tests() 62 self.assertEqual(len(tests), 1) 63 64 def _collect_tests_and_sort_test_name(self, runner): 65 return sorted([test.test_name() for test in runner._collect_tests()]) 66 67 def test_collect_tests_with_multile_files(self): 68 runner, port = self.create_runner(args=['PerformanceTests/test1.html', 'test2.html']) 69 70 def add_file(filename): 71 port.host.filesystem.files[runner._host.filesystem.join(runner._base_path, filename)] = 'some content' 72 73 add_file('test1.html') 74 add_file('test2.html') 75 add_file('test3.html') 76 port.host.filesystem.chdir(runner._port.perf_tests_dir()[:runner._port.perf_tests_dir().rfind(runner._host.filesystem.sep)]) 77 self.assertItemsEqual(self._collect_tests_and_sort_test_name(runner), ['test1.html', 'test2.html']) 78 79 def test_collect_tests_with_skipped_list(self): 80 runner, port = self.create_runner() 81 82 self._add_file(runner, 'inspector', 'test1.html') 83 self._add_file(runner, 'inspector', 'unsupported_test1.html') 84 self._add_file(runner, 'inspector', 'test2.html') 85 self._add_file(runner, 'inspector/resources', 'resource_file.html') 86 self._add_file(runner, 'unsupported', 'unsupported_test2.html') 87 port.skipped_perf_tests = lambda: ['inspector/unsupported_test1.html', 'unsupported'] 88 self.assertItemsEqual(self._collect_tests_and_sort_test_name(runner), ['inspector/test1.html', 'inspector/test2.html']) 89 90 def test_collect_tests_with_skipped_list_and_files(self): 91 runner, port = self.create_runner(args=['Suite/Test1.html', 'Suite/SkippedTest1.html', 'SkippedSuite/Test1.html']) 92 93 self._add_file(runner, 'SkippedSuite', 'Test1.html') 94 self._add_file(runner, 'SkippedSuite', 'Test2.html') 95 self._add_file(runner, 'Suite', 'Test1.html') 96 self._add_file(runner, 'Suite', 'Test2.html') 97 self._add_file(runner, 'Suite', 'SkippedTest1.html') 98 self._add_file(runner, 'Suite', 'SkippedTest2.html') 99 port.skipped_perf_tests = lambda: ['Suite/SkippedTest1.html', 'Suite/SkippedTest1.html', 'SkippedSuite'] 100 self.assertItemsEqual(self._collect_tests_and_sort_test_name(runner), 101 ['SkippedSuite/Test1.html', 'Suite/SkippedTest1.html', 'Suite/Test1.html']) 102 103 def test_collect_tests_with_ignored_skipped_list(self): 104 runner, port = self.create_runner(args=['--force']) 105 106 self._add_file(runner, 'inspector', 'test1.html') 107 self._add_file(runner, 'inspector', 'unsupported_test1.html') 108 self._add_file(runner, 'inspector', 'test2.html') 109 self._add_file(runner, 'inspector/resources', 'resource_file.html') 110 self._add_file(runner, 'unsupported', 'unsupported_test2.html') 111 port.skipped_perf_tests = lambda: ['inspector/unsupported_test1.html', 'unsupported'] 112 self.assertItemsEqual(self._collect_tests_and_sort_test_name(runner), ['inspector/test1.html', 'inspector/test2.html', 'inspector/unsupported_test1.html', 'unsupported/unsupported_test2.html']) 113 114 def test_default_args(self): 115 runner, port = self.create_runner() 116 options, args = PerfTestsRunner._parse_args([]) 117 self.assertTrue(options.build) 118 self.assertEqual(options.time_out_ms, 600 * 1000) 119 self.assertTrue(options.generate_results) 120 self.assertTrue(options.show_results) 121 self.assertTrue(options.use_skipped_list) 122 self.assertEqual(options.repeat, 1) 123 self.assertEqual(options.test_runner_count, DEFAULT_TEST_RUNNER_COUNT) 124 125 def test_parse_args(self): 126 runner, port = self.create_runner() 127 options, args = PerfTestsRunner._parse_args([ 128 '--build-directory=folder42', 129 '--platform=platform42', 130 '--builder-name', 'webkit-mac-1', 131 '--build-number=56', 132 '--time-out-ms=42', 133 '--no-show-results', 134 '--reset-results', 135 '--output-json-path=a/output.json', 136 '--slave-config-json-path=a/source.json', 137 '--test-results-server=somehost', 138 '--additional-drt-flag=--enable-threaded-parser', 139 '--additional-drt-flag=--awesomesauce', 140 '--repeat=5', 141 '--test-runner-count=5', 142 '--debug']) 143 self.assertTrue(options.build) 144 self.assertEqual(options.build_directory, 'folder42') 145 self.assertEqual(options.platform, 'platform42') 146 self.assertEqual(options.builder_name, 'webkit-mac-1') 147 self.assertEqual(options.build_number, '56') 148 self.assertEqual(options.time_out_ms, '42') 149 self.assertEqual(options.configuration, 'Debug') 150 self.assertFalse(options.show_results) 151 self.assertTrue(options.reset_results) 152 self.assertEqual(options.output_json_path, 'a/output.json') 153 self.assertEqual(options.slave_config_json_path, 'a/source.json') 154 self.assertEqual(options.test_results_server, 'somehost') 155 self.assertEqual(options.additional_drt_flag, ['--enable-threaded-parser', '--awesomesauce']) 156 self.assertEqual(options.repeat, 5) 157 self.assertEqual(options.test_runner_count, 5) 158 159 def test_upload_json(self): 160 runner, port = self.create_runner() 161 port.host.filesystem.files['/mock-checkout/some.json'] = 'some content' 162 163 class MockFileUploader: 164 called = [] 165 upload_single_text_file_throws = False 166 upload_single_text_file_return_value = None 167 168 @classmethod 169 def reset(cls): 170 cls.called = [] 171 cls.upload_single_text_file_throws = False 172 cls.upload_single_text_file_return_value = None 173 174 def __init__(mock, url, timeout): 175 self.assertEqual(url, 'https://some.host/some/path') 176 self.assertTrue(isinstance(timeout, int) and timeout) 177 mock.called.append('FileUploader') 178 179 def upload_single_text_file(mock, filesystem, content_type, filename): 180 self.assertEqual(filesystem, port.host.filesystem) 181 self.assertEqual(content_type, 'application/json') 182 self.assertEqual(filename, 'some.json') 183 mock.called.append('upload_single_text_file') 184 if mock.upload_single_text_file_throws: 185 raise Exception 186 return mock.upload_single_text_file_return_value 187 188 MockFileUploader.upload_single_text_file_return_value = StringIO.StringIO('OK') 189 self.assertTrue(runner._upload_json('some.host', 'some.json', '/some/path', MockFileUploader)) 190 self.assertEqual(MockFileUploader.called, ['FileUploader', 'upload_single_text_file']) 191 192 MockFileUploader.reset() 193 MockFileUploader.upload_single_text_file_return_value = StringIO.StringIO('Some error') 194 output = OutputCapture() 195 output.capture_output() 196 self.assertFalse(runner._upload_json('some.host', 'some.json', '/some/path', MockFileUploader)) 197 _, _, logs = output.restore_output() 198 self.assertEqual(logs, 'Uploaded JSON to https://some.host/some/path but got a bad response:\nSome error\n') 199 200 # Throwing an exception upload_single_text_file shouldn't blow up _upload_json 201 MockFileUploader.reset() 202 MockFileUploader.upload_single_text_file_throws = True 203 self.assertFalse(runner._upload_json('some.host', 'some.json', '/some/path', MockFileUploader)) 204 self.assertEqual(MockFileUploader.called, ['FileUploader', 'upload_single_text_file']) 205 206 MockFileUploader.reset() 207 MockFileUploader.upload_single_text_file_return_value = StringIO.StringIO('{"status": "OK"}') 208 self.assertTrue(runner._upload_json('some.host', 'some.json', '/some/path', MockFileUploader)) 209 self.assertEqual(MockFileUploader.called, ['FileUploader', 'upload_single_text_file']) 210 211 MockFileUploader.reset() 212 MockFileUploader.upload_single_text_file_return_value = StringIO.StringIO('{"status": "SomethingHasFailed", "failureStored": false}') 213 output = OutputCapture() 214 output.capture_output() 215 self.assertFalse(runner._upload_json('some.host', 'some.json', '/some/path', MockFileUploader)) 216 _, _, logs = output.restore_output() 217 serialized_json = json.dumps({'status': 'SomethingHasFailed', 'failureStored': False}, indent=4) 218 self.assertEqual(logs, 'Uploaded JSON to https://some.host/some/path but got an error:\n%s\n' % serialized_json) 219
