1 # Copyright (C) 2010 Google Inc. All rights reserved. 2 # 3 # Redistribution and use in source and binary forms, with or without 4 # modification, are permitted provided that the following conditions are 5 # met: 6 # 7 # * Redistributions of source code must retain the above copyright 8 # notice, this list of conditions and the following disclaimer. 9 # * Redistributions in binary form must reproduce the above 10 # copyright notice, this list of conditions and the following disclaimer 11 # in the documentation and/or other materials provided with the 12 # distribution. 13 # * Neither the name of Google Inc. nor the names of its 14 # contributors may be used to endorse or promote products derived from 15 # this software without specific prior written permission. 16 # 17 # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 18 # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 19 # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 20 # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 21 # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 22 # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 23 # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 24 # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 25 # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 26 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 27 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 28 29 import logging 30 import optparse 31 import sys 32 import tempfile 33 import webkitpy.thirdparty.unittest2 as unittest 34 35 from webkitpy.common.system.executive import Executive, ScriptError 36 from webkitpy.common.system import executive_mock 37 from webkitpy.common.system.filesystem_mock import MockFileSystem 38 from webkitpy.common.system.outputcapture import OutputCapture 39 from webkitpy.common.system.path import abspath_to_uri 40 from webkitpy.thirdparty.mock import Mock 41 from webkitpy.tool.mocktool import MockOptions 42 from webkitpy.common.system.executive_mock import MockExecutive, MockExecutive2 43 from webkitpy.common.system.systemhost_mock import MockSystemHost 44 45 from webkitpy.layout_tests.port import Port, Driver, DriverOutput 46 from webkitpy.layout_tests.port.test import add_unit_tests_to_mock_filesystem, TestPort 47 48 class PortTest(unittest.TestCase): 49 def make_port(self, executive=None, with_tests=False, port_name=None, **kwargs): 50 host = MockSystemHost() 51 if executive: 52 host.executive = executive 53 if with_tests: 54 add_unit_tests_to_mock_filesystem(host.filesystem) 55 return TestPort(host, **kwargs) 56 return Port(host, port_name or 'baseport', **kwargs) 57 58 def test_default_child_processes(self): 59 port = self.make_port() 60 self.assertIsNotNone(port.default_child_processes()) 61 62 def test_format_wdiff_output_as_html(self): 63 output = "OUTPUT %s %s %s" % (Port._WDIFF_DEL, Port._WDIFF_ADD, Port._WDIFF_END) 64 html = self.make_port()._format_wdiff_output_as_html(output) 65 expected_html = "<head><style>.del { background: #faa; } .add { background: #afa; }</style></head><pre>OUTPUT <span class=del> <span class=add> </span></pre>" 66 self.assertEqual(html, expected_html) 67 68 def test_wdiff_command(self): 69 port = self.make_port() 70 port._path_to_wdiff = lambda: "/path/to/wdiff" 71 command = port._wdiff_command("/actual/path", "/expected/path") 72 expected_command = [ 73 "/path/to/wdiff", 74 "--start-delete=##WDIFF_DEL##", 75 "--end-delete=##WDIFF_END##", 76 "--start-insert=##WDIFF_ADD##", 77 "--end-insert=##WDIFF_END##", 78 "/actual/path", 79 "/expected/path", 80 ] 81 self.assertEqual(command, expected_command) 82 83 def _file_with_contents(self, contents, encoding="utf-8"): 84 new_file = tempfile.NamedTemporaryFile() 85 new_file.write(contents.encode(encoding)) 86 new_file.flush() 87 return new_file 88 89 def test_pretty_patch_os_error(self): 90 port = self.make_port(executive=executive_mock.MockExecutive2(exception=OSError)) 91 oc = OutputCapture() 92 oc.capture_output() 93 self.assertEqual(port.pretty_patch_text("patch.txt"), 94 port._pretty_patch_error_html) 95 96 # This tests repeated calls to make sure we cache the result. 97 self.assertEqual(port.pretty_patch_text("patch.txt"), 98 port._pretty_patch_error_html) 99 oc.restore_output() 100 101 def test_pretty_patch_script_error(self): 102 # FIXME: This is some ugly white-box test hacking ... 103 port = self.make_port(executive=executive_mock.MockExecutive2(exception=ScriptError)) 104 port._pretty_patch_available = True 105 self.assertEqual(port.pretty_patch_text("patch.txt"), 106 port._pretty_patch_error_html) 107 108 # This tests repeated calls to make sure we cache the result. 109 self.assertEqual(port.pretty_patch_text("patch.txt"), 110 port._pretty_patch_error_html) 111 112 def test_wdiff_text(self): 113 port = self.make_port() 114 port.wdiff_available = lambda: True 115 port._run_wdiff = lambda a, b: 'PASS' 116 self.assertEqual('PASS', port.wdiff_text(None, None)) 117 118 def test_diff_text(self): 119 port = self.make_port() 120 # Make sure that we don't run into decoding exceptions when the 121 # filenames are unicode, with regular or malformed input (expected or 122 # actual input is always raw bytes, not unicode). 123 port.diff_text('exp', 'act', 'exp.txt', 'act.txt') 124 port.diff_text('exp', 'act', u'exp.txt', 'act.txt') 125 port.diff_text('exp', 'act', u'a\xac\u1234\u20ac\U00008000', 'act.txt') 126 127 port.diff_text('exp' + chr(255), 'act', 'exp.txt', 'act.txt') 128 port.diff_text('exp' + chr(255), 'act', u'exp.txt', 'act.txt') 129 130 # Though expected and actual files should always be read in with no 131 # encoding (and be stored as str objects), test unicode inputs just to 132 # be safe. 133 port.diff_text(u'exp', 'act', 'exp.txt', 'act.txt') 134 port.diff_text( 135 u'a\xac\u1234\u20ac\U00008000', 'act', 'exp.txt', 'act.txt') 136 137 # And make sure we actually get diff output. 138 diff = port.diff_text('foo', 'bar', 'exp.txt', 'act.txt') 139 self.assertIn('foo', diff) 140 self.assertIn('bar', diff) 141 self.assertIn('exp.txt', diff) 142 self.assertIn('act.txt', diff) 143 self.assertNotIn('nosuchthing', diff) 144 145 def test_setup_test_run(self): 146 port = self.make_port() 147 # This routine is a no-op. We just test it for coverage. 148 port.setup_test_run() 149 150 def test_test_dirs(self): 151 port = self.make_port() 152 port.host.filesystem.write_text_file(port.layout_tests_dir() + '/canvas/test', '') 153 port.host.filesystem.write_text_file(port.layout_tests_dir() + '/css2.1/test', '') 154 dirs = port.test_dirs() 155 self.assertIn('canvas', dirs) 156 self.assertIn('css2.1', dirs) 157 158 def test_skipped_perf_tests(self): 159 port = self.make_port() 160 161 def add_text_file(dirname, filename, content='some content'): 162 dirname = port.host.filesystem.join(port.perf_tests_dir(), dirname) 163 port.host.filesystem.maybe_make_directory(dirname) 164 port.host.filesystem.write_text_file(port.host.filesystem.join(dirname, filename), content) 165 166 add_text_file('inspector', 'test1.html') 167 add_text_file('inspector', 'unsupported_test1.html') 168 add_text_file('inspector', 'test2.html') 169 add_text_file('inspector/resources', 'resource_file.html') 170 add_text_file('unsupported', 'unsupported_test2.html') 171 add_text_file('', 'Skipped', '\n'.join(['Layout', '', 'SunSpider', 'Supported/some-test.html'])) 172 self.assertEqual(port.skipped_perf_tests(), ['Layout', 'SunSpider', 'Supported/some-test.html']) 173 174 def test_get_option__set(self): 175 options, args = optparse.OptionParser().parse_args([]) 176 options.foo = 'bar' 177 port = self.make_port(options=options) 178 self.assertEqual(port.get_option('foo'), 'bar') 179 180 def test_get_option__unset(self): 181 port = self.make_port() 182 self.assertIsNone(port.get_option('foo')) 183 184 def test_get_option__default(self): 185 port = self.make_port() 186 self.assertEqual(port.get_option('foo', 'bar'), 'bar') 187 188 def test_additional_platform_directory(self): 189 port = self.make_port(port_name='foo') 190 port.default_baseline_search_path = lambda: ['LayoutTests/platform/foo'] 191 layout_test_dir = port.layout_tests_dir() 192 test_file = 'fast/test.html' 193 194 # No additional platform directory 195 self.assertEqual( 196 port.expected_baselines(test_file, '.txt'), 197 [(None, 'fast/test-expected.txt')]) 198 self.assertEqual(port.baseline_path(), 'LayoutTests/platform/foo') 199 200 # Simple additional platform directory 201 port._options.additional_platform_directory = ['/tmp/local-baselines'] 202 port._filesystem.write_text_file('/tmp/local-baselines/fast/test-expected.txt', 'foo') 203 self.assertEqual( 204 port.expected_baselines(test_file, '.txt'), 205 [('/tmp/local-baselines', 'fast/test-expected.txt')]) 206 self.assertEqual(port.baseline_path(), '/tmp/local-baselines') 207 208 # Multiple additional platform directories 209 port._options.additional_platform_directory = ['/foo', '/tmp/local-baselines'] 210 self.assertEqual( 211 port.expected_baselines(test_file, '.txt'), 212 [('/tmp/local-baselines', 'fast/test-expected.txt')]) 213 self.assertEqual(port.baseline_path(), '/foo') 214 215 def test_nonexistant_expectations(self): 216 port = self.make_port(port_name='foo') 217 port.expectations_files = lambda: ['/mock-checkout/LayoutTests/platform/exists/TestExpectations', '/mock-checkout/LayoutTests/platform/nonexistant/TestExpectations'] 218 port._filesystem.write_text_file('/mock-checkout/LayoutTests/platform/exists/TestExpectations', '') 219 self.assertEqual('\n'.join(port.expectations_dict().keys()), '/mock-checkout/LayoutTests/platform/exists/TestExpectations') 220 221 def test_additional_expectations(self): 222 port = self.make_port(port_name='foo') 223 port.port_name = 'foo' 224 port._filesystem.write_text_file('/mock-checkout/LayoutTests/platform/foo/TestExpectations', '') 225 port._filesystem.write_text_file( 226 '/tmp/additional-expectations-1.txt', 'content1\n') 227 port._filesystem.write_text_file( 228 '/tmp/additional-expectations-2.txt', 'content2\n') 229 230 self.assertEqual('\n'.join(port.expectations_dict().values()), '') 231 232 port._options.additional_expectations = [ 233 '/tmp/additional-expectations-1.txt'] 234 self.assertEqual('\n'.join(port.expectations_dict().values()), '\ncontent1\n') 235 236 port._options.additional_expectations = [ 237 '/tmp/nonexistent-file', '/tmp/additional-expectations-1.txt'] 238 self.assertEqual('\n'.join(port.expectations_dict().values()), '\ncontent1\n') 239 240 port._options.additional_expectations = [ 241 '/tmp/additional-expectations-1.txt', '/tmp/additional-expectations-2.txt'] 242 self.assertEqual('\n'.join(port.expectations_dict().values()), '\ncontent1\n\ncontent2\n') 243 244 def test_additional_env_var(self): 245 port = self.make_port(options=optparse.Values({'additional_env_var': ['FOO=BAR', 'BAR=FOO']})) 246 self.assertEqual(port.get_option('additional_env_var'), ['FOO=BAR', 'BAR=FOO']) 247 environment = port.setup_environ_for_server() 248 self.assertTrue(('FOO' in environment) & ('BAR' in environment)) 249 self.assertEqual(environment['FOO'], 'BAR') 250 self.assertEqual(environment['BAR'], 'FOO') 251 252 def test_find_no_paths_specified(self): 253 port = self.make_port(with_tests=True) 254 layout_tests_dir = port.layout_tests_dir() 255 tests = port.tests([]) 256 self.assertNotEqual(len(tests), 0) 257 258 def test_find_one_test(self): 259 port = self.make_port(with_tests=True) 260 tests = port.tests(['failures/expected/image.html']) 261 self.assertEqual(len(tests), 1) 262 263 def test_find_glob(self): 264 port = self.make_port(with_tests=True) 265 tests = port.tests(['failures/expected/im*']) 266 self.assertEqual(len(tests), 2) 267 268 def test_find_with_skipped_directories(self): 269 port = self.make_port(with_tests=True) 270 tests = port.tests(['userscripts']) 271 self.assertNotIn('userscripts/resources/iframe.html', tests) 272 273 def test_find_with_skipped_directories_2(self): 274 port = self.make_port(with_tests=True) 275 tests = port.tests(['userscripts/resources']) 276 self.assertEqual(tests, []) 277 278 def test_is_test_file(self): 279 filesystem = MockFileSystem() 280 self.assertTrue(Port.is_test_file(filesystem, '', 'foo.html')) 281 self.assertTrue(Port.is_test_file(filesystem, '', 'foo.svg')) 282 self.assertTrue(Port.is_test_file(filesystem, '', 'test-ref-test.html')) 283 self.assertFalse(Port.is_test_file(filesystem, '', 'foo.png')) 284 self.assertFalse(Port.is_test_file(filesystem, '', 'foo-expected.html')) 285 self.assertFalse(Port.is_test_file(filesystem, '', 'foo-expected.svg')) 286 self.assertFalse(Port.is_test_file(filesystem, '', 'foo-expected.xht')) 287 self.assertFalse(Port.is_test_file(filesystem, '', 'foo-expected-mismatch.html')) 288 self.assertFalse(Port.is_test_file(filesystem, '', 'foo-expected-mismatch.svg')) 289 self.assertFalse(Port.is_test_file(filesystem, '', 'foo-expected-mismatch.xhtml')) 290 self.assertFalse(Port.is_test_file(filesystem, '', 'foo-ref.html')) 291 self.assertFalse(Port.is_test_file(filesystem, '', 'foo-notref.html')) 292 self.assertFalse(Port.is_test_file(filesystem, '', 'foo-notref.xht')) 293 self.assertFalse(Port.is_test_file(filesystem, '', 'foo-ref.xhtml')) 294 self.assertFalse(Port.is_test_file(filesystem, '', 'ref-foo.html')) 295 self.assertFalse(Port.is_test_file(filesystem, '', 'notref-foo.xhr')) 296 297 def test_parse_reftest_list(self): 298 port = self.make_port(with_tests=True) 299 port.host.filesystem.files['bar/reftest.list'] = "\n".join(["== test.html test-ref.html", 300 "", 301 "# some comment", 302 "!= test-2.html test-notref.html # more comments", 303 "== test-3.html test-ref.html", 304 "== test-3.html test-ref2.html", 305 "!= test-3.html test-notref.html"]) 306 307 reftest_list = Port._parse_reftest_list(port.host.filesystem, 'bar') 308 self.assertEqual(reftest_list, {'bar/test.html': [('==', 'bar/test-ref.html')], 309 'bar/test-2.html': [('!=', 'bar/test-notref.html')], 310 'bar/test-3.html': [('==', 'bar/test-ref.html'), ('==', 'bar/test-ref2.html'), ('!=', 'bar/test-notref.html')]}) 311 312 def test_reference_files(self): 313 port = self.make_port(with_tests=True) 314 self.assertEqual(port.reference_files('passes/svgreftest.svg'), [('==', port.layout_tests_dir() + '/passes/svgreftest-expected.svg')]) 315 self.assertEqual(port.reference_files('passes/xhtreftest.svg'), [('==', port.layout_tests_dir() + '/passes/xhtreftest-expected.html')]) 316 self.assertEqual(port.reference_files('passes/phpreftest.php'), [('!=', port.layout_tests_dir() + '/passes/phpreftest-expected-mismatch.svg')]) 317 318 def test_operating_system(self): 319 self.assertEqual('mac', self.make_port().operating_system()) 320 321 def test_http_server_supports_ipv6(self): 322 port = self.make_port() 323 self.assertTrue(port.http_server_supports_ipv6()) 324 port.host.platform.os_name = 'cygwin' 325 self.assertFalse(port.http_server_supports_ipv6()) 326 port.host.platform.os_name = 'win' 327 self.assertTrue(port.http_server_supports_ipv6()) 328 329 def test_check_httpd_success(self): 330 port = self.make_port(executive=MockExecutive2()) 331 port._path_to_apache = lambda: '/usr/sbin/httpd' 332 capture = OutputCapture() 333 capture.capture_output() 334 self.assertTrue(port.check_httpd()) 335 _, _, logs = capture.restore_output() 336 self.assertEqual('', logs) 337 338 def test_httpd_returns_error_code(self): 339 port = self.make_port(executive=MockExecutive2(exit_code=1)) 340 port._path_to_apache = lambda: '/usr/sbin/httpd' 341 capture = OutputCapture() 342 capture.capture_output() 343 self.assertFalse(port.check_httpd()) 344 _, _, logs = capture.restore_output() 345 self.assertEqual('httpd seems broken. Cannot run http tests.\n', logs) 346 347 def test_test_exists(self): 348 port = self.make_port(with_tests=True) 349 self.assertTrue(port.test_exists('passes')) 350 self.assertTrue(port.test_exists('passes/text.html')) 351 self.assertFalse(port.test_exists('passes/does_not_exist.html')) 352 353 self.assertTrue(port.test_exists('virtual')) 354 self.assertFalse(port.test_exists('virtual/does_not_exist.html')) 355 self.assertTrue(port.test_exists('virtual/passes/text.html')) 356 357 def test_test_isfile(self): 358 port = self.make_port(with_tests=True) 359 self.assertFalse(port.test_isfile('passes')) 360 self.assertTrue(port.test_isfile('passes/text.html')) 361 self.assertFalse(port.test_isfile('passes/does_not_exist.html')) 362 363 self.assertFalse(port.test_isfile('virtual')) 364 self.assertTrue(port.test_isfile('virtual/passes/text.html')) 365 self.assertFalse(port.test_isfile('virtual/does_not_exist.html')) 366 367 def test_test_isdir(self): 368 port = self.make_port(with_tests=True) 369 self.assertTrue(port.test_isdir('passes')) 370 self.assertFalse(port.test_isdir('passes/text.html')) 371 self.assertFalse(port.test_isdir('passes/does_not_exist.html')) 372 self.assertFalse(port.test_isdir('passes/does_not_exist/')) 373 374 self.assertTrue(port.test_isdir('virtual')) 375 self.assertFalse(port.test_isdir('virtual/does_not_exist.html')) 376 self.assertFalse(port.test_isdir('virtual/does_not_exist/')) 377 self.assertFalse(port.test_isdir('virtual/passes/text.html')) 378 379 def test_tests(self): 380 port = self.make_port(with_tests=True) 381 tests = port.tests([]) 382 self.assertIn('passes/text.html', tests) 383 self.assertIn('virtual/passes/text.html', tests) 384 385 tests = port.tests(['passes']) 386 self.assertIn('passes/text.html', tests) 387 self.assertIn('passes/passes/test-virtual-passes.html', tests) 388 self.assertNotIn('virtual/passes/text.html', tests) 389 390 tests = port.tests(['virtual/passes']) 391 self.assertNotIn('passes/text.html', tests) 392 self.assertIn('virtual/passes/test-virtual-passes.html', tests) 393 self.assertIn('virtual/passes/passes/test-virtual-passes.html', tests) 394 self.assertNotIn('virtual/passes/test-virtual-virtual/passes.html', tests) 395 self.assertNotIn('virtual/passes/virtual/passes/test-virtual-passes.html', tests) 396 397 def test_build_path(self): 398 port = self.make_port(options=optparse.Values({'build_directory': '/my-build-directory/'})) 399 self.assertEqual(port._build_path(), '/my-build-directory/Release') 400 401 def test_dont_require_http_server(self): 402 port = self.make_port() 403 self.assertEqual(port.requires_http_server(), False) 404 405 406 class NaturalCompareTest(unittest.TestCase): 407 def setUp(self): 408 self._port = TestPort(MockSystemHost()) 409 410 def assert_cmp(self, x, y, result): 411 self.assertEqual(cmp(self._port._natural_sort_key(x), self._port._natural_sort_key(y)), result) 412 413 def test_natural_compare(self): 414 self.assert_cmp('a', 'a', 0) 415 self.assert_cmp('ab', 'a', 1) 416 self.assert_cmp('a', 'ab', -1) 417 self.assert_cmp('', '', 0) 418 self.assert_cmp('', 'ab', -1) 419 self.assert_cmp('1', '2', -1) 420 self.assert_cmp('2', '1', 1) 421 self.assert_cmp('1', '10', -1) 422 self.assert_cmp('2', '10', -1) 423 self.assert_cmp('foo_1.html', 'foo_2.html', -1) 424 self.assert_cmp('foo_1.1.html', 'foo_2.html', -1) 425 self.assert_cmp('foo_1.html', 'foo_10.html', -1) 426 self.assert_cmp('foo_2.html', 'foo_10.html', -1) 427 self.assert_cmp('foo_23.html', 'foo_10.html', 1) 428 self.assert_cmp('foo_23.html', 'foo_100.html', -1) 429 430 431 class KeyCompareTest(unittest.TestCase): 432 def setUp(self): 433 self._port = TestPort(MockSystemHost()) 434 435 def assert_cmp(self, x, y, result): 436 self.assertEqual(cmp(self._port.test_key(x), self._port.test_key(y)), result) 437 438 def test_test_key(self): 439 self.assert_cmp('/a', '/a', 0) 440 self.assert_cmp('/a', '/b', -1) 441 self.assert_cmp('/a2', '/a10', -1) 442 self.assert_cmp('/a2/foo', '/a10/foo', -1) 443 self.assert_cmp('/a/foo11', '/a/foo2', 1) 444 self.assert_cmp('/ab', '/a/a/b', -1) 445 self.assert_cmp('/a/a/b', '/ab', 1) 446 self.assert_cmp('/foo-bar/baz', '/foo/baz', -1) 447
