1 # Copyright (C) 2010 Google Inc. All rights reserved. 2 # 3 # Redistribution and use in source and binary forms, with or without 4 # modification, are permitted provided that the following conditions are 5 # met: 6 # 7 # * Redistributions of source code must retain the above copyright 8 # notice, this list of conditions and the following disclaimer. 9 # * Redistributions in binary form must reproduce the above 10 # copyright notice, this list of conditions and the following disclaimer 11 # in the documentation and/or other materials provided with the 12 # distribution. 13 # * Neither the name of Google Inc. nor the names of its 14 # contributors may be used to endorse or promote products derived from 15 # this software without specific prior written permission. 16 # 17 # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 18 # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 19 # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 20 # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 21 # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 22 # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 23 # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 24 # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 25 # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 26 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 27 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 28 29 import logging 30 import optparse 31 import sys 32 import tempfile 33 import webkitpy.thirdparty.unittest2 as unittest 34 35 from webkitpy.common.system.executive import Executive, ScriptError 36 from webkitpy.common.system import executive_mock 37 from webkitpy.common.system.filesystem_mock import MockFileSystem 38 from webkitpy.common.system.outputcapture import OutputCapture 39 from webkitpy.common.system.path import abspath_to_uri 40 from webkitpy.thirdparty.mock import Mock 41 from webkitpy.tool.mocktool import MockOptions 42 from webkitpy.common.system.executive_mock import MockExecutive, MockExecutive2 43 from webkitpy.common.system.systemhost_mock import MockSystemHost 44 45 from webkitpy.layout_tests.port import Port, Driver, DriverOutput 46 from webkitpy.layout_tests.port.base import VirtualTestSuite 47 from webkitpy.layout_tests.port.test import add_unit_tests_to_mock_filesystem, TestPort 48 49 class PortTest(unittest.TestCase): 50 def make_port(self, executive=None, with_tests=False, port_name=None, **kwargs): 51 host = MockSystemHost() 52 if executive: 53 host.executive = executive 54 if with_tests: 55 add_unit_tests_to_mock_filesystem(host.filesystem) 56 return TestPort(host, **kwargs) 57 return Port(host, port_name or 'baseport', **kwargs) 58 59 def test_default_child_processes(self): 60 port = self.make_port() 61 self.assertIsNotNone(port.default_child_processes()) 62 63 def test_format_wdiff_output_as_html(self): 64 output = "OUTPUT %s %s %s" % (Port._WDIFF_DEL, Port._WDIFF_ADD, Port._WDIFF_END) 65 html = self.make_port()._format_wdiff_output_as_html(output) 66 expected_html = "<head><style>.del { background: #faa; } .add { background: #afa; }</style></head><pre>OUTPUT <span class=del> <span class=add> </span></pre>" 67 self.assertEqual(html, expected_html) 68 69 def test_wdiff_command(self): 70 port = self.make_port() 71 port._path_to_wdiff = lambda: "/path/to/wdiff" 72 command = port._wdiff_command("/actual/path", "/expected/path") 73 expected_command = [ 74 "/path/to/wdiff", 75 "--start-delete=##WDIFF_DEL##", 76 "--end-delete=##WDIFF_END##", 77 "--start-insert=##WDIFF_ADD##", 78 "--end-insert=##WDIFF_END##", 79 "/actual/path", 80 "/expected/path", 81 ] 82 self.assertEqual(command, expected_command) 83 84 def _file_with_contents(self, contents, encoding="utf-8"): 85 new_file = tempfile.NamedTemporaryFile() 86 new_file.write(contents.encode(encoding)) 87 new_file.flush() 88 return new_file 89 90 def test_pretty_patch_os_error(self): 91 port = self.make_port(executive=executive_mock.MockExecutive2(exception=OSError)) 92 oc = OutputCapture() 93 oc.capture_output() 94 self.assertEqual(port.pretty_patch_text("patch.txt"), 95 port._pretty_patch_error_html) 96 97 # This tests repeated calls to make sure we cache the result. 98 self.assertEqual(port.pretty_patch_text("patch.txt"), 99 port._pretty_patch_error_html) 100 oc.restore_output() 101 102 def test_pretty_patch_script_error(self): 103 # FIXME: This is some ugly white-box test hacking ... 104 port = self.make_port(executive=executive_mock.MockExecutive2(exception=ScriptError)) 105 port._pretty_patch_available = True 106 self.assertEqual(port.pretty_patch_text("patch.txt"), 107 port._pretty_patch_error_html) 108 109 # This tests repeated calls to make sure we cache the result. 110 self.assertEqual(port.pretty_patch_text("patch.txt"), 111 port._pretty_patch_error_html) 112 113 def test_wdiff_text(self): 114 port = self.make_port() 115 port.wdiff_available = lambda: True 116 port._run_wdiff = lambda a, b: 'PASS' 117 self.assertEqual('PASS', port.wdiff_text(None, None)) 118 119 def test_diff_text(self): 120 port = self.make_port() 121 # Make sure that we don't run into decoding exceptions when the 122 # filenames are unicode, with regular or malformed input (expected or 123 # actual input is always raw bytes, not unicode). 124 port.diff_text('exp', 'act', 'exp.txt', 'act.txt') 125 port.diff_text('exp', 'act', u'exp.txt', 'act.txt') 126 port.diff_text('exp', 'act', u'a\xac\u1234\u20ac\U00008000', 'act.txt') 127 128 port.diff_text('exp' + chr(255), 'act', 'exp.txt', 'act.txt') 129 port.diff_text('exp' + chr(255), 'act', u'exp.txt', 'act.txt') 130 131 # Though expected and actual files should always be read in with no 132 # encoding (and be stored as str objects), test unicode inputs just to 133 # be safe. 134 port.diff_text(u'exp', 'act', 'exp.txt', 'act.txt') 135 port.diff_text( 136 u'a\xac\u1234\u20ac\U00008000', 'act', 'exp.txt', 'act.txt') 137 138 # And make sure we actually get diff output. 139 diff = port.diff_text('foo', 'bar', 'exp.txt', 'act.txt') 140 self.assertIn('foo', diff) 141 self.assertIn('bar', diff) 142 self.assertIn('exp.txt', diff) 143 self.assertIn('act.txt', diff) 144 self.assertNotIn('nosuchthing', diff) 145 146 def test_setup_test_run(self): 147 port = self.make_port() 148 # This routine is a no-op. We just test it for coverage. 149 port.setup_test_run() 150 151 def test_test_dirs(self): 152 port = self.make_port() 153 port.host.filesystem.write_text_file(port.layout_tests_dir() + '/canvas/test', '') 154 port.host.filesystem.write_text_file(port.layout_tests_dir() + '/css2.1/test', '') 155 dirs = port.test_dirs() 156 self.assertIn('canvas', dirs) 157 self.assertIn('css2.1', dirs) 158 159 def test_skipped_perf_tests(self): 160 port = self.make_port() 161 162 def add_text_file(dirname, filename, content='some content'): 163 dirname = port.host.filesystem.join(port.perf_tests_dir(), dirname) 164 port.host.filesystem.maybe_make_directory(dirname) 165 port.host.filesystem.write_text_file(port.host.filesystem.join(dirname, filename), content) 166 167 add_text_file('inspector', 'test1.html') 168 add_text_file('inspector', 'unsupported_test1.html') 169 add_text_file('inspector', 'test2.html') 170 add_text_file('inspector/resources', 'resource_file.html') 171 add_text_file('unsupported', 'unsupported_test2.html') 172 add_text_file('', 'Skipped', '\n'.join(['Layout', '', 'SunSpider', 'Supported/some-test.html'])) 173 self.assertEqual(port.skipped_perf_tests(), ['Layout', 'SunSpider', 'Supported/some-test.html']) 174 175 def test_get_option__set(self): 176 options, args = optparse.OptionParser().parse_args([]) 177 options.foo = 'bar' 178 port = self.make_port(options=options) 179 self.assertEqual(port.get_option('foo'), 'bar') 180 181 def test_get_option__unset(self): 182 port = self.make_port() 183 self.assertIsNone(port.get_option('foo')) 184 185 def test_get_option__default(self): 186 port = self.make_port() 187 self.assertEqual(port.get_option('foo', 'bar'), 'bar') 188 189 def test_additional_platform_directory(self): 190 port = self.make_port(port_name='foo') 191 port.default_baseline_search_path = lambda: ['LayoutTests/platform/foo'] 192 layout_test_dir = port.layout_tests_dir() 193 test_file = 'fast/test.html' 194 195 # No additional platform directory 196 self.assertEqual( 197 port.expected_baselines(test_file, '.txt'), 198 [(None, 'fast/test-expected.txt')]) 199 self.assertEqual(port.baseline_path(), 'LayoutTests/platform/foo') 200 201 # Simple additional platform directory 202 port._options.additional_platform_directory = ['/tmp/local-baselines'] 203 port._filesystem.write_text_file('/tmp/local-baselines/fast/test-expected.txt', 'foo') 204 self.assertEqual( 205 port.expected_baselines(test_file, '.txt'), 206 [('/tmp/local-baselines', 'fast/test-expected.txt')]) 207 self.assertEqual(port.baseline_path(), '/tmp/local-baselines') 208 209 # Multiple additional platform directories 210 port._options.additional_platform_directory = ['/foo', '/tmp/local-baselines'] 211 self.assertEqual( 212 port.expected_baselines(test_file, '.txt'), 213 [('/tmp/local-baselines', 'fast/test-expected.txt')]) 214 self.assertEqual(port.baseline_path(), '/foo') 215 216 def test_nonexistant_expectations(self): 217 port = self.make_port(port_name='foo') 218 port.expectations_files = lambda: ['/mock-checkout/third_party/WebKit/LayoutTests/platform/exists/TestExpectations', '/mock-checkout/third_party/WebKit/LayoutTests/platform/nonexistant/TestExpectations'] 219 port._filesystem.write_text_file('/mock-checkout/third_party/WebKit/LayoutTests/platform/exists/TestExpectations', '') 220 self.assertEqual('\n'.join(port.expectations_dict().keys()), '/mock-checkout/third_party/WebKit/LayoutTests/platform/exists/TestExpectations') 221 222 def test_additional_expectations(self): 223 port = self.make_port(port_name='foo') 224 port.port_name = 'foo' 225 port._filesystem.write_text_file('/mock-checkout/third_party/WebKit/LayoutTests/platform/foo/TestExpectations', '') 226 port._filesystem.write_text_file( 227 '/tmp/additional-expectations-1.txt', 'content1\n') 228 port._filesystem.write_text_file( 229 '/tmp/additional-expectations-2.txt', 'content2\n') 230 231 self.assertEqual('\n'.join(port.expectations_dict().values()), '') 232 233 port._options.additional_expectations = [ 234 '/tmp/additional-expectations-1.txt'] 235 self.assertEqual('\n'.join(port.expectations_dict().values()), 'content1\n') 236 237 port._options.additional_expectations = [ 238 '/tmp/nonexistent-file', '/tmp/additional-expectations-1.txt'] 239 self.assertEqual('\n'.join(port.expectations_dict().values()), 'content1\n') 240 241 port._options.additional_expectations = [ 242 '/tmp/additional-expectations-1.txt', '/tmp/additional-expectations-2.txt'] 243 self.assertEqual('\n'.join(port.expectations_dict().values()), 'content1\n\ncontent2\n') 244 245 def test_additional_env_var(self): 246 port = self.make_port(options=optparse.Values({'additional_env_var': ['FOO=BAR', 'BAR=FOO']})) 247 self.assertEqual(port.get_option('additional_env_var'), ['FOO=BAR', 'BAR=FOO']) 248 environment = port.setup_environ_for_server() 249 self.assertTrue(('FOO' in environment) & ('BAR' in environment)) 250 self.assertEqual(environment['FOO'], 'BAR') 251 self.assertEqual(environment['BAR'], 'FOO') 252 253 def test_find_no_paths_specified(self): 254 port = self.make_port(with_tests=True) 255 layout_tests_dir = port.layout_tests_dir() 256 tests = port.tests([]) 257 self.assertNotEqual(len(tests), 0) 258 259 def test_find_one_test(self): 260 port = self.make_port(with_tests=True) 261 tests = port.tests(['failures/expected/image.html']) 262 self.assertEqual(len(tests), 1) 263 264 def test_find_glob(self): 265 port = self.make_port(with_tests=True) 266 tests = port.tests(['failures/expected/im*']) 267 self.assertEqual(len(tests), 2) 268 269 def test_find_with_skipped_directories(self): 270 port = self.make_port(with_tests=True) 271 tests = port.tests(['userscripts']) 272 self.assertNotIn('userscripts/resources/iframe.html', tests) 273 274 def test_find_with_skipped_directories_2(self): 275 port = self.make_port(with_tests=True) 276 tests = port.tests(['userscripts/resources']) 277 self.assertEqual(tests, []) 278 279 def test_is_test_file(self): 280 filesystem = MockFileSystem() 281 self.assertTrue(Port.is_test_file(filesystem, '', 'foo.html')) 282 self.assertTrue(Port.is_test_file(filesystem, '', 'foo.svg')) 283 self.assertTrue(Port.is_test_file(filesystem, '', 'test-ref-test.html')) 284 self.assertFalse(Port.is_test_file(filesystem, '', 'foo.png')) 285 self.assertFalse(Port.is_test_file(filesystem, '', 'foo-expected.html')) 286 self.assertFalse(Port.is_test_file(filesystem, '', 'foo-expected.svg')) 287 self.assertFalse(Port.is_test_file(filesystem, '', 'foo-expected.xht')) 288 self.assertFalse(Port.is_test_file(filesystem, '', 'foo-expected-mismatch.html')) 289 self.assertFalse(Port.is_test_file(filesystem, '', 'foo-expected-mismatch.svg')) 290 self.assertFalse(Port.is_test_file(filesystem, '', 'foo-expected-mismatch.xhtml')) 291 self.assertFalse(Port.is_test_file(filesystem, '', 'foo-ref.html')) 292 self.assertFalse(Port.is_test_file(filesystem, '', 'foo-notref.html')) 293 self.assertFalse(Port.is_test_file(filesystem, '', 'foo-notref.xht')) 294 self.assertFalse(Port.is_test_file(filesystem, '', 'foo-ref.xhtml')) 295 self.assertFalse(Port.is_test_file(filesystem, '', 'ref-foo.html')) 296 self.assertFalse(Port.is_test_file(filesystem, '', 'notref-foo.xhr')) 297 298 def test_parse_reftest_list(self): 299 port = self.make_port(with_tests=True) 300 port.host.filesystem.files['bar/reftest.list'] = "\n".join(["== test.html test-ref.html", 301 "", 302 "# some comment", 303 "!= test-2.html test-notref.html # more comments", 304 "== test-3.html test-ref.html", 305 "== test-3.html test-ref2.html", 306 "!= test-3.html test-notref.html", 307 "fuzzy(80,500) == test-3 test-ref.html"]) 308 309 # Note that we don't support the syntax in the last line; the code should ignore it, rather than crashing. 310 311 reftest_list = Port._parse_reftest_list(port.host.filesystem, 'bar') 312 self.assertEqual(reftest_list, {'bar/test.html': [('==', 'bar/test-ref.html')], 313 'bar/test-2.html': [('!=', 'bar/test-notref.html')], 314 'bar/test-3.html': [('==', 'bar/test-ref.html'), ('==', 'bar/test-ref2.html'), ('!=', 'bar/test-notref.html')]}) 315 316 def test_reference_files(self): 317 port = self.make_port(with_tests=True) 318 self.assertEqual(port.reference_files('passes/svgreftest.svg'), [('==', port.layout_tests_dir() + '/passes/svgreftest-expected.svg')]) 319 self.assertEqual(port.reference_files('passes/xhtreftest.svg'), [('==', port.layout_tests_dir() + '/passes/xhtreftest-expected.html')]) 320 self.assertEqual(port.reference_files('passes/phpreftest.php'), [('!=', port.layout_tests_dir() + '/passes/phpreftest-expected-mismatch.svg')]) 321 322 def test_operating_system(self): 323 self.assertEqual('mac', self.make_port().operating_system()) 324 325 def test_http_server_supports_ipv6(self): 326 port = self.make_port() 327 self.assertTrue(port.http_server_supports_ipv6()) 328 port.host.platform.os_name = 'cygwin' 329 self.assertFalse(port.http_server_supports_ipv6()) 330 port.host.platform.os_name = 'win' 331 self.assertTrue(port.http_server_supports_ipv6()) 332 333 def test_check_httpd_success(self): 334 port = self.make_port(executive=MockExecutive2()) 335 port.path_to_apache = lambda: '/usr/sbin/httpd' 336 capture = OutputCapture() 337 capture.capture_output() 338 self.assertTrue(port.check_httpd()) 339 _, _, logs = capture.restore_output() 340 self.assertEqual('', logs) 341 342 def test_httpd_returns_error_code(self): 343 port = self.make_port(executive=MockExecutive2(exit_code=1)) 344 port.path_to_apache = lambda: '/usr/sbin/httpd' 345 capture = OutputCapture() 346 capture.capture_output() 347 self.assertFalse(port.check_httpd()) 348 _, _, logs = capture.restore_output() 349 self.assertEqual('httpd seems broken. Cannot run http tests.\n', logs) 350 351 def test_test_exists(self): 352 port = self.make_port(with_tests=True) 353 self.assertTrue(port.test_exists('passes')) 354 self.assertTrue(port.test_exists('passes/text.html')) 355 self.assertFalse(port.test_exists('passes/does_not_exist.html')) 356 357 self.assertTrue(port.test_exists('virtual')) 358 self.assertFalse(port.test_exists('virtual/does_not_exist.html')) 359 self.assertTrue(port.test_exists('virtual/passes/text.html')) 360 361 def test_test_isfile(self): 362 port = self.make_port(with_tests=True) 363 self.assertFalse(port.test_isfile('passes')) 364 self.assertTrue(port.test_isfile('passes/text.html')) 365 self.assertFalse(port.test_isfile('passes/does_not_exist.html')) 366 367 self.assertFalse(port.test_isfile('virtual')) 368 self.assertTrue(port.test_isfile('virtual/passes/text.html')) 369 self.assertFalse(port.test_isfile('virtual/does_not_exist.html')) 370 371 def test_test_isdir(self): 372 port = self.make_port(with_tests=True) 373 self.assertTrue(port.test_isdir('passes')) 374 self.assertFalse(port.test_isdir('passes/text.html')) 375 self.assertFalse(port.test_isdir('passes/does_not_exist.html')) 376 self.assertFalse(port.test_isdir('passes/does_not_exist/')) 377 378 self.assertTrue(port.test_isdir('virtual')) 379 self.assertFalse(port.test_isdir('virtual/does_not_exist.html')) 380 self.assertFalse(port.test_isdir('virtual/does_not_exist/')) 381 self.assertFalse(port.test_isdir('virtual/passes/text.html')) 382 383 def test_tests(self): 384 port = self.make_port(with_tests=True) 385 tests = port.tests([]) 386 self.assertIn('passes/text.html', tests) 387 self.assertIn('virtual/passes/text.html', tests) 388 389 tests = port.tests(['passes']) 390 self.assertIn('passes/text.html', tests) 391 self.assertIn('passes/passes/test-virtual-passes.html', tests) 392 self.assertNotIn('virtual/passes/text.html', tests) 393 394 tests = port.tests(['virtual/passes']) 395 self.assertNotIn('passes/text.html', tests) 396 self.assertIn('virtual/passes/test-virtual-passes.html', tests) 397 self.assertIn('virtual/passes/passes/test-virtual-passes.html', tests) 398 self.assertNotIn('virtual/passes/test-virtual-virtual/passes.html', tests) 399 self.assertNotIn('virtual/passes/virtual/passes/test-virtual-passes.html', tests) 400 401 def test_build_path(self): 402 port = self.make_port(options=optparse.Values({'build_directory': '/my-build-directory/'})) 403 self.assertEqual(port._build_path(), '/my-build-directory/Release') 404 405 def test_dont_require_http_server(self): 406 port = self.make_port() 407 self.assertEqual(port.requires_http_server(), False) 408 409 410 class NaturalCompareTest(unittest.TestCase): 411 def setUp(self): 412 self._port = TestPort(MockSystemHost()) 413 414 def assert_cmp(self, x, y, result): 415 self.assertEqual(cmp(self._port._natural_sort_key(x), self._port._natural_sort_key(y)), result) 416 417 def test_natural_compare(self): 418 self.assert_cmp('a', 'a', 0) 419 self.assert_cmp('ab', 'a', 1) 420 self.assert_cmp('a', 'ab', -1) 421 self.assert_cmp('', '', 0) 422 self.assert_cmp('', 'ab', -1) 423 self.assert_cmp('1', '2', -1) 424 self.assert_cmp('2', '1', 1) 425 self.assert_cmp('1', '10', -1) 426 self.assert_cmp('2', '10', -1) 427 self.assert_cmp('foo_1.html', 'foo_2.html', -1) 428 self.assert_cmp('foo_1.1.html', 'foo_2.html', -1) 429 self.assert_cmp('foo_1.html', 'foo_10.html', -1) 430 self.assert_cmp('foo_2.html', 'foo_10.html', -1) 431 self.assert_cmp('foo_23.html', 'foo_10.html', 1) 432 self.assert_cmp('foo_23.html', 'foo_100.html', -1) 433 434 435 class KeyCompareTest(unittest.TestCase): 436 def setUp(self): 437 self._port = TestPort(MockSystemHost()) 438 439 def assert_cmp(self, x, y, result): 440 self.assertEqual(cmp(self._port.test_key(x), self._port.test_key(y)), result) 441 442 def test_test_key(self): 443 self.assert_cmp('/a', '/a', 0) 444 self.assert_cmp('/a', '/b', -1) 445 self.assert_cmp('/a2', '/a10', -1) 446 self.assert_cmp('/a2/foo', '/a10/foo', -1) 447 self.assert_cmp('/a/foo11', '/a/foo2', 1) 448 self.assert_cmp('/ab', '/a/a/b', -1) 449 self.assert_cmp('/a/a/b', '/ab', 1) 450 self.assert_cmp('/foo-bar/baz', '/foo/baz', -1) 451 452 453 class VirtualTestSuiteTest(unittest.TestCase): 454 def test_basic(self): 455 suite = VirtualTestSuite('suite', 'base/foo', ['--args']) 456 self.assertEqual(suite.name, 'virtual/suite/base/foo') 457 self.assertEqual(suite.base, 'base/foo') 458 self.assertEqual(suite.args, ['--args']) 459 460 def test_no_slash(self): 461 suite = VirtualTestSuite('suite/bar', 'base/foo', ['--args']) 462 self.assertFalse(hasattr(suite, 'name')) 463 self.assertFalse(hasattr(suite, 'base')) 464 self.assertFalse(hasattr(suite, 'args')) 465 466 def test_legacy(self): 467 suite = VirtualTestSuite('suite/bar', 'base/foo', ['--args'], use_legacy_naming=True) 468 self.assertEqual(suite.name, 'virtual/suite/bar') 469 self.assertEqual(suite.base, 'base/foo') 470 self.assertEqual(suite.args, ['--args']) 471
