1 #!/usr/bin/env python 2 # Copyright (C) 2010 Google Inc. All rights reserved. 3 # 4 # Redistribution and use in source and binary forms, with or without 5 # modification, are permitted provided that the following conditions are 6 # met: 7 # 8 # * Redistributions of source code must retain the above copyright 9 # notice, this list of conditions and the following disclaimer. 10 # * Redistributions in binary form must reproduce the above 11 # copyright notice, this list of conditions and the following disclaimer 12 # in the documentation and/or other materials provided with the 13 # distribution. 14 # * Neither the Google name nor the names of its 15 # contributors may be used to endorse or promote products derived from 16 # this software without specific prior written permission. 17 # 18 # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 19 # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 20 # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 21 # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 22 # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 23 # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 24 # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 25 # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 26 # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 27 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 28 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 29 30 """Abstract base class of Port-specific entrypoints for the layout tests 31 test infrastructure (the Port and Driver classes).""" 32 33 import cgi 34 import difflib 35 import errno 36 import os 37 import subprocess 38 import sys 39 40 import apache_http_server 41 import http_server 42 import websocket_server 43 44 # Python bug workaround. See Port.wdiff_text() for an explanation. 45 _wdiff_available = True 46 47 48 # FIXME: This class should merge with webkitpy.webkit_port at some point. 49 class Port(object): 50 """Abstract class for Port-specific hooks for the layout_test package. 51 """ 52 53 def __init__(self, port_name=None, options=None): 54 self._name = port_name 55 self._options = options 56 self._helper = None 57 self._http_server = None 58 self._webkit_base_dir = None 59 self._websocket_server = None 60 61 def baseline_path(self): 62 """Return the absolute path to the directory to store new baselines 63 in for this port.""" 64 raise NotImplementedError('Port.baseline_path') 65 66 def baseline_search_path(self): 67 """Return a list of absolute paths to directories to search under for 68 baselines. The directories are searched in order.""" 69 raise NotImplementedError('Port.baseline_search_path') 70 71 def check_sys_deps(self): 72 """If the port needs to do some runtime checks to ensure that the 73 tests can be run successfully, they should be done here. 74 75 Returns whether the system is properly configured.""" 76 raise NotImplementedError('Port.check_sys_deps') 77 78 def compare_text(self, actual_text, expected_text): 79 """Return whether or not the two strings are *not* equal. This 80 routine is used to diff text output. 81 82 While this is a generic routine, we include it in the Port 83 interface so that it can be overriden for testing purposes.""" 84 return actual_text != expected_text 85 86 def diff_image(self, actual_filename, expected_filename, diff_filename): 87 """Compare two image files and produce a delta image file. 88 89 Return 1 if the two files are different, 0 if they are the same. 90 Also produce a delta image of the two images and write that into 91 |diff_filename|. 92 93 While this is a generic routine, we include it in the Port 94 interface so that it can be overriden for testing purposes.""" 95 executable = self._path_to_image_diff() 96 cmd = [executable, '--diff', actual_filename, expected_filename, 97 diff_filename] 98 result = 1 99 try: 100 result = subprocess.call(cmd) 101 except OSError, e: 102 if e.errno == errno.ENOENT or e.errno == errno.EACCES: 103 _compare_available = False 104 else: 105 raise e 106 except ValueError: 107 # work around a race condition in Python 2.4's implementation 108 # of subprocess.Popen. See http://bugs.python.org/issue1199282 . 109 pass 110 return result 111 112 def diff_text(self, actual_text, expected_text, 113 actual_filename, expected_filename): 114 """Returns a string containing the diff of the two text strings 115 in 'unified diff' format. 116 117 While this is a generic routine, we include it in the Port 118 interface so that it can be overriden for testing purposes.""" 119 diff = difflib.unified_diff(expected_text.splitlines(True), 120 actual_text.splitlines(True), 121 expected_filename, 122 actual_filename) 123 return ''.join(diff) 124 125 def expected_baselines(self, filename, suffix, all_baselines=False): 126 """Given a test name, finds where the baseline results are located. 127 128 Args: 129 filename: absolute filename to test file 130 suffix: file suffix of the expected results, including dot; e.g. 131 '.txt' or '.png'. This should not be None, but may be an empty 132 string. 133 all_baselines: If True, return an ordered list of all baseline paths 134 for the given platform. If False, return only the first one. 135 Returns 136 a list of ( platform_dir, results_filename ), where 137 platform_dir - abs path to the top of the results tree (or test 138 tree) 139 results_filename - relative path from top of tree to the results 140 file 141 (os.path.join of the two gives you the full path to the file, 142 unless None was returned.) 143 Return values will be in the format appropriate for the current 144 platform (e.g., "\\" for path separators on Windows). If the results 145 file is not found, then None will be returned for the directory, 146 but the expected relative pathname will still be returned. 147 148 This routine is generic but lives here since it is used in 149 conjunction with the other baseline and filename routines that are 150 platform specific. 151 """ 152 testname = os.path.splitext(self.relative_test_filename(filename))[0] 153 154 baseline_filename = testname + '-expected' + suffix 155 156 baseline_search_path = self.baseline_search_path() 157 158 baselines = [] 159 for platform_dir in baseline_search_path: 160 if os.path.exists(os.path.join(platform_dir, baseline_filename)): 161 baselines.append((platform_dir, baseline_filename)) 162 163 if not all_baselines and baselines: 164 return baselines 165 166 # If it wasn't found in a platform directory, return the expected 167 # result in the test directory, even if no such file actually exists. 168 platform_dir = self.layout_tests_dir() 169 if os.path.exists(os.path.join(platform_dir, baseline_filename)): 170 baselines.append((platform_dir, baseline_filename)) 171 172 if baselines: 173 return baselines 174 175 return [(None, baseline_filename)] 176 177 def expected_filename(self, filename, suffix): 178 """Given a test name, returns an absolute path to its expected results. 179 180 If no expected results are found in any of the searched directories, 181 the directory in which the test itself is located will be returned. 182 The return value is in the format appropriate for the platform 183 (e.g., "\\" for path separators on windows). 184 185 Args: 186 filename: absolute filename to test file 187 suffix: file suffix of the expected results, including dot; e.g. '.txt' 188 or '.png'. This should not be None, but may be an empty string. 189 platform: the most-specific directory name to use to build the 190 search list of directories, e.g., 'chromium-win', or 191 'chromium-mac-leopard' (we follow the WebKit format) 192 193 This routine is generic but is implemented here to live alongside 194 the other baseline and filename manipulation routines. 195 """ 196 platform_dir, baseline_filename = self.expected_baselines( 197 filename, suffix)[0] 198 if platform_dir: 199 return os.path.join(platform_dir, baseline_filename) 200 return os.path.join(self.layout_tests_dir(), baseline_filename) 201 202 def filename_to_uri(self, filename): 203 """Convert a test file to a URI.""" 204 LAYOUTTEST_HTTP_DIR = "http/tests/" 205 LAYOUTTEST_WEBSOCKET_DIR = "websocket/tests/" 206 207 relative_path = self.relative_test_filename(filename) 208 port = None 209 use_ssl = False 210 211 if relative_path.startswith(LAYOUTTEST_HTTP_DIR): 212 # http/tests/ run off port 8000 and ssl/ off 8443 213 relative_path = relative_path[len(LAYOUTTEST_HTTP_DIR):] 214 port = 8000 215 elif relative_path.startswith(LAYOUTTEST_WEBSOCKET_DIR): 216 # websocket/tests/ run off port 8880 and 9323 217 # Note: the root is /, not websocket/tests/ 218 port = 8880 219 220 # Make http/tests/local run as local files. This is to mimic the 221 # logic in run-webkit-tests. 222 # 223 # TODO(dpranke): remove the media reference and the SSL reference? 224 if (port and not relative_path.startswith("local/") and 225 not relative_path.startswith("media/")): 226 if relative_path.startswith("ssl/"): 227 port += 443 228 protocol = "https" 229 else: 230 protocol = "http" 231 return "%s://127.0.0.1:%u/%s" % (protocol, port, relative_path) 232 233 if sys.platform in ('cygwin', 'win32'): 234 return "file:///" + self.get_absolute_path(filename) 235 return "file://" + self.get_absolute_path(filename) 236 237 def get_absolute_path(self, filename): 238 """Return the absolute path in unix format for the given filename. 239 240 This routine exists so that platforms that don't use unix filenames 241 can convert accordingly.""" 242 return os.path.abspath(filename) 243 244 def layout_tests_dir(self): 245 """Return the absolute path to the top of the LayoutTests directory.""" 246 return self.path_from_webkit_base('LayoutTests') 247 248 def maybe_make_directory(self, *path): 249 """Creates the specified directory if it doesn't already exist.""" 250 try: 251 os.makedirs(os.path.join(*path)) 252 except OSError, e: 253 if e.errno != errno.EEXIST: 254 raise 255 256 def name(self): 257 """Return the name of the port (e.g., 'mac', 'chromium-win-xp'). 258 259 Note that this is different from the test_platform_name(), which 260 may be different (e.g., 'win-xp' instead of 'chromium-win-xp'.""" 261 return self._name 262 263 def num_cores(self): 264 """Return the number of cores/cpus available on this machine. 265 266 This routine is used to determine the default amount of parallelism 267 used by run-chromium-webkit-tests.""" 268 raise NotImplementedError('Port.num_cores') 269 270 def path_from_webkit_base(self, *comps): 271 """Returns the full path to path made by joining the top of the 272 WebKit source tree and the list of path components in |*comps|.""" 273 if not self._webkit_base_dir: 274 abspath = os.path.abspath(__file__) 275 self._webkit_base_dir = abspath[0:abspath.find('WebKitTools')] 276 return os.path.join(self._webkit_base_dir, *comps) 277 278 def remove_directory(self, *path): 279 """Recursively removes a directory, even if it's marked read-only. 280 281 Remove the directory located at *path, if it exists. 282 283 shutil.rmtree() doesn't work on Windows if any of the files 284 or directories are read-only, which svn repositories and 285 some .svn files are. We need to be able to force the files 286 to be writable (i.e., deletable) as we traverse the tree. 287 288 Even with all this, Windows still sometimes fails to delete a file, 289 citing a permission error (maybe something to do with antivirus 290 scans or disk indexing). The best suggestion any of the user 291 forums had was to wait a bit and try again, so we do that too. 292 It's hand-waving, but sometimes it works. :/ 293 """ 294 file_path = os.path.join(*path) 295 if not os.path.exists(file_path): 296 return 297 298 win32 = False 299 if sys.platform == 'win32': 300 win32 = True 301 # Some people don't have the APIs installed. In that case we'll do 302 # without. 303 try: 304 win32api = __import__('win32api') 305 win32con = __import__('win32con') 306 except ImportError: 307 win32 = False 308 309 def remove_with_retry(rmfunc, path): 310 os.chmod(path, stat.S_IWRITE) 311 if win32: 312 win32api.SetFileAttributes(path, 313 win32con.FILE_ATTRIBUTE_NORMAL) 314 try: 315 return rmfunc(path) 316 except EnvironmentError, e: 317 if e.errno != errno.EACCES: 318 raise 319 print 'Failed to delete %s: trying again' % repr(path) 320 time.sleep(0.1) 321 return rmfunc(path) 322 else: 323 324 def remove_with_retry(rmfunc, path): 325 if os.path.islink(path): 326 return os.remove(path) 327 else: 328 return rmfunc(path) 329 330 for root, dirs, files in os.walk(file_path, topdown=False): 331 # For POSIX: making the directory writable guarantees 332 # removability. Windows will ignore the non-read-only 333 # bits in the chmod value. 334 os.chmod(root, 0770) 335 for name in files: 336 remove_with_retry(os.remove, os.path.join(root, name)) 337 for name in dirs: 338 remove_with_retry(os.rmdir, os.path.join(root, name)) 339 340 remove_with_retry(os.rmdir, file_path) 341 342 def test_platform_name(self): 343 return self._name 344 345 def relative_test_filename(self, filename): 346 """Relative unix-style path for a filename under the LayoutTests 347 directory. Filenames outside the LayoutTests directory should raise 348 an error.""" 349 return filename[len(self.layout_tests_dir()) + 1:] 350 351 def results_directory(self): 352 """Absolute path to the place to store the test results.""" 353 raise NotImplemented('Port.results_directory') 354 355 def setup_test_run(self): 356 """This routine can be overridden to perform any port-specific 357 work that shouuld be done at the beginning of a test run.""" 358 pass 359 360 def show_html_results_file(self, results_filename): 361 """This routine should display the HTML file pointed at by 362 results_filename in a users' browser.""" 363 raise NotImplementedError('Port.show_html_results_file') 364 365 def start_driver(self, png_path, options): 366 """Starts a new test Driver and returns a handle to the object.""" 367 raise NotImplementedError('Port.start_driver') 368 369 def start_helper(self): 370 """Start a layout test helper if needed on this port. The test helper 371 is used to reconfigure graphics settings and do other things that 372 may be necessary to ensure a known test configuration.""" 373 raise NotImplementedError('Port.start_helper') 374 375 def start_http_server(self): 376 """Start a web server if it is available. Do nothing if 377 it isn't. This routine is allowed to (and may) fail if a server 378 is already running.""" 379 if self._options.use_apache: 380 self._http_server = apache_http_server.LayoutTestApacheHttpd(self, 381 self._options.results_directory) 382 else: 383 self._http_server = http_server.Lighttpd(self, 384 self._options.results_directory) 385 self._http_server.start() 386 387 def start_websocket_server(self): 388 """Start a websocket server if it is available. Do nothing if 389 it isn't. This routine is allowed to (and may) fail if a server 390 is already running.""" 391 self._websocket_server = websocket_server.PyWebSocket(self, 392 self._options.results_directory) 393 self._websocket_server.start() 394 395 def stop_helper(self): 396 """Shut down the test helper if it is running. Do nothing if 397 it isn't, or it isn't available.""" 398 raise NotImplementedError('Port.stop_helper') 399 400 def stop_http_server(self): 401 """Shut down the http server if it is running. Do nothing if 402 it isn't, or it isn't available.""" 403 if self._http_server: 404 self._http_server.stop() 405 406 def stop_websocket_server(self): 407 """Shut down the websocket server if it is running. Do nothing if 408 it isn't, or it isn't available.""" 409 if self._websocket_server: 410 self._websocket_server.stop() 411 412 def test_expectations(self): 413 """Returns the test expectations for this port. 414 415 Basically this string should contain the equivalent of a 416 test_expectations file. See test_expectations.py for more details.""" 417 raise NotImplementedError('Port.test_expectations') 418 419 def test_base_platform_names(self): 420 """Return a list of the 'base' platforms on your port. The base 421 platforms represent different architectures, operating systems, 422 or implementations (as opposed to different versions of a single 423 platform). For example, 'mac' and 'win' might be different base 424 platforms, wherease 'mac-tiger' and 'mac-leopard' might be 425 different platforms. This routine is used by the rebaselining tool 426 and the dashboards, and the strings correspond to the identifiers 427 in your test expectations (*not* necessarily the platform names 428 themselves).""" 429 raise NotImplementedError('Port.base_test_platforms') 430 431 def test_platform_name(self): 432 """Returns the string that corresponds to the given platform name 433 in the test expectations. This may be the same as name(), or it 434 may be different. For example, chromium returns 'mac' for 435 'chromium-mac'.""" 436 raise NotImplementedError('Port.test_platform_name') 437 438 def test_platforms(self): 439 """Returns the list of test platform identifiers as used in the 440 test_expectations and on dashboards, the rebaselining tool, etc. 441 442 Note that this is not necessarily the same as the list of ports, 443 which must be globally unique (e.g., both 'chromium-mac' and 'mac' 444 might return 'mac' as a test_platform name'.""" 445 raise NotImplementedError('Port.platforms') 446 447 def version(self): 448 """Returns a string indicating the version of a given platform, e.g. 449 '-leopard' or '-xp'. 450 451 This is used to help identify the exact port when parsing test 452 expectations, determining search paths, and logging information.""" 453 raise NotImplementedError('Port.version') 454 455 def wdiff_text(self, actual_filename, expected_filename): 456 """Returns a string of HTML indicating the word-level diff of the 457 contents of the two filenames. Returns an empty string if word-level 458 diffing isn't available.""" 459 executable = self._path_to_wdiff() 460 cmd = [executable, 461 '--start-delete=##WDIFF_DEL##', 462 '--end-delete=##WDIFF_END##', 463 '--start-insert=##WDIFF_ADD##', 464 '--end-insert=##WDIFF_END##', 465 expected_filename, 466 actual_filename] 467 global _wdiff_available 468 result = '' 469 try: 470 # Python's Popen has a bug that causes any pipes opened to a 471 # process that can't be executed to be leaked. Since this 472 # code is specifically designed to tolerate exec failures 473 # to gracefully handle cases where wdiff is not installed, 474 # the bug results in a massive file descriptor leak. As a 475 # workaround, if an exec failure is ever experienced for 476 # wdiff, assume it's not available. This will leak one 477 # file descriptor but that's better than leaking each time 478 # wdiff would be run. 479 # 480 # http://mail.python.org/pipermail/python-list/ 481 # 2008-August/505753.html 482 # http://bugs.python.org/issue3210 483 # 484 # It also has a threading bug, so we don't output wdiff if 485 # the Popen raises a ValueError. 486 # http://bugs.python.org/issue1236 487 if _wdiff_available: 488 try: 489 wdiff = subprocess.Popen(cmd, 490 stdout=subprocess.PIPE).communicate()[0] 491 except ValueError, e: 492 # Working around a race in Python 2.4's implementation 493 # of Popen(). 494 wdiff = '' 495 wdiff = cgi.escape(wdiff) 496 wdiff = wdiff.replace('##WDIFF_DEL##', '<span class=del>') 497 wdiff = wdiff.replace('##WDIFF_ADD##', '<span class=add>') 498 wdiff = wdiff.replace('##WDIFF_END##', '</span>') 499 result = '<head><style>.del { background: #faa; } ' 500 result += '.add { background: #afa; }</style></head>' 501 result += '<pre>' + wdiff + '</pre>' 502 except OSError, e: 503 if (e.errno == errno.ENOENT or e.errno == errno.EACCES or 504 e.errno == errno.ECHILD): 505 _wdiff_available = False 506 else: 507 raise e 508 return result 509 510 # 511 # PROTECTED ROUTINES 512 # 513 # The routines below should only be called by routines in this class 514 # or any of its subclasses. 515 # 516 517 def _kill_process(self, pid): 518 """Forcefully kill a process. 519 520 This routine should not be used or needed generically, but can be 521 used in helper files like http_server.py.""" 522 raise NotImplementedError('Port.kill_process') 523 524 def _path_to_apache(self): 525 """Returns the full path to the apache binary. 526 527 This is needed only by ports that use the apache_http_server module.""" 528 raise NotImplementedError('Port.path_to_apache') 529 530 def _path_to_apache_config_file(self): 531 """Returns the full path to the apache binary. 532 533 This is needed only by ports that use the apache_http_server module.""" 534 raise NotImplementedError('Port.path_to_apache_config_file') 535 536 def _path_to_driver(self): 537 """Returns the full path to the test driver (DumpRenderTree).""" 538 raise NotImplementedError('Port.path_to_driver') 539 540 def _path_to_helper(self): 541 """Returns the full path to the layout_test_helper binary, which 542 is used to help configure the system for the test run, or None 543 if no helper is needed. 544 545 This is likely only used by start/stop_helper().""" 546 raise NotImplementedError('Port._path_to_helper') 547 548 def _path_to_image_diff(self): 549 """Returns the full path to the image_diff binary, or None if it 550 is not available. 551 552 This is likely used only by diff_image()""" 553 raise NotImplementedError('Port.path_to_image_diff') 554 555 def _path_to_lighttpd(self): 556 """Returns the path to the LigHTTPd binary. 557 558 This is needed only by ports that use the http_server.py module.""" 559 raise NotImplementedError('Port._path_to_lighttpd') 560 561 def _path_to_lighttpd_modules(self): 562 """Returns the path to the LigHTTPd modules directory. 563 564 This is needed only by ports that use the http_server.py module.""" 565 raise NotImplementedError('Port._path_to_lighttpd_modules') 566 567 def _path_to_lighttpd_php(self): 568 """Returns the path to the LigHTTPd PHP executable. 569 570 This is needed only by ports that use the http_server.py module.""" 571 raise NotImplementedError('Port._path_to_lighttpd_php') 572 573 def _path_to_wdiff(self): 574 """Returns the full path to the wdiff binary, or None if it is 575 not available. 576 577 This is likely used only by wdiff_text()""" 578 raise NotImplementedError('Port._path_to_wdiff') 579 580 def _shut_down_http_server(self, pid): 581 """Forcefully and synchronously kills the web server. 582 583 This routine should only be called from http_server.py or its 584 subclasses.""" 585 raise NotImplementedError('Port._shut_down_http_server') 586 587 def _webkit_baseline_path(self, platform): 588 """Return the full path to the top of the baseline tree for a 589 given platform.""" 590 return os.path.join(self.layout_tests_dir(), 'platform', 591 platform) 592 593 594 class Driver: 595 """Abstract interface for the DumpRenderTree interface.""" 596 597 def __init__(self, port, png_path, options): 598 """Initialize a Driver to subsequently run tests. 599 600 Typically this routine will spawn DumpRenderTree in a config 601 ready for subsequent input. 602 603 port - reference back to the port object. 604 png_path - an absolute path for the driver to write any image 605 data for a test (as a PNG). If no path is provided, that 606 indicates that pixel test results will not be checked. 607 options - any port-specific driver options.""" 608 raise NotImplementedError('Driver.__init__') 609 610 def run_test(self, uri, timeout, checksum): 611 """Run a single test and return the results. 612 613 Note that it is okay if a test times out or crashes and leaves 614 the driver in an indeterminate state. The upper layers of the program 615 are responsible for cleaning up and ensuring things are okay. 616 617 uri - a full URI for the given test 618 timeout - number of milliseconds to wait before aborting this test. 619 checksum - if present, the expected checksum for the image for this 620 test 621 622 Returns a tuple of the following: 623 crash - a boolean indicating whether the driver crashed on the test 624 timeout - a boolean indicating whehter the test timed out 625 checksum - a string containing the checksum of the image, if 626 present 627 output - any text output 628 error - any unexpected or additional (or error) text output 629 630 Note that the image itself should be written to the path that was 631 specified in the __init__() call.""" 632 raise NotImplementedError('Driver.run_test') 633 634 def poll(self): 635 """Returns None if the Driver is still running. Returns the returncode 636 if it has exited.""" 637 raise NotImplementedError('Driver.poll') 638 639 def returncode(self): 640 """Returns the system-specific returncode if the Driver has stopped or 641 exited.""" 642 raise NotImplementedError('Driver.returncode') 643 644 def stop(self): 645 raise NotImplementedError('Driver.stop') 646
