1 #!/usr/bin/python 2 # Copyright (C) 2010 Google Inc. All rights reserved. 3 # 4 # Redistribution and use in source and binary forms, with or without 5 # modification, are permitted provided that the following conditions are 6 # met: 7 # 8 # * Redistributions of source code must retain the above copyright 9 # notice, this list of conditions and the following disclaimer. 10 # * Redistributions in binary form must reproduce the above 11 # copyright notice, this list of conditions and the following disclaimer 12 # in the documentation and/or other materials provided with the 13 # distribution. 14 # * Neither the name of Google Inc. nor the names of its 15 # contributors may be used to endorse or promote products derived from 16 # this software without specific prior written permission. 17 # 18 # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 19 # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 20 # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 21 # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 22 # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 23 # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 24 # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 25 # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 26 # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 27 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 28 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 29 30 """Unit tests for rebaseline_chromium_webkit_tests.py.""" 31 32 import unittest 33 34 from webkitpy.tool import mocktool 35 from webkitpy.common.system import urlfetcher_mock 36 from webkitpy.common.system import filesystem_mock 37 from webkitpy.common.system import zipfileset_mock 38 from webkitpy.common.system import outputcapture 39 from webkitpy.common.system.executive import Executive, ScriptError 40 41 from webkitpy.layout_tests import port 42 from webkitpy.layout_tests import rebaseline_chromium_webkit_tests 43 44 45 class MockPort(object): 46 def __init__(self, image_diff_exists): 47 self.image_diff_exists = image_diff_exists 48 49 def check_image_diff(self, override_step, logging): 50 return self.image_diff_exists 51 52 53 def get_mock_get(config_expectations): 54 def mock_get(port_name, options): 55 return MockPort(config_expectations[options.configuration]) 56 return mock_get 57 58 59 ARCHIVE_URL = 'http://localhost/layout_test_results' 60 61 62 def test_options(): 63 return mocktool.MockOptions(configuration=None, 64 backup=False, 65 html_directory='/tmp', 66 archive_url=ARCHIVE_URL, 67 force_archive_url=None, 68 verbose=False, 69 quiet=False, 70 platforms=None) 71 72 73 def test_host_port_and_filesystem(options, expectations): 74 filesystem = port.unit_test_filesystem() 75 host_port_obj = port.get('test', options, filesystem=filesystem, 76 user=mocktool.MockUser()) 77 78 expectations_path = host_port_obj.path_to_test_expectations_file() 79 filesystem.write_text_file(expectations_path, expectations) 80 return (host_port_obj, filesystem) 81 82 83 def test_url_fetcher(filesystem): 84 urls = { 85 ARCHIVE_URL + '/Webkit_Mac10_6/': '<a href="4/">', 86 ARCHIVE_URL + '/Webkit_Mac10_5/': '<a href="1/"><a href="2/">', 87 ARCHIVE_URL + '/Webkit_Win7/': '<a href="1/">', 88 ARCHIVE_URL + '/Webkit_Vista/': '<a href="1/">', 89 ARCHIVE_URL + '/Webkit_Win/': '<a href="1/">', 90 ARCHIVE_URL + '/Webkit_Linux/': '<a href="1/">', 91 } 92 return urlfetcher_mock.make_fetcher_cls(urls)(filesystem) 93 94 95 def test_zip_factory(): 96 ziphashes = { 97 ARCHIVE_URL + '/Webkit_Mac10_5/2/layout-test-results.zip': { 98 'layout-test-results/failures/expected/image-actual.txt': 'new-image-txt', 99 'layout-test-results/failures/expected/image-actual.checksum': 'new-image-checksum', 100 'layout-test-results/failures/expected/image-actual.png': 'new-image-png', 101 'layout-test-results/failures/expected/image_checksum-actual.txt': 'png-comment-txt', 102 'layout-test-results/failures/expected/image_checksum-actual.checksum': '0123456789', 103 'layout-test-results/failures/expected/image_checksum-actual.png': 'tEXtchecksum\x000123456789', 104 }, 105 ARCHIVE_URL + '/Webkit_Mac10_6/4/layout-test-results.zip': { 106 'layout-test-results/failures/expected/image-actual.txt': 'new-image-txt', 107 'layout-test-results/failures/expected/image-actual.checksum': 'new-image-checksum', 108 'layout-test-results/failures/expected/image-actual.png': 'new-image-png', 109 }, 110 ARCHIVE_URL + '/Webkit_Vista/1/layout-test-results.zip': { 111 'layout-test-results/failures/expected/image-actual.txt': 'win-image-txt', 112 'layout-test-results/failures/expected/image-actual.checksum': 'win-image-checksum', 113 'layout-test-results/failures/expected/image-actual.png': 'win-image-png', 114 }, 115 ARCHIVE_URL + '/Webkit_Win7/1/layout-test-results.zip': { 116 'layout-test-results/failures/expected/image-actual.txt': 'win-image-txt', 117 'layout-test-results/failures/expected/image-actual.checksum': 'win-image-checksum', 118 'layout-test-results/failures/expected/image-actual.png': 'win-image-png', 119 }, 120 ARCHIVE_URL + '/Webkit_Win/1/layout-test-results.zip': { 121 'layout-test-results/failures/expected/image-actual.txt': 'win-image-txt', 122 'layout-test-results/failures/expected/image-actual.checksum': 'win-image-checksum', 123 'layout-test-results/failures/expected/image-actual.png': 'win-image-png', 124 }, 125 ARCHIVE_URL + '/Webkit_Linux/1/layout-test-results.zip': { 126 'layout-test-results/failures/expected/image-actual.txt': 'win-image-txt', 127 'layout-test-results/failures/expected/image-actual.checksum': 'win-image-checksum', 128 'layout-test-results/failures/expected/image-actual.png': 'win-image-png', 129 }, 130 } 131 return zipfileset_mock.make_factory(ziphashes) 132 133 134 def test_archive(orig_archive_dict): 135 new_archive_dict = {} 136 for platform, dirname in orig_archive_dict.iteritems(): 137 platform = platform.replace('chromium', 'test') 138 new_archive_dict[platform] = dirname 139 return new_archive_dict 140 141 142 class TestGetHostPortObject(unittest.TestCase): 143 def assert_result(self, release_present, debug_present, valid_port_obj): 144 # Tests whether we get a valid port object returned when we claim 145 # that Image diff is (or isn't) present in the two configs. 146 port.get = get_mock_get({'Release': release_present, 147 'Debug': debug_present}) 148 options = mocktool.MockOptions(configuration=None, 149 html_directory='/tmp') 150 port_obj = rebaseline_chromium_webkit_tests.get_host_port_object(options) 151 if valid_port_obj: 152 self.assertNotEqual(port_obj, None) 153 else: 154 self.assertEqual(port_obj, None) 155 156 def test_get_host_port_object(self): 157 # Save the normal port.get() function for future testing. 158 old_get = port.get 159 160 # Test whether we get a valid port object back for the four 161 # possible cases of having ImageDiffs built. It should work when 162 # there is at least one binary present. 163 self.assert_result(False, False, False) 164 self.assert_result(True, False, True) 165 self.assert_result(False, True, True) 166 self.assert_result(True, True, True) 167 168 # Restore the normal port.get() function. 169 port.get = old_get 170 171 172 class TestOptions(unittest.TestCase): 173 def test_parse_options(self): 174 (options, target_options) = rebaseline_chromium_webkit_tests.parse_options([]) 175 self.assertTrue(target_options.chromium) 176 self.assertEqual(options.tolerance, 0) 177 178 (options, target_options) = rebaseline_chromium_webkit_tests.parse_options(['--target-platform', 'qt']) 179 self.assertFalse(hasattr(target_options, 'chromium')) 180 self.assertEqual(options.tolerance, 0) 181 182 183 class TestRebaseliner(unittest.TestCase): 184 def setUp(self): 185 if not hasattr(self, '_orig_archive'): 186 self._orig_archive = rebaseline_chromium_webkit_tests.ARCHIVE_DIR_NAME_DICT 187 rebaseline_chromium_webkit_tests.ARCHIVE_DIR_NAME_DICT = test_archive(self._orig_archive) 188 189 def tearDown(self): 190 rebaseline_chromium_webkit_tests.ARCHIVE_DIR_NAME_DICT = self._orig_archive 191 192 def make_rebaseliner(self, expectations): 193 options = test_options() 194 host_port_obj, filesystem = test_host_port_and_filesystem(options, expectations) 195 196 target_options = options 197 target_port_obj = port.get('test', target_options, 198 filesystem=filesystem) 199 target_port_obj._expectations = expectations 200 platform = target_port_obj.name() 201 202 url_fetcher = test_url_fetcher(filesystem) 203 zip_factory = test_zip_factory() 204 mock_scm = mocktool.MockSCM(filesystem) 205 rebaseliner = rebaseline_chromium_webkit_tests.Rebaseliner(host_port_obj, 206 target_port_obj, platform, options, url_fetcher, zip_factory, mock_scm) 207 return rebaseliner, filesystem 208 209 def test_noop(self): 210 # this method tests that was can at least instantiate an object, even 211 # if there is nothing to do. 212 rebaseliner, filesystem = self.make_rebaseliner("") 213 rebaseliner.run() 214 self.assertEqual(len(filesystem.written_files), 1) 215 216 def test_rebaselining_tests(self): 217 rebaseliner, filesystem = self.make_rebaseliner( 218 "BUGX REBASELINE MAC : failures/expected/image.html = IMAGE") 219 compile_success = rebaseliner._compile_rebaselining_tests() 220 self.assertTrue(compile_success) 221 self.assertEqual(set(['failures/expected/image.html']), rebaseliner._rebaselining_tests) 222 223 def test_rebaselining_tests_should_ignore_reftests(self): 224 rebaseliner, filesystem = self.make_rebaseliner( 225 "BUGX REBASELINE : failures/expected/reftest.html = IMAGE") 226 compile_success = rebaseliner._compile_rebaselining_tests() 227 self.assertFalse(compile_success) 228 self.assertFalse(rebaseliner._rebaselining_tests) 229 230 def test_one_platform(self): 231 rebaseliner, filesystem = self.make_rebaseliner( 232 "BUGX REBASELINE MAC : failures/expected/image.html = IMAGE") 233 rebaseliner.run() 234 # We expect to have written 12 files over the course of this rebaseline: 235 # *) 3 files in /__im_tmp for the extracted archive members 236 # *) 3 new baselines under '/test.checkout/LayoutTests' 237 # *) 4 files in /tmp for the new and old baselines in the result file 238 # (-{old,new}.{txt,png} 239 # *) 1 text diff in /tmp for the result file (-diff.txt). We don't 240 # create image diffs (FIXME?) and don't display the checksums. 241 # *) 1 updated test_expectations file 242 self.assertEqual(len(filesystem.written_files), 12) 243 self.assertEqual(filesystem.files['/test.checkout/LayoutTests/platform/test-mac-leopard/failures/expected/image-expected.checksum'], 'new-image-checksum') 244 self.assertEqual(filesystem.files['/test.checkout/LayoutTests/platform/test-mac-leopard/failures/expected/image-expected.png'], 'new-image-png') 245 self.assertEqual(filesystem.files['/test.checkout/LayoutTests/platform/test-mac-leopard/failures/expected/image-expected.txt'], 'new-image-txt') 246 247 def test_all_platforms(self): 248 rebaseliner, filesystem = self.make_rebaseliner( 249 "BUGX REBASELINE : failures/expected/image.html = IMAGE") 250 rebaseliner.run() 251 # See comment in test_one_platform for an explanation of the 12 written tests. 252 # Note that even though the rebaseline is marked for all platforms, each 253 # rebaseliner only ever does one. 254 self.assertEqual(len(filesystem.written_files), 12) 255 self.assertEqual(filesystem.files['/test.checkout/LayoutTests/platform/test-mac-leopard/failures/expected/image-expected.checksum'], 'new-image-checksum') 256 self.assertEqual(filesystem.files['/test.checkout/LayoutTests/platform/test-mac-leopard/failures/expected/image-expected.png'], 'new-image-png') 257 self.assertEqual(filesystem.files['/test.checkout/LayoutTests/platform/test-mac-leopard/failures/expected/image-expected.txt'], 'new-image-txt') 258 259 def test_png_file_with_comment(self): 260 rebaseliner, filesystem = self.make_rebaseliner( 261 "BUGX REBASELINE MAC : failures/expected/image_checksum.html = IMAGE") 262 compile_success = rebaseliner._compile_rebaselining_tests() 263 self.assertTrue(compile_success) 264 self.assertEqual(set(['failures/expected/image_checksum.html']), rebaseliner._rebaselining_tests) 265 rebaseliner.run() 266 # There is one less file written than |test_one_platform| because we only 267 # write 2 expectations (the png and the txt file). 268 self.assertEqual(len(filesystem.written_files), 11) 269 self.assertEqual(filesystem.files['/test.checkout/LayoutTests/platform/test-mac-leopard/failures/expected/image_checksum-expected.png'], 'tEXtchecksum\x000123456789') 270 self.assertEqual(filesystem.files['/test.checkout/LayoutTests/platform/test-mac-leopard/failures/expected/image_checksum-expected.txt'], 'png-comment-txt') 271 self.assertFalse(filesystem.files.get('/test.checkout/LayoutTests/platform/test-mac-leopard/failures/expected/image_checksum-expected.checksum', None)) 272 273 def test_png_file_with_comment_remove_old_checksum(self): 274 rebaseliner, filesystem = self.make_rebaseliner( 275 "BUGX REBASELINE MAC : failures/expected/image_checksum.html = IMAGE") 276 filesystem.files['/test.checkout/LayoutTests/platform/test-mac-leopard/failures/expected/image_checksum-expected.png'] = 'old' 277 filesystem.files['/test.checkout/LayoutTests/platform/test-mac-leopard/failures/expected/image_checksum-expected.checksum'] = 'old' 278 filesystem.files['/test.checkout/LayoutTests/platform/test-mac-leopard/failures/expected/image_checksum-expected.txt'] = 'old' 279 280 compile_success = rebaseliner._compile_rebaselining_tests() 281 self.assertTrue(compile_success) 282 self.assertEqual(set(['failures/expected/image_checksum.html']), rebaseliner._rebaselining_tests) 283 rebaseliner.run() 284 # There is one more file written than |test_png_file_with_comment_remove_old_checksum| 285 # because we also delete the old checksum. 286 self.assertEqual(len(filesystem.written_files), 12) 287 self.assertEqual(filesystem.files['/test.checkout/LayoutTests/platform/test-mac-leopard/failures/expected/image_checksum-expected.png'], 'tEXtchecksum\x000123456789') 288 self.assertEqual(filesystem.files['/test.checkout/LayoutTests/platform/test-mac-leopard/failures/expected/image_checksum-expected.txt'], 'png-comment-txt') 289 self.assertEqual(filesystem.files.get('/test.checkout/LayoutTests/platform/test-mac-leopard/failures/expected/image_checksum-expected.checksum', None), None) 290 291 def test_png_file_with_comment_as_duplicate(self): 292 rebaseliner, filesystem = self.make_rebaseliner( 293 "BUGX REBASELINE MAC : failures/expected/image_checksum.html = IMAGE") 294 filesystem.files['/test.checkout/LayoutTests/platform/test-mac-snowleopard/failures/expected/image_checksum-expected.png'] = 'tEXtchecksum\x000123456789' 295 filesystem.files['/test.checkout/LayoutTests/platform/test-mac-snowleopard/failures/expected/image_checksum-expected.txt'] = 'png-comment-txt' 296 297 compile_success = rebaseliner._compile_rebaselining_tests() 298 self.assertTrue(compile_success) 299 self.assertEqual(set(['failures/expected/image_checksum.html']), rebaseliner._rebaselining_tests) 300 rebaseliner.run() 301 self.assertEqual(filesystem.files.get('/test.checkout/LayoutTests/platform/test-mac-leopard/failures/expected/image_checksum-expected.png', None), None) 302 self.assertEqual(filesystem.files.get('/test.checkout/LayoutTests/platform/test-mac-leopard/failures/expected/image_checksum-expected.txt', None), None) 303 self.assertEqual(filesystem.files.get('/test.checkout/LayoutTests/platform/test-mac-leopard/failures/expected/image_checksum-expected.checksum', None), None) 304 305 def test_diff_baselines_txt(self): 306 rebaseliner, filesystem = self.make_rebaseliner("") 307 port = rebaseliner._port 308 output = port.expected_text( 309 port._filesystem.join(port.layout_tests_dir(), 'passes/text.html')) 310 self.assertFalse(rebaseliner._diff_baselines(output, output, 311 is_image=False)) 312 313 def test_diff_baselines_png(self): 314 rebaseliner, filesystem = self.make_rebaseliner('') 315 port = rebaseliner._port 316 image = port.expected_image( 317 port._filesystem.join(port.layout_tests_dir(), 'passes/image.html')) 318 self.assertFalse(rebaseliner._diff_baselines(image, image, 319 is_image=True)) 320 321 322 class TestRealMain(unittest.TestCase): 323 def setUp(self): 324 if not hasattr(self, '_orig_archive'): 325 self._orig_archive = rebaseline_chromium_webkit_tests.ARCHIVE_DIR_NAME_DICT 326 rebaseline_chromium_webkit_tests.ARCHIVE_DIR_NAME_DICT = test_archive(self._orig_archive) 327 328 def tearDown(self): 329 rebaseline_chromium_webkit_tests.ARCHIVE_DIR_NAME_DICT = self._orig_archive 330 331 def test_all_platforms(self): 332 expectations = "BUGX REBASELINE : failures/expected/image.html = IMAGE" 333 334 options = test_options() 335 host_port_obj, filesystem = test_host_port_and_filesystem(options, expectations) 336 url_fetcher = test_url_fetcher(filesystem) 337 zip_factory = test_zip_factory() 338 mock_scm = mocktool.MockSCM() 339 oc = outputcapture.OutputCapture() 340 oc.capture_output() 341 res = rebaseline_chromium_webkit_tests.real_main(options, options, 342 host_port_obj, host_port_obj, url_fetcher, zip_factory, mock_scm) 343 oc.restore_output() 344 345 # We expect to have written 36 files over the course of this rebaseline: 346 # *) 6*3 files in /__im_tmp/ for the archived members of the 6 ports 347 # *) 2*3 files in /test.checkout for actually differing baselines 348 # *) 1 file in /test.checkout for the updated test_expectations file 349 # *) 2*4 files in /tmp for the old/new baselines for the two actual ports 350 # *) 2 files in /tmp for the text diffs for the two ports 351 # *) 1 file in /tmp for the rebaseline results html file 352 self.assertEqual(res, 0) 353 self.assertEqual(len(filesystem.written_files), 36) 354 355 356 class TestHtmlGenerator(unittest.TestCase): 357 def make_generator(self, files, tests): 358 options = mocktool.MockOptions(configuration=None, html_directory='/tmp') 359 host_port = port.get('test', options, filesystem=port.unit_test_filesystem(files)) 360 generator = rebaseline_chromium_webkit_tests.HtmlGenerator(host_port, 361 target_port=None, options=options, platforms=['test-mac-leopard'], rebaselining_tests=tests) 362 return generator, host_port 363 364 def test_generate_baseline_links(self): 365 files = { 366 "/tmp/foo-expected-mac-old.txt": "", 367 "/tmp/foo-expected-mac-new.txt": "", 368 "/tmp/foo-expected-mac-diff.txt": "", 369 } 370 tests = ["foo.txt"] 371 generator, host_port = self.make_generator(files, tests) 372 links = generator._generate_baseline_links("foo", ".txt", "mac") 373 expected_links = '<td align=center><a href="file:///tmp/foo-expected-mac-old.txt">foo-expected.txt</a></td><td align=center><a href="file:///tmp/foo-expected-mac-new.txt">foo-expected.txt</a></td><td align=center><a href="file:///tmp/foo-expected-mac-diff.txt">Diff</a></td>' 374 self.assertEqual(links, expected_links) 375 376 377 if __name__ == '__main__': 378 unittest.main() 379
