1 # Copyright (C) 2010 Google Inc. All rights reserved. 2 # 3 # Redistribution and use in source and binary forms, with or without 4 # modification, are permitted provided that the following conditions are 5 # met: 6 # 7 # * Redistributions of source code must retain the above copyright 8 # notice, this list of conditions and the following disclaimer. 9 # * Redistributions in binary form must reproduce the above 10 # copyright notice, this list of conditions and the following disclaimer 11 # in the documentation and/or other materials provided with the 12 # distribution. 13 # * Neither the name of Google Inc. nor the names of its 14 # contributors may be used to endorse or promote products derived from 15 # this software without specific prior written permission. 16 # 17 # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 18 # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 19 # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 20 # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 21 # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 22 # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 23 # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 24 # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 25 # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 26 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 27 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 28 29 import json 30 import webkitpy.thirdparty.unittest2 as unittest 31 32 from webkitpy.common.net import layouttestresults_unittest 33 from webkitpy.common.host_mock import MockHost 34 from webkitpy.layout_tests.layout_package.json_results_generator import strip_json_wrapper 35 from webkitpy.layout_tests.port.base import Port 36 from webkitpy.tool.commands.rebaselineserver import TestConfig, RebaselineServer 37 from webkitpy.tool.servers import rebaselineserver 38 39 40 class RebaselineTestTest(unittest.TestCase): 41 def test_text_rebaseline_update(self): 42 self._assertRebaseline( 43 test_files=( 44 'fast/text-expected.txt', 45 'platform/mac/fast/text-expected.txt', 46 ), 47 results_files=( 48 'fast/text-actual.txt', 49 ), 50 test_name='fast/text.html', 51 baseline_target='mac', 52 baseline_move_to='none', 53 expected_success=True, 54 expected_log=[ 55 'Rebaselining fast/text...', 56 ' Updating baselines for mac', 57 ' Updated text-expected.txt', 58 ]) 59 60 def test_text_rebaseline_new(self): 61 self._assertRebaseline( 62 test_files=( 63 'fast/text-expected.txt', 64 ), 65 results_files=( 66 'fast/text-actual.txt', 67 ), 68 test_name='fast/text.html', 69 baseline_target='mac', 70 baseline_move_to='none', 71 expected_success=True, 72 expected_log=[ 73 'Rebaselining fast/text...', 74 ' Updating baselines for mac', 75 ' Updated text-expected.txt', 76 ]) 77 78 def test_text_rebaseline_move_no_op_1(self): 79 self._assertRebaseline( 80 test_files=( 81 'fast/text-expected.txt', 82 'platform/win/fast/text-expected.txt', 83 ), 84 results_files=( 85 'fast/text-actual.txt', 86 ), 87 test_name='fast/text.html', 88 baseline_target='mac', 89 baseline_move_to='mac-leopard', 90 expected_success=True, 91 expected_log=[ 92 'Rebaselining fast/text...', 93 ' Updating baselines for mac', 94 ' Updated text-expected.txt', 95 ]) 96 97 def test_text_rebaseline_move_no_op_2(self): 98 self._assertRebaseline( 99 test_files=( 100 'fast/text-expected.txt', 101 'platform/mac/fast/text-expected.checksum', 102 ), 103 results_files=( 104 'fast/text-actual.txt', 105 ), 106 test_name='fast/text.html', 107 baseline_target='mac', 108 baseline_move_to='mac-leopard', 109 expected_success=True, 110 expected_log=[ 111 'Rebaselining fast/text...', 112 ' Moving current mac baselines to mac-leopard', 113 ' No current baselines to move', 114 ' Updating baselines for mac', 115 ' Updated text-expected.txt', 116 ]) 117 118 def test_text_rebaseline_move(self): 119 self._assertRebaseline( 120 test_files=( 121 'fast/text-expected.txt', 122 'platform/mac/fast/text-expected.txt', 123 ), 124 results_files=( 125 'fast/text-actual.txt', 126 ), 127 test_name='fast/text.html', 128 baseline_target='mac', 129 baseline_move_to='mac-leopard', 130 expected_success=True, 131 expected_log=[ 132 'Rebaselining fast/text...', 133 ' Moving current mac baselines to mac-leopard', 134 ' Moved text-expected.txt', 135 ' Updating baselines for mac', 136 ' Updated text-expected.txt', 137 ]) 138 139 def test_text_rebaseline_move_only_images(self): 140 self._assertRebaseline( 141 test_files=( 142 'fast/image-expected.txt', 143 'platform/mac/fast/image-expected.txt', 144 'platform/mac/fast/image-expected.png', 145 'platform/mac/fast/image-expected.checksum', 146 ), 147 results_files=( 148 'fast/image-actual.png', 149 'fast/image-actual.checksum', 150 ), 151 test_name='fast/image.html', 152 baseline_target='mac', 153 baseline_move_to='mac-leopard', 154 expected_success=True, 155 expected_log=[ 156 'Rebaselining fast/image...', 157 ' Moving current mac baselines to mac-leopard', 158 ' Moved image-expected.checksum', 159 ' Moved image-expected.png', 160 ' Updating baselines for mac', 161 ' Updated image-expected.checksum', 162 ' Updated image-expected.png', 163 ]) 164 165 def test_text_rebaseline_move_already_exist(self): 166 self._assertRebaseline( 167 test_files=( 168 'fast/text-expected.txt', 169 'platform/mac-leopard/fast/text-expected.txt', 170 'platform/mac/fast/text-expected.txt', 171 ), 172 results_files=( 173 'fast/text-actual.txt', 174 ), 175 test_name='fast/text.html', 176 baseline_target='mac', 177 baseline_move_to='mac-leopard', 178 expected_success=False, 179 expected_log=[ 180 'Rebaselining fast/text...', 181 ' Moving current mac baselines to mac-leopard', 182 ' Already had baselines in mac-leopard, could not move existing mac ones', 183 ]) 184 185 def test_image_rebaseline(self): 186 self._assertRebaseline( 187 test_files=( 188 'fast/image-expected.txt', 189 'platform/mac/fast/image-expected.png', 190 'platform/mac/fast/image-expected.checksum', 191 ), 192 results_files=( 193 'fast/image-actual.png', 194 'fast/image-actual.checksum', 195 ), 196 test_name='fast/image.html', 197 baseline_target='mac', 198 baseline_move_to='none', 199 expected_success=True, 200 expected_log=[ 201 'Rebaselining fast/image...', 202 ' Updating baselines for mac', 203 ' Updated image-expected.checksum', 204 ' Updated image-expected.png', 205 ]) 206 207 def test_gather_baselines(self): 208 example_json = layouttestresults_unittest.LayoutTestResultsTest.example_full_results_json 209 results_json = json.loads(strip_json_wrapper(example_json)) 210 server = RebaselineServer() 211 server._test_config = get_test_config() 212 server._gather_baselines(results_json) 213 self.assertEqual(results_json['tests']['svg/dynamic-updates/SVGFEDropShadowElement-dom-stdDeviation-attr.html']['state'], 'needs_rebaseline') 214 self.assertNotIn('prototype-chocolate.html', results_json['tests']) 215 216 def _assertRebaseline(self, test_files, results_files, test_name, baseline_target, baseline_move_to, expected_success, expected_log): 217 log = [] 218 test_config = get_test_config(test_files, results_files) 219 success = rebaselineserver._rebaseline_test( 220 test_name, 221 baseline_target, 222 baseline_move_to, 223 test_config, 224 log=lambda l: log.append(l)) 225 self.assertEqual(expected_log, log) 226 self.assertEqual(expected_success, success) 227 228 229 class GetActualResultFilesTest(unittest.TestCase): 230 def test(self): 231 test_config = get_test_config(result_files=( 232 'fast/text-actual.txt', 233 'fast2/text-actual.txt', 234 'fast/text2-actual.txt', 235 'fast/text-notactual.txt', 236 )) 237 self.assertItemsEqual( 238 ('text-actual.txt',), 239 rebaselineserver._get_actual_result_files( 240 'fast/text.html', test_config)) 241 242 243 class GetBaselinesTest(unittest.TestCase): 244 def test_no_baselines(self): 245 self._assertBaselines( 246 test_files=(), 247 test_name='fast/missing.html', 248 expected_baselines={}) 249 250 def test_text_baselines(self): 251 self._assertBaselines( 252 test_files=( 253 'fast/text-expected.txt', 254 'platform/mac/fast/text-expected.txt', 255 ), 256 test_name='fast/text.html', 257 expected_baselines={ 258 'mac': {'.txt': True}, 259 'base': {'.txt': False}, 260 }) 261 262 def test_image_and_text_baselines(self): 263 self._assertBaselines( 264 test_files=( 265 'fast/image-expected.txt', 266 'platform/mac/fast/image-expected.png', 267 'platform/mac/fast/image-expected.checksum', 268 'platform/win/fast/image-expected.png', 269 'platform/win/fast/image-expected.checksum', 270 ), 271 test_name='fast/image.html', 272 expected_baselines={ 273 'base': {'.txt': True}, 274 'mac': {'.checksum': True, '.png': True}, 275 'win': {'.checksum': False, '.png': False}, 276 }) 277 278 def test_extra_baselines(self): 279 self._assertBaselines( 280 test_files=( 281 'fast/text-expected.txt', 282 'platform/nosuchplatform/fast/text-expected.txt', 283 ), 284 test_name='fast/text.html', 285 expected_baselines={'base': {'.txt': True}}) 286 287 def _assertBaselines(self, test_files, test_name, expected_baselines): 288 actual_baselines = rebaselineserver.get_test_baselines(test_name, get_test_config(test_files)) 289 self.assertEqual(expected_baselines, actual_baselines) 290 291 292 def get_test_config(test_files=[], result_files=[]): 293 host = MockHost() 294 port = host.port_factory.get() 295 layout_tests_directory = port.layout_tests_dir() 296 results_directory = port.results_directory() 297 298 for file in test_files: 299 host.filesystem.write_binary_file(host.filesystem.join(layout_tests_directory, file), '') 300 for file in result_files: 301 host.filesystem.write_binary_file(host.filesystem.join(results_directory, file), '') 302 303 class TestMacPort(Port): 304 port_name = "mac" 305 FALLBACK_PATHS = {'': ['mac']} 306 307 return TestConfig( 308 TestMacPort(host, 'mac'), 309 layout_tests_directory, 310 results_directory, 311 ('mac', 'mac-leopard', 'win', 'linux'), 312 host.filesystem, 313 host.scm()) 314
