1 # Copyright (C) 2013 Google Inc. All rights reserved. 2 # 3 # Redistribution and use in source and binary forms, with or without 4 # modification, are permitted provided that the following conditions are 5 # met: 6 # 7 # * Redistributions of source code must retain the above copyright 8 # notice, this list of conditions and the following disclaimer. 9 # * Redistributions in binary form must reproduce the above 10 # copyright notice, this list of conditions and the following disclaimer 11 # in the documentation and/or other materials provided with the 12 # distribution. 13 # * Neither the name of Google Inc. nor the names of its 14 # contributors may be used to endorse or promote products derived from 15 # this software without specific prior written permission. 16 # 17 # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 18 # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 19 # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 20 # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 21 # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 22 # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 23 # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 24 # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 25 # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 26 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 27 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 28 29 """Moves a directory of LayoutTests. 30 31 Given a path to a directory of LayoutTests, moves that directory, including all recursive children, 32 to the specified destination path. Updates all references in tests and resources to reflect the new 33 location. Also moves any corresponding platform-specific expected results and updates the test 34 expectations to reflect the move. 35 36 If the destination directory does not exist, it and any missing parent directories are created. If 37 the destination directory already exists, the child members of the origin directory are added to the 38 destination directory. If any of the child members clash with existing members of the destination 39 directory, the move fails. 40 41 Note that when new entries are added to the test expectations, no attempt is made to group or merge 42 them with existing entries. This should be be done manually and with lint-test-expectations. 43 """ 44 45 import copy 46 import logging 47 import optparse 48 import os 49 import re 50 import urlparse 51 52 from webkitpy.common.checkout.scm.detection import SCMDetector 53 from webkitpy.common.host import Host 54 from webkitpy.common.system.executive import Executive 55 from webkitpy.common.system.filesystem import FileSystem 56 from webkitpy.layout_tests.port.base import Port 57 from webkitpy.layout_tests.models.test_expectations import TestExpectations 58 59 60 logging.basicConfig() 61 _log = logging.getLogger(__name__) 62 _log.setLevel(logging.INFO) 63 64 PLATFORM_DIRECTORY = 'platform' 65 66 class LayoutTestsMover(object): 67 68 def __init__(self, port=None): 69 self._port = port 70 if not self._port: 71 host = Host() 72 # Given that we use include_overrides=False and model_all_expectations=True when 73 # constructing the TestExpectations object, it doesn't matter which Port object we use. 74 self._port = host.port_factory.get() 75 self._port.host.initialize_scm() 76 self._filesystem = self._port.host.filesystem 77 self._scm = self._port.host.scm() 78 self._layout_tests_root = self._port.layout_tests_dir() 79 80 def _scm_path(self, *paths): 81 return self._filesystem.join('LayoutTests', *paths) 82 83 def _is_child_path(self, parent, possible_child): 84 normalized_parent = self._filesystem.normpath(parent) 85 normalized_child = self._filesystem.normpath(possible_child) 86 # We need to add a trailing separator to parent to avoid returning true for cases like 87 # parent='/foo/b', and possible_child='/foo/bar/baz'. 88 return normalized_parent == normalized_child or normalized_child.startswith(normalized_parent + self._filesystem.sep) 89 90 def _move_path(self, path, origin, destination): 91 if not self._is_child_path(origin, path): 92 return path 93 return self._filesystem.normpath(self._filesystem.join(destination, self._filesystem.relpath(path, origin))) 94 95 def _validate_input(self): 96 if not self._filesystem.isdir(self._absolute_origin): 97 raise Exception('Source path %s is not a directory' % self._origin) 98 if not self._is_child_path(self._layout_tests_root, self._absolute_origin): 99 raise Exception('Source path %s is not in LayoutTests directory' % self._origin) 100 if self._filesystem.isfile(self._absolute_destination): 101 raise Exception('Destination path %s is a file' % self._destination) 102 if not self._is_child_path(self._layout_tests_root, self._absolute_destination): 103 raise Exception('Destination path %s is not in LayoutTests directory' % self._destination) 104 105 # If destination is an existing directory, we move the children of origin into destination. 106 # However, if any of the children of origin would clash with existing children of 107 # destination, we fail. 108 # FIXME: Consider adding support for recursively moving into an existing directory. 109 if self._filesystem.isdir(self._absolute_destination): 110 for file_path in self._filesystem.listdir(self._absolute_origin): 111 if self._filesystem.exists(self._filesystem.join(self._absolute_destination, file_path)): 112 raise Exception('Origin path %s clashes with existing destination path %s' % 113 (self._filesystem.join(self._origin, file_path), self._filesystem.join(self._destination, file_path))) 114 115 def _get_expectations_for_test(self, model, test_path): 116 """Given a TestExpectationsModel object, finds all expectations that match the specified 117 test, specified as a relative path. Handles the fact that expectations may be keyed by 118 directory. 119 """ 120 expectations = set() 121 if model.has_test(test_path): 122 expectations.add(model.get_expectation_line(test_path)) 123 test_path = self._filesystem.dirname(test_path) 124 while not test_path == '': 125 # The model requires a trailing slash for directories. 126 test_path_for_model = test_path + '/' 127 if model.has_test(test_path_for_model): 128 expectations.add(model.get_expectation_line(test_path_for_model)) 129 test_path = self._filesystem.dirname(test_path) 130 return expectations 131 132 def _get_expectations(self, model, path): 133 """Given a TestExpectationsModel object, finds all expectations for all tests under the 134 specified relative path. 135 """ 136 expectations = set() 137 for test in self._filesystem.files_under(self._filesystem.join(self._layout_tests_root, path), dirs_to_skip=['script-tests', 'resources'], 138 file_filter=Port.is_test_file): 139 expectations = expectations.union(self._get_expectations_for_test(model, self._filesystem.relpath(test, self._layout_tests_root))) 140 return expectations 141 142 @staticmethod 143 def _clone_expectation_line_for_path(expectation_line, path): 144 """Clones a TestExpectationLine object and updates the clone to apply to the specified 145 relative path. 146 """ 147 clone = copy.copy(expectation_line) 148 clone.original_string = re.compile(expectation_line.name).sub(path, expectation_line.original_string) 149 clone.name = path 150 clone.path = path 151 # FIXME: Should we search existing expectations for matches, like in 152 # TestExpectationsParser._collect_matching_tests()? 153 clone.matching_tests = [path] 154 return clone 155 156 def _update_expectations(self): 157 """Updates all test expectations that are affected by the move. 158 """ 159 _log.info('Updating expectations') 160 test_expectations = TestExpectations(self._port, include_overrides=False, model_all_expectations=True) 161 162 for expectation in self._get_expectations(test_expectations.model(), self._origin): 163 path = expectation.path 164 if self._is_child_path(self._origin, path): 165 # If the existing expectation is a child of the moved path, we simply replace it 166 # with an expectation for the updated path. 167 new_path = self._move_path(path, self._origin, self._destination) 168 _log.debug('Updating expectation for %s to %s' % (path, new_path)) 169 test_expectations.remove_expectation_line(path) 170 test_expectations.add_expectation_line(LayoutTestsMover._clone_expectation_line_for_path(expectation, new_path)) 171 else: 172 # If the existing expectation is not a child of the moved path, we have to leave it 173 # in place. But we also add a new expectation for the destination path. 174 new_path = self._destination 175 _log.warning('Copying expectation for %s to %s. You should check that these expectations are still correct.' % 176 (path, new_path)) 177 test_expectations.add_expectation_line(LayoutTestsMover._clone_expectation_line_for_path(expectation, new_path)) 178 179 expectations_file = self._port.path_to_generic_test_expectations_file() 180 self._filesystem.write_text_file(expectations_file, 181 TestExpectations.list_to_string(test_expectations._expectations, reconstitute_only_these=[])) 182 self._scm.add(self._filesystem.relpath(expectations_file, self._scm.checkout_root)) 183 184 def _find_references(self, input_files): 185 """Attempts to find all references to other files in the supplied list of files. Returns a 186 dictionary that maps from an absolute file path to an array of reference strings. 187 """ 188 reference_regex = re.compile(r'(?:(?:src=|href=|importScripts\(|url\()(?:"([^"]+)"|\'([^\']+)\')|url\(([^\)\'"]+)\))') 189 references = {} 190 for input_file in input_files: 191 matches = reference_regex.findall(self._filesystem.read_binary_file(input_file)) 192 if matches: 193 references[input_file] = [filter(None, match)[0] for match in matches] 194 return references 195 196 def _get_updated_reference(self, root, reference): 197 """For a reference <reference> in a directory <root>, determines the updated reference. 198 Returns the the updated reference, or None if no update is required. 199 """ 200 # If the reference is an absolute path or url, it's safe. 201 if reference.startswith('/') or urlparse.urlparse(reference).scheme: 202 return None 203 204 # Both the root path and the target of the reference my be subject to the move, so there are 205 # four cases to consider. In the case where both or neither are subject to the move, the 206 # reference doesn't need updating. 207 # 208 # This is true even if the reference includes superfluous dot segments which mention a moved 209 # directory, as dot segments are collapsed during URL normalization. For example, if 210 # foo.html contains a reference 'bar/../script.js', this remains valid (though ugly) even if 211 # bar/ is moved to baz/, because the reference is always normalized to 'script.js'. 212 absolute_reference = self._filesystem.normpath(self._filesystem.join(root, reference)) 213 if self._is_child_path(self._absolute_origin, root) == self._is_child_path(self._absolute_origin, absolute_reference): 214 return None; 215 216 new_root = self._move_path(root, self._absolute_origin, self._absolute_destination) 217 new_absolute_reference = self._move_path(absolute_reference, self._absolute_origin, self._absolute_destination) 218 return self._filesystem.relpath(new_absolute_reference, new_root) 219 220 def _get_all_updated_references(self, references): 221 """Determines the updated references due to the move. Returns a dictionary that maps from an 222 absolute file path to a dictionary that maps from a reference string to the corresponding 223 updated reference. 224 """ 225 updates = {} 226 for file_path in references.keys(): 227 root = self._filesystem.dirname(file_path) 228 # sript-tests/TEMPLATE.html files contain references which are written as if the file 229 # were in the parent directory. This special-casing is ugly, but there are plans to 230 # remove script-tests. 231 if root.endswith('script-tests') and file_path.endswith('TEMPLATE.html'): 232 root = self._filesystem.dirname(root) 233 local_updates = {} 234 for reference in references[file_path]: 235 update = self._get_updated_reference(root, reference) 236 if update: 237 local_updates[reference] = update 238 if local_updates: 239 updates[file_path] = local_updates 240 return updates 241 242 def _update_file(self, path, updates): 243 contents = self._filesystem.read_binary_file(path) 244 # Note that this regex isn't quite as strict as that used to find the references, but this 245 # avoids the need for alternative match groups, which simplifies things. 246 for target in updates.keys(): 247 regex = re.compile(r'((?:src=|href=|importScripts\(|url\()["\']?)%s(["\']?)' % target) 248 contents = regex.sub(r'\1%s\2' % updates[target], contents) 249 self._filesystem.write_binary_file(path, contents) 250 self._scm.add(path) 251 252 def _update_test_source_files(self): 253 def is_test_source_file(filesystem, dirname, basename): 254 pass_regex = re.compile(r'\.(css|js)$') 255 fail_regex = re.compile(r'-expected\.') 256 return (Port.is_test_file(filesystem, dirname, basename) or pass_regex.search(basename)) and not fail_regex.search(basename) 257 258 test_source_files = self._filesystem.files_under(self._layout_tests_root, file_filter=is_test_source_file) 259 _log.info('Considering %s test source files for references' % len(test_source_files)) 260 references = self._find_references(test_source_files) 261 _log.info('Considering references in %s files' % len(references)) 262 updates = self._get_all_updated_references(references) 263 _log.info('Updating references in %s files' % len(updates)) 264 count = 0 265 for file_path in updates.keys(): 266 self._update_file(file_path, updates[file_path]) 267 count += 1 268 if count % 1000 == 0 or count == len(updates): 269 _log.debug('Updated references in %s files' % count) 270 271 def _move_directory(self, origin, destination): 272 """Moves the directory <origin> to <destination>. If <destination> is a directory, moves the 273 children of <origin> into <destination>. Uses relative paths. 274 """ 275 absolute_origin = self._filesystem.join(self._layout_tests_root, origin) 276 if not self._filesystem.isdir(absolute_origin): 277 return 278 _log.info('Moving directory %s to %s' % (origin, destination)) 279 # Note that FileSystem.move() may silently overwrite existing files, but we 280 # check for this in _validate_input(). 281 absolute_destination = self._filesystem.join(self._layout_tests_root, destination) 282 self._filesystem.maybe_make_directory(absolute_destination) 283 for directory in self._filesystem.listdir(absolute_origin): 284 self._scm.move(self._scm_path(origin, directory), self._scm_path(destination, directory)) 285 self._filesystem.rmtree(absolute_origin) 286 287 def _move_files(self): 288 """Moves the all files that correspond to the move, including platform-specific expected 289 results. 290 """ 291 self._move_directory(self._origin, self._destination) 292 for directory in self._filesystem.listdir(self._filesystem.join(self._layout_tests_root, PLATFORM_DIRECTORY)): 293 self._move_directory(self._filesystem.join(PLATFORM_DIRECTORY, directory, self._origin), 294 self._filesystem.join(PLATFORM_DIRECTORY, directory, self._destination)) 295 296 def _commit_changes(self): 297 if not self._scm.supports_local_commits(): 298 return 299 title = 'Move LayoutTests directory %s to %s' % (self._origin, self._destination) 300 _log.info('Committing change \'%s\'' % title) 301 self._scm.commit_locally_with_message('%s\n\nThis commit was automatically generated by move-layout-tests.' % title, 302 commit_all_working_directory_changes=False) 303 304 def move(self, origin, destination): 305 self._origin = origin 306 self._destination = destination 307 self._absolute_origin = self._filesystem.join(self._layout_tests_root, self._origin) 308 self._absolute_destination = self._filesystem.join(self._layout_tests_root, self._destination) 309 self._validate_input() 310 self._update_expectations() 311 self._update_test_source_files() 312 self._move_files() 313 # FIXME: Handle virtual test suites. 314 self._commit_changes() 315 316 def main(argv): 317 parser = optparse.OptionParser(description=__doc__) 318 parser.add_option('--origin', 319 help=('The directory of tests to move, as a relative path from the LayoutTests directory.')) 320 parser.add_option('--destination', 321 help=('The new path for the directory of tests, as a relative path from the LayoutTests directory.')) 322 options, _ = parser.parse_args() 323 LayoutTestsMover().move(options.origin, options.destination) 324