1 # Copyright (c) 2012 The Chromium Authors. All rights reserved. 2 # Use of this source code is governed by a BSD-style license that can be 3 # found in the LICENSE file. 4 import sys 5 import os 6 import re 7 8 class DepsException(Exception): 9 pass 10 11 """ 12 The core of this script is the calc_load_sequence function. In total, this 13 walks over the provided javascript files and figures out their dependencies 14 using the module definitions provided in each file. This allows us to, for 15 example, have a trio of modules: 16 17 foo.js: 18 base.require('bar'); 19 and bar.js: 20 base.require('baz'); 21 22 calc_load_sequence(['foo'], '.') will yield: 23 [Module('baz'), Module('bar'), Module('foo')] 24 25 which is, based on the dependencies, the correct sequence in which to load 26 those modules. 27 """ 28 29 class ResourceFinder(object): 30 """Helper code for finding a module given a name and current module. 31 32 The dependency resolution code in Module.resolve will find bits of code in the 33 actual javascript that says things require('bar'). This 34 code is responsible for figuring out what filename corresponds to 'bar' given 35 a Module('foo'). 36 """ 37 def __init__(self, root_dir): 38 self._root_dir = root_dir 39 pass 40 41 @property 42 def root_dir(self): 43 return self._root_dir 44 45 def _find_and_load_filename(self, absolute_path): 46 if not os.path.exists(absolute_path): 47 return None, None 48 49 f = open(absolute_path, 'r') 50 contents = f.read() 51 f.close() 52 53 return absolute_path, contents 54 55 def _find_and_load(self, current_module, requested_name, extension): 56 assert current_module.filename 57 pathy_name = requested_name.replace(".", os.sep) 58 filename = pathy_name + extension 59 absolute_path = os.path.join(self._root_dir, filename) 60 return self._find_and_load_filename(absolute_path) 61 62 def find_and_load_module(self, current_module, requested_module_name): 63 return self._find_and_load(current_module, requested_module_name, ".js") 64 65 def find_and_load_raw_script(self, current_module, filename): 66 absolute_path = os.path.join(self._root_dir, filename) 67 return self._find_and_load_filename(absolute_path) 68 69 def find_and_load_style_sheet(self, 70 current_module, requested_style_sheet_name): 71 return self._find_and_load( 72 current_module, requested_style_sheet_name, ".css") 73 74 75 class StyleSheet(object): 76 """Represents a stylesheet resource referenced by a module via the 77 base.requireStylesheet(xxx) directive.""" 78 def __init__(self, name, filename, contents): 79 self.name = name 80 self.filename = filename 81 self.contents = contents 82 83 def __repr__(self): 84 return "StyleSheet(%s)" % self.name 85 86 class RawScript(object): 87 """Represents a raw script resource referenced by a module via the 88 base.requireRawScript(xxx) directive.""" 89 def __init__(self, name, filename, contents): 90 self.name = name 91 self.filename = filename 92 self.contents = contents 93 94 def __repr__(self): 95 return "RawScript(%s)" % self.name 96 97 def _tokenize_js(text): 98 rest = text 99 tokens = ["//", "/*", "*/", "\n"] 100 while len(rest): 101 indices = [rest.find(token) for token in tokens] 102 found_indices = [index for index in indices if index >= 0] 103 104 if len(found_indices) == 0: 105 # end of string 106 yield rest 107 return 108 109 min_index = min(found_indices) 110 token_with_min = tokens[indices.index(min_index)] 111 112 if min_index > 0: 113 yield rest[:min_index] 114 115 yield rest[min_index:min_index + len(token_with_min)] 116 rest = rest[min_index + len(token_with_min):] 117 118 def _strip_js_comments(text): 119 result_tokens = [] 120 token_stream = _tokenize_js(text).__iter__() 121 while True: 122 try: 123 t = token_stream.next() 124 except StopIteration: 125 break 126 127 if t == "//": 128 while True: 129 try: 130 t2 = token_stream.next() 131 if t2 == "\n": 132 break 133 except StopIteration: 134 break 135 elif t == '/*': 136 nesting = 1 137 while True: 138 try: 139 t2 = token_stream.next() 140 if t2 == "/*": 141 nesting += 1 142 elif t2 == "*/": 143 nesting -= 1 144 if nesting == 0: 145 break 146 except StopIteration: 147 break 148 else: 149 result_tokens.append(t) 150 return "".join(result_tokens) 151 152 def _MangleRawScriptFilenameToModuleName(filename): 153 name = filename 154 name = name.replace(os.sep, ':') 155 name = name.replace('..', '!!') 156 return name 157 158 class Module(object): 159 """Represents a javascript module. It can either be directly requested, e.g. 160 passed in by name to calc_load_sequence, or created by being referenced a 161 module via the base.require(xxx) directive. 162 163 Interesting properties on this object are: 164 165 - filename: the file of the actual module 166 - contents: the actual text contents of the module 167 - style_sheets: StyleSheet objects that this module relies on for styling 168 information. 169 - dependent_modules: other modules that this module needs in order to run 170 """ 171 def __init__(self, name = None): 172 self.name = name 173 self.filename = None 174 self.contents = None 175 176 self.dependent_module_names = [] 177 self.dependent_modules = [] 178 self.dependent_raw_script_names = [] 179 self.dependent_raw_scripts = [] 180 self.style_sheet_names = [] 181 self.style_sheets = [] 182 183 def __repr__(self): 184 return "Module(%s)" % self.name 185 186 def load_and_parse(self, module_filename, 187 module_contents = None, 188 decl_required = True): 189 if not module_contents: 190 f = open(module_filename, 'r') 191 self.contents = f.read() 192 f.close() 193 else: 194 self.contents = module_contents 195 self.filename = module_filename 196 self.parse_definition_(self.contents, decl_required) 197 198 def resolve(self, all_resources, resource_finder): 199 if "scripts" not in all_resources: 200 all_resources["scripts"] = {} 201 if "style_sheets" not in all_resources: 202 all_resources["style_sheets"] = {} 203 if "raw_scripts" not in all_resources: 204 all_resources["raw_scripts"] = {} 205 206 assert self.filename 207 208 for name in self.dependent_module_names: 209 if name in all_resources["scripts"]: 210 assert all_resources["scripts"][name].contents 211 self.dependent_modules.append(all_resources["scripts"][name]) 212 continue 213 214 filename, contents = resource_finder.find_and_load_module(self, name) 215 if not filename: 216 raise DepsException("Could not find a file for module %s" % name) 217 218 module = Module(name) 219 all_resources["scripts"][name] = module 220 self.dependent_modules.append(module) 221 module.load_and_parse(filename, contents) 222 module.resolve(all_resources, resource_finder) 223 224 for name in self.dependent_raw_script_names: 225 filename, contents = resource_finder.find_and_load_raw_script(self, name) 226 if not filename: 227 raise DepsException("Could not find a file for module %s" % name) 228 229 if name in all_resources["raw_scripts"]: 230 assert all_resources["raw_scripts"][name].contents 231 self.dependent_raw_scripts.append(all_resources["raw_scripts"][name]) 232 continue 233 234 raw_script = RawScript(name, filename, contents) 235 all_resources["raw_scripts"][name] = raw_script 236 self.dependent_raw_scripts.append(raw_script) 237 238 for name in self.style_sheet_names: 239 if name in all_resources["style_sheets"]: 240 assert all_resources["style_sheets"][name].contents 241 self.style_sheets.append(all_resources["scripts"][name]) 242 continue 243 244 filename, contents = resource_finder.find_and_load_style_sheet(self, name) 245 if not filename: 246 raise DepsException("Could not find a file for stylesheet %s" % name) 247 248 style_sheet = StyleSheet(name, filename, contents) 249 all_resources["style_sheets"][name] = style_sheet 250 self.style_sheets.append(style_sheet) 251 252 def compute_load_sequence_recursive(self, load_sequence, already_loaded_set): 253 for dependent_module in self.dependent_modules: 254 dependent_module.compute_load_sequence_recursive(load_sequence, 255 already_loaded_set) 256 if self.name not in already_loaded_set: 257 already_loaded_set.add(self.name) 258 load_sequence.append(self) 259 260 def parse_definition_(self, text, decl_required = True): 261 if not decl_required and not self.name: 262 raise Exception("Module.name must be set for decl_required to be false.") 263 264 stripped_text = _strip_js_comments(text) 265 rest = stripped_text 266 while True: 267 # Things to search for. 268 m_r = re.search("""base\s*\.\s*require\((["'])(.+?)\\1\)""", 269 rest, re.DOTALL) 270 m_s = re.search("""base\s*\.\s*requireStylesheet\((["'])(.+?)\\1\)""", 271 rest, re.DOTALL) 272 m_irs = re.search("""base\s*\.\s*requireRawScript\((["'])(.+?)\\1\)""", 273 rest, re.DOTALL) 274 matches = [m for m in [m_r, m_s, m_irs] if m] 275 276 # Figure out which was first. 277 matches.sort(key=lambda x: x.start()) 278 if len(matches): 279 m = matches[0] 280 else: 281 break 282 283 if m == m_r: 284 dependent_module_name = m.group(2) 285 if '/' in dependent_module_name: 286 raise DepsException("Slashes are not allowed in module names. " 287 "Use '.' instead: %s" % dependent_module_name) 288 if dependent_module_name.endswith('js'): 289 raise DepsException("module names shouldn't end with .js" 290 "The module system will append that for you: %s" % 291 dependent_module_name) 292 self.dependent_module_names.append(dependent_module_name) 293 elif m == m_s: 294 style_sheet_name = m.group(2) 295 if '/' in style_sheet_name: 296 raise DepsException("Slashes are not allowed in style sheet names. " 297 "Use '.' instead: %s" % style_sheet_name) 298 if style_sheet_name.endswith('.css'): 299 raise DepsException("Style sheets should not end in .css. " 300 "The module system will append that for you" % 301 style_sheet_name) 302 self.style_sheet_names.append(style_sheet_name) 303 elif m == m_irs: 304 name = m.group(2) 305 self.dependent_raw_script_names.append(name) 306 307 rest = rest[m.end():] 308 309 310 def calc_load_sequence(filenames, toplevel_dir): 311 """Given a list of starting javascript files, figure out all the Module 312 objects that need to be loaded to satisfiy their dependencies. 313 314 The javascript files shoud specify their dependencies in a format that is 315 textually equivalent to base.js' require syntax, namely: 316 317 base.require(module1); 318 base.require(module2); 319 base.requireStylesheet(stylesheet); 320 321 The output of this function is an array of Module objects ordered by 322 dependency. 323 """ 324 all_resources = {} 325 all_resources["scripts"] = {} 326 toplevel_modules = [] 327 root_dir = '' 328 if filenames: 329 root_dir = os.path.abspath(os.path.dirname(filenames[0])) 330 resource_finder = ResourceFinder(root_dir) 331 for filename in filenames: 332 if not os.path.exists(filename): 333 raise Exception("Could not find %s" % filename) 334 335 rel_filename = os.path.relpath(filename, toplevel_dir) 336 dirname = os.path.dirname(rel_filename) 337 modname = os.path.splitext(os.path.basename(rel_filename))[0] 338 if len(dirname): 339 name = dirname.replace('/', '.') + '.' + modname 340 else: 341 name = modname 342 343 if name in all_resources["scripts"]: 344 continue 345 346 module = Module(name) 347 module.load_and_parse(filename, decl_required = False) 348 all_resources["scripts"][module.name] = module 349 module.resolve(all_resources, resource_finder) 350 351 # Find the root modules: ones who have no dependencies. 352 module_ref_counts = {} 353 for module in all_resources["scripts"].values(): 354 module_ref_counts[module.name] = 0 355 356 def inc_ref_count(name): 357 module_ref_counts[name] = module_ref_counts[name] + 1 358 for module in all_resources["scripts"].values(): 359 for dependent_module in module.dependent_modules: 360 inc_ref_count(dependent_module.name) 361 362 root_modules = [all_resources["scripts"][name] 363 for name, ref_count in module_ref_counts.items() 364 if ref_count == 0] 365 366 root_modules.sort(lambda x, y: cmp(x.name, y.name)) 367 368 already_loaded_set = set() 369 load_sequence = [] 370 for module in root_modules: 371 module.compute_load_sequence_recursive(load_sequence, already_loaded_set) 372 return load_sequence 373
