1 #!/usr/bin/env python 2 # Copyright (c) 2012 Google Inc. All rights reserved. 3 # 4 # Redistribution and use in source and binary forms, with or without 5 # modification, are permitted provided that the following conditions are 6 # met: 7 # 8 # * Redistributions of source code must retain the above copyright 9 # notice, this list of conditions and the following disclaimer. 10 # * Redistributions in binary form must reproduce the above 11 # copyright notice, this list of conditions and the following disclaimer 12 # in the documentation and/or other materials provided with the 13 # distribution. 14 # * Neither the name of Google Inc. nor the names of its 15 # contributors may be used to endorse or promote products derived from 16 # this software without specific prior written permission. 17 # 18 # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 19 # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 20 # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 21 # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 22 # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 23 # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 24 # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 25 # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 26 # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 27 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 28 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 29 30 from modular_build import read_file, write_file 31 import os 32 import os.path as path 33 import generate_injected_script_externs 34 import generate_protocol_externs 35 import modular_build 36 import re 37 import shutil 38 import subprocess 39 import sys 40 import tempfile 41 try: 42 import simplejson as json 43 except ImportError: 44 import json 45 46 scripts_path = path.dirname(path.abspath(__file__)) 47 devtools_path = path.dirname(scripts_path) 48 inspector_path = path.join(path.dirname(devtools_path), 'core', 'inspector') 49 devtools_frontend_path = path.join(devtools_path, 'front_end') 50 global_externs_file = path.join(devtools_frontend_path, 'externs.js') 51 protocol_externs_file = path.join(devtools_frontend_path, 'protocol_externs.js') 52 webgl_rendering_context_idl_path = path.join(path.dirname(devtools_path), 'core', 'html', 'canvas', 'WebGLRenderingContextBase.idl') 53 injected_script_source_name = path.join(inspector_path, 'InjectedScriptSource.js') 54 canvas_injected_script_source_name = path.join(inspector_path, 'InjectedScriptCanvasModuleSource.js') 55 injected_script_externs_idl_names = [ 56 path.join(inspector_path, 'InjectedScriptHost.idl'), 57 path.join(inspector_path, 'JavaScriptCallFrame.idl'), 58 ] 59 closure_compiler_jar = path.join(scripts_path, 'closure', 'compiler.jar') 60 closure_runner_jar = path.join(scripts_path, 'compiler-runner', 'closure-runner.jar') 61 jsdoc_validator_jar = path.join(scripts_path, 'jsdoc-validator', 'jsdoc-validator.jar') 62 java_exec = 'java -Xms1024m -server -XX:+TieredCompilation' 63 64 jsmodule_name_prefix = 'jsmodule_' 65 runtime_module_name = '_runtime' 66 67 type_checked_jsdoc_tags_list = ['param', 'return', 'type', 'enum'] 68 type_checked_jsdoc_tags_or = '|'.join(type_checked_jsdoc_tags_list) 69 70 # Basic regex for invalid JsDoc types: an object type name ([A-Z][A-Za-z0-9.]+[A-Za-z0-9]) not preceded by '!', '?', ':' (this, new), or '.' (object property). 71 invalid_type_regex = re.compile(r'@(?:' + type_checked_jsdoc_tags_or + r')\s*\{.*(?<![!?:.A-Za-z0-9])([A-Z][A-Za-z0-9.]+[A-Za-z0-9])[^/]*\}') 72 invalid_type_designator_regex = re.compile(r'@(?:' + type_checked_jsdoc_tags_or + r')\s*.*(?<![{: ])([?!])=?\}') 73 error_warning_regex = re.compile(r'(?:WARNING|ERROR)') 74 75 errors_found = False 76 77 generate_protocol_externs.generate_protocol_externs(protocol_externs_file, path.join(devtools_path, 'protocol.json')) 78 79 80 def log_error(message): 81 print 'ERROR: ' + message 82 83 84 def error_excepthook(exctype, value, traceback): 85 print 'ERROR:' 86 sys.__excepthook__(exctype, value, traceback) 87 sys.excepthook = error_excepthook 88 89 90 loader = modular_build.DescriptorLoader(devtools_frontend_path) 91 descriptors = loader.load_application('devtools.json') 92 modules_by_name = descriptors.modules 93 94 95 def run_in_shell(command_line): 96 return subprocess.Popen(command_line, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, shell=True) 97 98 99 def hasErrors(output): 100 return re.search(error_warning_regex, output) != None 101 102 103 def verify_jsdoc_extra(additional_files): 104 return run_in_shell('%s -jar %s %s' % (java_exec, jsdoc_validator_jar, ' '.join(descriptors.all_compiled_files() + additional_files))) 105 106 107 def verify_jsdoc(additional_files): 108 def file_list(): 109 return descriptors.all_compiled_files() + additional_files 110 111 errors_found = False 112 for full_file_name in file_list(): 113 lineIndex = 0 114 with open(full_file_name, 'r') as sourceFile: 115 for line in sourceFile: 116 line = line.rstrip() 117 lineIndex += 1 118 if not line: 119 continue 120 if verify_jsdoc_line(full_file_name, lineIndex, line): 121 errors_found = True 122 return errors_found 123 124 125 def verify_jsdoc_line(fileName, lineIndex, line): 126 def print_error(message, errorPosition): 127 print '%s:%s: ERROR - %s\n%s\n%s\n' % (fileName, lineIndex, message, line, ' ' * errorPosition + '^') 128 129 errors_found = False 130 match = re.search(invalid_type_regex, line) 131 if match: 132 print_error('Type "%s" nullability not marked explicitly with "?" (nullable) or "!" (non-nullable)' % match.group(1), match.start(1)) 133 errors_found = True 134 135 match = re.search(invalid_type_designator_regex, line) 136 if (match): 137 print_error('Type nullability indicator misplaced, should precede type', match.start(1)) 138 errors_found = True 139 return errors_found 140 141 142 def check_java_path(): 143 proc = subprocess.Popen('which java', stdout=subprocess.PIPE, shell=True) 144 (javaPath, _) = proc.communicate() 145 146 if proc.returncode != 0: 147 print 'Cannot find java ("which java" return code = %d, should be 0)' % proc.returncode 148 sys.exit(1) 149 print 'Java executable: ' + re.sub(r'\n$', '', javaPath) 150 151 check_java_path() 152 153 modules_dir = tempfile.mkdtemp() 154 common_closure_args = ' --summary_detail_level 3 --jscomp_error visibility --compilation_level SIMPLE_OPTIMIZATIONS --warning_level VERBOSE --language_in ECMASCRIPT5 --accept_const_keyword --module_output_path_prefix %s/' % modules_dir 155 156 spawned_compiler_command = '%s -jar %s %s \\\n' % (java_exec, closure_compiler_jar, common_closure_args) 157 158 worker_modules_by_name = {} 159 dependents_by_module_name = {} 160 161 for module_name in descriptors.application: 162 module = descriptors.modules[module_name] 163 if descriptors.application[module_name].get('type', None) == 'worker': 164 worker_modules_by_name[module_name] = module 165 for dep in module.get('dependencies', []): 166 list = dependents_by_module_name.get(dep) 167 if not list: 168 list = [] 169 dependents_by_module_name[dep] = list 170 list.append(module_name) 171 172 173 def verify_worker_modules(): 174 for name in modules_by_name: 175 for dependency in modules_by_name[name].get('dependencies', []): 176 if dependency in worker_modules_by_name: 177 log_error('Module "%s" may not depend on the worker module "%s"' % (name, dependency)) 178 errors_found = True 179 180 verify_worker_modules() 181 182 183 def check_duplicate_files(): 184 185 def check_module(module, seen_files, seen_modules): 186 name = module['name'] 187 seen_modules[name] = True 188 for dep_name in module.get('dependencies', []): 189 if not dep_name in seen_modules: 190 check_module(modules_by_name[dep_name], seen_files, seen_modules) 191 for source in module.get('scripts', []): 192 referencing_module = seen_files.get(source) 193 if referencing_module: 194 log_error('Duplicate use of %s in "%s" (previously seen in "%s")' % (source, name, referencing_module)) 195 seen_files[source] = name 196 197 for module_name in worker_modules_by_name: 198 check_module(worker_modules_by_name[module_name], {}, {}) 199 200 print 'Checking duplicate files across modules...' 201 check_duplicate_files() 202 203 204 def module_arg(module_name): 205 return ' --module ' + jsmodule_name_prefix + module_name 206 207 208 def dump_module(name, recursively, processed_modules): 209 if name in processed_modules: 210 return '' 211 processed_modules[name] = True 212 module = modules_by_name[name] 213 skipped_scripts = set(module.get('skip_compilation', [])) 214 215 command = '' 216 dependencies = module.get('dependencies', []) 217 if recursively: 218 for dependency in dependencies: 219 command += dump_module(dependency, recursively, processed_modules) 220 command += module_arg(name) + ':' 221 filtered_scripts = descriptors.module_compiled_files(name) 222 command += str(len(filtered_scripts)) 223 firstDependency = True 224 for dependency in dependencies + [runtime_module_name]: 225 if firstDependency: 226 command += ':' 227 else: 228 command += ',' 229 firstDependency = False 230 command += jsmodule_name_prefix + dependency 231 for script in filtered_scripts: 232 command += ' --js ' + path.join(devtools_frontend_path, name, script) 233 return command 234 235 print 'Compiling frontend...' 236 237 compiler_args_file = tempfile.NamedTemporaryFile(mode='wt', delete=False) 238 try: 239 for name in descriptors.sorted_modules(): 240 closure_args = common_closure_args 241 closure_args += ' --externs ' + global_externs_file 242 closure_args += ' --externs ' + protocol_externs_file 243 runtime_module = module_arg(runtime_module_name) + ':1 --js ' + path.join(devtools_frontend_path, 'Runtime.js') 244 closure_args += runtime_module + dump_module(name, True, {}) 245 compiler_args_file.write('%s %s\n' % (name, closure_args)) 246 finally: 247 compiler_args_file.close() 248 249 closure_runner_command = '%s -jar %s --compiler-args-file %s' % (java_exec, closure_runner_jar, compiler_args_file.name) 250 modular_compiler_proc = run_in_shell(closure_runner_command) 251 252 253 def unclosure_injected_script(sourceFileName, outFileName): 254 255 source = read_file(sourceFileName) 256 257 def replace_function(matchobj): 258 return re.sub(r'@param', 'param', matchobj.group(1) or '') + '\n//' + matchobj.group(2) 259 260 # Comment out the closure function and its jsdocs 261 source = re.sub(r'(/\*\*(?:[\s\n]*\*\s*@param[^\n]+\n)+\s*\*/\s*)?\n(\(function)', replace_function, source, count=1) 262 263 # Comment out its return statement 264 source = re.sub(r'\n(\s*return\s+[^;]+;\s*\n\}\)\s*)$', '\n/*\\1*/', source) 265 266 # Replace the "var Object" override with a "self.Object" one 267 source = re.sub(r'\nvar Object =', '\nself.Object =', source, count=1) 268 269 write_file(outFileName, source) 270 271 injectedScriptSourceTmpFile = path.join(inspector_path, 'InjectedScriptSourceTmp.js') 272 injectedScriptCanvasModuleSourceTmpFile = path.join(inspector_path, 'InjectedScriptCanvasModuleSourceTmp.js') 273 274 unclosure_injected_script(injected_script_source_name, injectedScriptSourceTmpFile) 275 unclosure_injected_script(canvas_injected_script_source_name, injectedScriptCanvasModuleSourceTmpFile) 276 277 print 'Compiling InjectedScriptSource.js and InjectedScriptCanvasModuleSource.js...' 278 injected_script_externs_file = tempfile.NamedTemporaryFile(mode='wt', delete=False) 279 try: 280 generate_injected_script_externs.generate_injected_script_externs(injected_script_externs_idl_names, injected_script_externs_file) 281 finally: 282 injected_script_externs_file.close() 283 284 command = spawned_compiler_command 285 command += ' --externs ' + injected_script_externs_file.name + ' \\\n' 286 command += ' --externs ' + protocol_externs_file + ' \\\n' 287 command += ' --module ' + jsmodule_name_prefix + 'injected_script' + ':1' + ' \\\n' 288 command += ' --js ' + injectedScriptSourceTmpFile + ' \\\n' 289 command += ' --module ' + jsmodule_name_prefix + 'injected_canvas_script' + ':1:' + jsmodule_name_prefix + 'injected_script' + ' \\\n' 290 command += ' --js ' + injectedScriptCanvasModuleSourceTmpFile + ' \\\n' 291 command += '\n' 292 293 injectedScriptCompileProc = run_in_shell(command) 294 295 print 'Verifying JSDoc comments...' 296 additional_jsdoc_check_files = [injectedScriptSourceTmpFile, injectedScriptCanvasModuleSourceTmpFile] 297 errors_found |= verify_jsdoc(additional_jsdoc_check_files) 298 jsdocValidatorProc = verify_jsdoc_extra(additional_jsdoc_check_files) 299 300 print 'Checking generated code in InjectedScriptCanvasModuleSource.js...' 301 check_injected_webgl_calls_command = '%s/check_injected_webgl_calls_info.py %s %s' % (scripts_path, webgl_rendering_context_idl_path, canvas_injected_script_source_name) 302 canvasModuleCompileProc = run_in_shell(check_injected_webgl_calls_command) 303 304 print 'Validating InjectedScriptSource.js...' 305 check_injected_script_command = '%s/check_injected_script_source.py %s' % (scripts_path, injected_script_source_name) 306 validateInjectedScriptProc = run_in_shell(check_injected_script_command) 307 308 print 309 310 (jsdocValidatorOut, _) = jsdocValidatorProc.communicate() 311 if jsdocValidatorOut: 312 print ('JSDoc validator output:\n%s' % jsdocValidatorOut) 313 errors_found = True 314 315 (moduleCompileOut, _) = modular_compiler_proc.communicate() 316 print 'Modular compilation output:' 317 318 start_module_regex = re.compile(r'^@@ START_MODULE:(.+) @@$') 319 end_module_regex = re.compile(r'^@@ END_MODULE @@$') 320 321 in_module = False 322 skipped_modules = {} 323 error_count = 0 324 325 def skip_dependents(module_name): 326 for skipped_module in dependents_by_module_name.get(module_name, []): 327 skipped_modules[skipped_module] = True 328 329 # pylint: disable=E1103 330 for line in moduleCompileOut.splitlines(): 331 if not in_module: 332 match = re.search(start_module_regex, line) 333 if not match: 334 continue 335 in_module = True 336 module_error_count = 0 337 module_output = [] 338 module_name = match.group(1) 339 skip_module = skipped_modules.get(module_name) 340 if skip_module: 341 skip_dependents(module_name) 342 else: 343 match = re.search(end_module_regex, line) 344 if not match: 345 if not skip_module: 346 module_output.append(line) 347 if hasErrors(line): 348 error_count += 1 349 module_error_count += 1 350 skip_dependents(module_name) 351 continue 352 353 in_module = False 354 if skip_module: 355 print 'Skipping module %s...' % module_name 356 elif not module_error_count: 357 print 'Module %s compiled successfully: %s' % (module_name, module_output[0]) 358 else: 359 print 'Module %s compile failed: %s errors\n' % (module_name, module_error_count) 360 print os.linesep.join(module_output) 361 362 if error_count: 363 print 'Total Closure errors: %d\n' % error_count 364 errors_found = True 365 366 (injectedScriptCompileOut, _) = injectedScriptCompileProc.communicate() 367 print 'InjectedScriptSource.js and InjectedScriptCanvasModuleSource.js compilation output:\n', injectedScriptCompileOut 368 errors_found |= hasErrors(injectedScriptCompileOut) 369 370 (canvasModuleCompileOut, _) = canvasModuleCompileProc.communicate() 371 print 'InjectedScriptCanvasModuleSource.js generated code check output:\n', canvasModuleCompileOut 372 errors_found |= hasErrors(canvasModuleCompileOut) 373 374 (validateInjectedScriptOut, _) = validateInjectedScriptProc.communicate() 375 print 'Validate InjectedScriptSource.js output:\n', (validateInjectedScriptOut if validateInjectedScriptOut else '<empty>') 376 errors_found |= hasErrors(validateInjectedScriptOut) 377 378 if errors_found: 379 print 'ERRORS DETECTED' 380 381 os.remove(injectedScriptSourceTmpFile) 382 os.remove(injectedScriptCanvasModuleSourceTmpFile) 383 os.remove(compiler_args_file.name) 384 os.remove(injected_script_externs_file.name) 385 os.remove(protocol_externs_file) 386 shutil.rmtree(modules_dir, True) 387