Home | History | Annotate | Download | only in scripts 
      1 #!/usr/bin/env python
      2 # Copyright (c) 2012 Google Inc. All rights reserved.
      3 #
      4 # Redistribution and use in source and binary forms, with or without
      5 # modification, are permitted provided that the following conditions are
      6 # met:
      7 #
      8 #     * Redistributions of source code must retain the above copyright
      9 # notice, this list of conditions and the following disclaimer.
     10 #     * Redistributions in binary form must reproduce the above
     11 # copyright notice, this list of conditions and the following disclaimer
     12 # in the documentation and/or other materials provided with the
     13 # distribution.
     14 #     * Neither the name of Google Inc. nor the names of its
     15 # contributors may be used to endorse or promote products derived from
     16 # this software without specific prior written permission.
     17 #
     18 # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
     19 # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
     20 # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
     21 # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
     22 # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
     23 # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
     24 # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
     25 # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
     26 # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
     27 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
     28 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
     29 
     30 from modular_build import read_file, write_file
     31 import os
     32 import os.path as path
     33 import generate_injected_script_externs
     34 import generate_protocol_externs
     35 import modular_build
     36 import re
     37 import shutil
     38 import subprocess
     39 import sys
     40 import tempfile
     41 try:
     42     import simplejson as json
     43 except ImportError:
     44     import json
     45 
     46 scripts_path = path.dirname(path.abspath(__file__))
     47 devtools_path = path.dirname(scripts_path)
     48 inspector_path = path.join(path.dirname(devtools_path), 'core', 'inspector')
     49 devtools_frontend_path = path.join(devtools_path, 'front_end')
     50 global_externs_file = path.join(devtools_frontend_path, 'externs.js')
     51 protocol_externs_file = path.join(devtools_frontend_path, 'protocol_externs.js')
     52 webgl_rendering_context_idl_path = path.join(path.dirname(devtools_path), 'core', 'html', 'canvas', 'WebGLRenderingContextBase.idl')
     53 injected_script_source_name = path.join(inspector_path, 'InjectedScriptSource.js')
     54 canvas_injected_script_source_name = path.join(inspector_path, 'InjectedScriptCanvasModuleSource.js')
     55 injected_script_externs_idl_names = [
     56     path.join(inspector_path, 'InjectedScriptHost.idl'),
     57     path.join(inspector_path, 'JavaScriptCallFrame.idl'),
     58 ]
     59 closure_compiler_jar = path.join(scripts_path, 'closure', 'compiler.jar')
     60 closure_runner_jar = path.join(scripts_path, 'compiler-runner', 'closure-runner.jar')
     61 jsdoc_validator_jar = path.join(scripts_path, 'jsdoc-validator', 'jsdoc-validator.jar')
     62 java_exec = 'java -Xms1024m -server -XX:+TieredCompilation'
     63 
     64 jsmodule_name_prefix = 'jsmodule_'
     65 runtime_module_name = '_runtime'
     66 
     67 type_checked_jsdoc_tags_list = ['param', 'return', 'type', 'enum']
     68 type_checked_jsdoc_tags_or = '|'.join(type_checked_jsdoc_tags_list)
     69 
     70 # Basic regex for invalid JsDoc types: an object type name ([A-Z][A-Za-z0-9.]+[A-Za-z0-9]) not preceded by '!', '?', ':' (this, new), or '.' (object property).
     71 invalid_type_regex = re.compile(r'@(?:' + type_checked_jsdoc_tags_or + r')\s*\{.*(?<![!?:.A-Za-z0-9])([A-Z][A-Za-z0-9.]+[A-Za-z0-9])[^/]*\}')
     72 invalid_type_designator_regex = re.compile(r'@(?:' + type_checked_jsdoc_tags_or + r')\s*.*(?<![{: ])([?!])=?\}')
     73 error_warning_regex = re.compile(r'(?:WARNING|ERROR)')
     74 
     75 errors_found = False
     76 
     77 generate_protocol_externs.generate_protocol_externs(protocol_externs_file, path.join(devtools_path, 'protocol.json'))
     78 
     79 
     80 def log_error(message):
     81     print 'ERROR: ' + message
     82 
     83 
     84 def error_excepthook(exctype, value, traceback):
     85     print 'ERROR:'
     86     sys.__excepthook__(exctype, value, traceback)
     87 sys.excepthook = error_excepthook
     88 
     89 
     90 loader = modular_build.DescriptorLoader(devtools_frontend_path)
     91 descriptors = loader.load_application('devtools.json')
     92 modules_by_name = descriptors.modules
     93 
     94 
     95 def run_in_shell(command_line):
     96     return subprocess.Popen(command_line, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, shell=True)
     97 
     98 
     99 def hasErrors(output):
    100     return re.search(error_warning_regex, output) != None
    101 
    102 
    103 def verify_jsdoc_extra(additional_files):
    104     return run_in_shell('%s -jar %s %s' % (java_exec, jsdoc_validator_jar, ' '.join(descriptors.all_compiled_files() + additional_files)))
    105 
    106 
    107 def verify_jsdoc(additional_files):
    108     def file_list():
    109         return descriptors.all_compiled_files() + additional_files
    110 
    111     errors_found = False
    112     for full_file_name in file_list():
    113         lineIndex = 0
    114         with open(full_file_name, 'r') as sourceFile:
    115             for line in sourceFile:
    116                 line = line.rstrip()
    117                 lineIndex += 1
    118                 if not line:
    119                     continue
    120                 if verify_jsdoc_line(full_file_name, lineIndex, line):
    121                     errors_found = True
    122     return errors_found
    123 
    124 
    125 def verify_jsdoc_line(fileName, lineIndex, line):
    126     def print_error(message, errorPosition):
    127         print '%s:%s: ERROR - %s\n%s\n%s\n' % (fileName, lineIndex, message, line, ' ' * errorPosition + '^')
    128 
    129     errors_found = False
    130     match = re.search(invalid_type_regex, line)
    131     if match:
    132         print_error('Type "%s" nullability not marked explicitly with "?" (nullable) or "!" (non-nullable)' % match.group(1), match.start(1))
    133         errors_found = True
    134 
    135     match = re.search(invalid_type_designator_regex, line)
    136     if (match):
    137         print_error('Type nullability indicator misplaced, should precede type', match.start(1))
    138         errors_found = True
    139     return errors_found
    140 
    141 
    142 def check_java_path():
    143     proc = subprocess.Popen('which java', stdout=subprocess.PIPE, shell=True)
    144     (javaPath, _) = proc.communicate()
    145 
    146     if proc.returncode != 0:
    147         print 'Cannot find java ("which java" return code = %d, should be 0)' % proc.returncode
    148         sys.exit(1)
    149     print 'Java executable: ' + re.sub(r'\n$', '', javaPath)
    150 
    151 check_java_path()
    152 
    153 modules_dir = tempfile.mkdtemp()
    154 common_closure_args = ' --summary_detail_level 3 --jscomp_error visibility --compilation_level SIMPLE_OPTIMIZATIONS --warning_level VERBOSE --language_in ECMASCRIPT5 --accept_const_keyword --module_output_path_prefix %s/' % modules_dir
    155 
    156 spawned_compiler_command = '%s -jar %s %s \\\n' % (java_exec, closure_compiler_jar, common_closure_args)
    157 
    158 worker_modules_by_name = {}
    159 dependents_by_module_name = {}
    160 
    161 for module_name in descriptors.application:
    162     module = descriptors.modules[module_name]
    163     if descriptors.application[module_name].get('type', None) == 'worker':
    164         worker_modules_by_name[module_name] = module
    165     for dep in module.get('dependencies', []):
    166         list = dependents_by_module_name.get(dep)
    167         if not list:
    168             list = []
    169             dependents_by_module_name[dep] = list
    170         list.append(module_name)
    171 
    172 
    173 def verify_worker_modules():
    174     for name in modules_by_name:
    175         for dependency in modules_by_name[name].get('dependencies', []):
    176             if dependency in worker_modules_by_name:
    177                 log_error('Module "%s" may not depend on the worker module "%s"' % (name, dependency))
    178                 errors_found = True
    179 
    180 verify_worker_modules()
    181 
    182 
    183 def check_duplicate_files():
    184 
    185     def check_module(module, seen_files, seen_modules):
    186         name = module['name']
    187         seen_modules[name] = True
    188         for dep_name in module.get('dependencies', []):
    189             if not dep_name in seen_modules:
    190                 check_module(modules_by_name[dep_name], seen_files, seen_modules)
    191         for source in module.get('scripts', []):
    192             referencing_module = seen_files.get(source)
    193             if referencing_module:
    194                 log_error('Duplicate use of %s in "%s" (previously seen in "%s")' % (source, name, referencing_module))
    195             seen_files[source] = name
    196 
    197     for module_name in worker_modules_by_name:
    198         check_module(worker_modules_by_name[module_name], {}, {})
    199 
    200 print 'Checking duplicate files across modules...'
    201 check_duplicate_files()
    202 
    203 
    204 def module_arg(module_name):
    205     return ' --module ' + jsmodule_name_prefix + module_name
    206 
    207 
    208 def dump_module(name, recursively, processed_modules):
    209     if name in processed_modules:
    210         return ''
    211     processed_modules[name] = True
    212     module = modules_by_name[name]
    213     skipped_scripts = set(module.get('skip_compilation', []))
    214 
    215     command = ''
    216     dependencies = module.get('dependencies', [])
    217     if recursively:
    218         for dependency in dependencies:
    219             command += dump_module(dependency, recursively, processed_modules)
    220     command += module_arg(name) + ':'
    221     filtered_scripts = descriptors.module_compiled_files(name)
    222     command += str(len(filtered_scripts))
    223     firstDependency = True
    224     for dependency in dependencies + [runtime_module_name]:
    225         if firstDependency:
    226             command += ':'
    227         else:
    228             command += ','
    229         firstDependency = False
    230         command += jsmodule_name_prefix + dependency
    231     for script in filtered_scripts:
    232         command += ' --js ' + path.join(devtools_frontend_path, name, script)
    233     return command
    234 
    235 print 'Compiling frontend...'
    236 
    237 compiler_args_file = tempfile.NamedTemporaryFile(mode='wt', delete=False)
    238 try:
    239     for name in descriptors.sorted_modules():
    240         closure_args = common_closure_args
    241         closure_args += ' --externs ' + global_externs_file
    242         closure_args += ' --externs ' + protocol_externs_file
    243         runtime_module = module_arg(runtime_module_name) + ':1 --js ' + path.join(devtools_frontend_path, 'Runtime.js')
    244         closure_args += runtime_module + dump_module(name, True, {})
    245         compiler_args_file.write('%s %s\n' % (name, closure_args))
    246 finally:
    247     compiler_args_file.close()
    248 
    249 closure_runner_command = '%s -jar %s --compiler-args-file %s' % (java_exec, closure_runner_jar, compiler_args_file.name)
    250 modular_compiler_proc = run_in_shell(closure_runner_command)
    251 
    252 
    253 def unclosure_injected_script(sourceFileName, outFileName):
    254 
    255     source = read_file(sourceFileName)
    256 
    257     def replace_function(matchobj):
    258         return re.sub(r'@param', 'param', matchobj.group(1) or '') + '\n//' + matchobj.group(2)
    259 
    260     # Comment out the closure function and its jsdocs
    261     source = re.sub(r'(/\*\*(?:[\s\n]*\*\s*@param[^\n]+\n)+\s*\*/\s*)?\n(\(function)', replace_function, source, count=1)
    262 
    263     # Comment out its return statement
    264     source = re.sub(r'\n(\s*return\s+[^;]+;\s*\n\}\)\s*)$', '\n/*\\1*/', source)
    265 
    266     # Replace the "var Object" override with a "self.Object" one
    267     source = re.sub(r'\nvar Object =', '\nself.Object =', source, count=1)
    268 
    269     write_file(outFileName, source)
    270 
    271 injectedScriptSourceTmpFile = path.join(inspector_path, 'InjectedScriptSourceTmp.js')
    272 injectedScriptCanvasModuleSourceTmpFile = path.join(inspector_path, 'InjectedScriptCanvasModuleSourceTmp.js')
    273 
    274 unclosure_injected_script(injected_script_source_name, injectedScriptSourceTmpFile)
    275 unclosure_injected_script(canvas_injected_script_source_name, injectedScriptCanvasModuleSourceTmpFile)
    276 
    277 print 'Compiling InjectedScriptSource.js and InjectedScriptCanvasModuleSource.js...'
    278 injected_script_externs_file = tempfile.NamedTemporaryFile(mode='wt', delete=False)
    279 try:
    280     generate_injected_script_externs.generate_injected_script_externs(injected_script_externs_idl_names, injected_script_externs_file)
    281 finally:
    282     injected_script_externs_file.close()
    283 
    284 command = spawned_compiler_command
    285 command += '    --externs ' + injected_script_externs_file.name + ' \\\n'
    286 command += '    --externs ' + protocol_externs_file + ' \\\n'
    287 command += '    --module ' + jsmodule_name_prefix + 'injected_script' + ':1' + ' \\\n'
    288 command += '        --js ' + injectedScriptSourceTmpFile + ' \\\n'
    289 command += '    --module ' + jsmodule_name_prefix + 'injected_canvas_script' + ':1:' + jsmodule_name_prefix + 'injected_script' + ' \\\n'
    290 command += '        --js ' + injectedScriptCanvasModuleSourceTmpFile + ' \\\n'
    291 command += '\n'
    292 
    293 injectedScriptCompileProc = run_in_shell(command)
    294 
    295 print 'Verifying JSDoc comments...'
    296 additional_jsdoc_check_files = [injectedScriptSourceTmpFile, injectedScriptCanvasModuleSourceTmpFile]
    297 errors_found |= verify_jsdoc(additional_jsdoc_check_files)
    298 jsdocValidatorProc = verify_jsdoc_extra(additional_jsdoc_check_files)
    299 
    300 print 'Checking generated code in InjectedScriptCanvasModuleSource.js...'
    301 check_injected_webgl_calls_command = '%s/check_injected_webgl_calls_info.py %s %s' % (scripts_path, webgl_rendering_context_idl_path, canvas_injected_script_source_name)
    302 canvasModuleCompileProc = run_in_shell(check_injected_webgl_calls_command)
    303 
    304 print 'Validating InjectedScriptSource.js...'
    305 check_injected_script_command = '%s/check_injected_script_source.py %s' % (scripts_path, injected_script_source_name)
    306 validateInjectedScriptProc = run_in_shell(check_injected_script_command)
    307 
    308 print
    309 
    310 (jsdocValidatorOut, _) = jsdocValidatorProc.communicate()
    311 if jsdocValidatorOut:
    312     print ('JSDoc validator output:\n%s' % jsdocValidatorOut)
    313     errors_found = True
    314 
    315 (moduleCompileOut, _) = modular_compiler_proc.communicate()
    316 print 'Modular compilation output:'
    317 
    318 start_module_regex = re.compile(r'^@@ START_MODULE:(.+) @@$')
    319 end_module_regex = re.compile(r'^@@ END_MODULE @@$')
    320 
    321 in_module = False
    322 skipped_modules = {}
    323 error_count = 0
    324 
    325 def skip_dependents(module_name):
    326     for skipped_module in dependents_by_module_name.get(module_name, []):
    327         skipped_modules[skipped_module] = True
    328 
    329 # pylint: disable=E1103
    330 for line in moduleCompileOut.splitlines():
    331     if not in_module:
    332         match = re.search(start_module_regex, line)
    333         if not match:
    334             continue
    335         in_module = True
    336         module_error_count = 0
    337         module_output = []
    338         module_name = match.group(1)
    339         skip_module = skipped_modules.get(module_name)
    340         if skip_module:
    341             skip_dependents(module_name)
    342     else:
    343         match = re.search(end_module_regex, line)
    344         if not match:
    345             if not skip_module:
    346                 module_output.append(line)
    347                 if hasErrors(line):
    348                     error_count += 1
    349                     module_error_count += 1
    350                     skip_dependents(module_name)
    351             continue
    352 
    353         in_module = False
    354         if skip_module:
    355             print 'Skipping module %s...' % module_name
    356         elif not module_error_count:
    357             print 'Module %s compiled successfully: %s' % (module_name, module_output[0])
    358         else:
    359             print 'Module %s compile failed: %s errors\n' % (module_name, module_error_count)
    360             print os.linesep.join(module_output)
    361 
    362 if error_count:
    363     print 'Total Closure errors: %d\n' % error_count
    364     errors_found = True
    365 
    366 (injectedScriptCompileOut, _) = injectedScriptCompileProc.communicate()
    367 print 'InjectedScriptSource.js and InjectedScriptCanvasModuleSource.js compilation output:\n', injectedScriptCompileOut
    368 errors_found |= hasErrors(injectedScriptCompileOut)
    369 
    370 (canvasModuleCompileOut, _) = canvasModuleCompileProc.communicate()
    371 print 'InjectedScriptCanvasModuleSource.js generated code check output:\n', canvasModuleCompileOut
    372 errors_found |= hasErrors(canvasModuleCompileOut)
    373 
    374 (validateInjectedScriptOut, _) = validateInjectedScriptProc.communicate()
    375 print 'Validate InjectedScriptSource.js output:\n', (validateInjectedScriptOut if validateInjectedScriptOut else '<empty>')
    376 errors_found |= hasErrors(validateInjectedScriptOut)
    377 
    378 if errors_found:
    379     print 'ERRORS DETECTED'
    380 
    381 os.remove(injectedScriptSourceTmpFile)
    382 os.remove(injectedScriptCanvasModuleSourceTmpFile)
    383 os.remove(compiler_args_file.name)
    384 os.remove(injected_script_externs_file.name)
    385 os.remove(protocol_externs_file)
    386 shutil.rmtree(modules_dir, True)
    387