Home | History | Annotate | Download | only in tools 
      1 #!/usr/bin/env python
      2 
      3 # Copyright 2014 The Chromium Authors. All rights reserved.
      4 # Use of this source code is governed by a BSD-style license that can be
      5 # found in the LICENSE file.
      6 
      7 '''Produces various output formats from a set of JavaScript files with
      8 closure style require/provide calls.
      9 
     10 Scans one or more directory trees for JavaScript files.  Then, from a
     11 given list of top-level files, sorts all required input files topologically.
     12 The top-level files are appended to the sorted list in the order specified
     13 on the command line.  If no root directories are specified, the source
     14 files are assumed to be ordered already and no dependency analysis is
     15 performed.  The resulting file list can then be used in one of the following
     16 ways:
     17 
     18 - list: a plain list of files, one per line is output.
     19 
     20 - html: a series of html <script> tags with src attributes containing paths
     21   is output.
     22 
     23 - bundle: a concatenation of all the files, separated by newlines is output.
     24 
     25 - compressed_bundle: A bundle where non-significant whitespace, including
     26   comments, has been stripped is output.
     27 
     28 - copy: the files are copied, or hard linked if possible, to the destination
     29   directory.  In this case, no output is generated.
     30 '''
     31 
     32 
     33 import optparse
     34 import os
     35 import re
     36 import shutil
     37 import sys
     38 
     39 _SCRIPT_DIR = os.path.realpath(os.path.dirname(__file__))
     40 _CHROME_SOURCE = os.path.realpath(
     41     os.path.join(_SCRIPT_DIR, *[os.path.pardir] * 6))
     42 sys.path.insert(0, os.path.join(
     43     _CHROME_SOURCE, 'third_party/WebKit/Source/build/scripts'))
     44 sys.path.insert(0, os.path.join(
     45     _CHROME_SOURCE, ('chrome/third_party/chromevox/third_party/' +
     46                      'closure-library/closure/bin/build')))
     47 import depstree
     48 import rjsmin
     49 import source
     50 import treescan
     51 
     52 
     53 def Die(message):
     54   '''Prints an error message and exit the program.'''
     55   print >>sys.stderr, message
     56   sys.exit(1)
     57 
     58 
     59 class SourceWithPaths(source.Source):
     60   '''A source.Source object with its relative input and output paths'''
     61 
     62   def __init__(self, content, in_path, out_path):
     63     super(SourceWithPaths, self).__init__(content)
     64     self._in_path = in_path
     65     self._out_path = out_path
     66 
     67   def GetInPath(self):
     68     return self._in_path
     69 
     70   def GetOutPath(self):
     71     return self._out_path
     72 
     73 
     74 class Bundle():
     75   '''An ordered list of sources without duplicates.'''
     76 
     77   def __init__(self):
     78     self._added_paths = set()
     79     self._added_sources = []
     80 
     81   def Add(self, sources):
     82     '''Appends one or more source objects the list if it doesn't already
     83     exist.
     84 
     85     Args:
     86       sources: A SourceWithPath or an iterable of such objects.
     87     '''
     88     if isinstance(sources, SourceWithPaths):
     89       sources = [sources]
     90     for source in sources:
     91       path = source.GetInPath()
     92       if path not in self._added_paths:
     93         self._added_paths.add(path)
     94         self._added_sources.append(source)
     95 
     96   def GetInPaths(self):
     97     return (source.GetInPath() for source in self._added_sources)
     98 
     99   def GetOutPaths(self):
    100     return (source.GetOutPath() for source in self._added_sources)
    101 
    102   def GetSources(self):
    103     return self._added_sources
    104 
    105   def GetUncompressedSource(self):
    106     return '\n'.join((s.GetSource() for s in self._added_sources))
    107 
    108   def GetCompressedSource(self):
    109     return rjsmin.jsmin(self.GetUncompressedSource())
    110 
    111 
    112 class PathRewriter():
    113   '''A list of simple path rewrite rules to map relative input paths to
    114   relative output paths.
    115   '''
    116 
    117   def __init__(self, specs=[]):
    118     '''Args:
    119       specs: A list of mappings, each consisting of the input prefix and
    120         the corresponding output prefix separated by colons.
    121     '''
    122     self._prefix_map = []
    123     for spec in specs:
    124       parts = spec.split(':')
    125       if len(parts) != 2:
    126         Die('Invalid prefix rewrite spec %s' % spec)
    127       if not parts[0].endswith('/') and parts[0] != '':
    128         parts[0] += '/'
    129       self._prefix_map.append(parts)
    130     self._prefix_map.sort(reverse=True)
    131 
    132   def RewritePath(self, in_path):
    133     '''Rewrites an input path according to the list of rules.
    134 
    135     Args:
    136       in_path, str: The input path to rewrite.
    137     Returns:
    138       str: The corresponding output path.
    139     '''
    140     for in_prefix, out_prefix in self._prefix_map:
    141       if in_path.startswith(in_prefix):
    142         return os.path.join(out_prefix, in_path[len(in_prefix):])
    143     return in_path
    144 
    145 
    146 def ReadSources(roots=[], source_files=[], need_source_text=False,
    147                 path_rewriter=PathRewriter(), exclude=[]):
    148   '''Reads all source specified on the command line, including sources
    149   included by --root options.
    150   '''
    151 
    152   def EnsureSourceLoaded(in_path, sources):
    153     if in_path not in sources:
    154       out_path = path_rewriter.RewritePath(in_path)
    155       sources[in_path] = SourceWithPaths(source.GetFileContents(in_path),
    156                                          in_path, out_path)
    157 
    158   # Only read the actual source file if we will do a dependency analysis or
    159   # the caller asks for it.
    160   need_source_text = need_source_text or len(roots) > 0
    161   sources = {}
    162   for root in roots:
    163     for name in treescan.ScanTreeForJsFiles(root):
    164       if any((r.search(name) for r in exclude)):
    165         continue
    166       EnsureSourceLoaded(name, sources)
    167   for path in source_files:
    168     if need_source_text:
    169       EnsureSourceLoaded(path, sources)
    170     else:
    171       # Just add an empty representation of the source.
    172       sources[path] = SourceWithPaths(
    173           '', path, path_rewriter.RewritePath(path))
    174   return sources
    175 
    176 
    177 def _GetBase(sources):
    178   '''Gets the closure base.js file if present among the sources.
    179 
    180   Args:
    181     sources: Dictionary with input path names as keys and SourceWithPaths
    182       as values.
    183   Returns:
    184     SourceWithPath: The source file providing the goog namespace.
    185   '''
    186   for source in sources.itervalues():
    187     if (os.path.basename(source.GetInPath()) == 'base.js' and
    188         'goog' in source.provides):
    189       return source
    190   Die('goog.base not provided by any file.')
    191 
    192 
    193 def CalcDeps(bundle, sources, top_level):
    194   '''Calculates dependencies for a set of top-level files.
    195 
    196   Args:
    197     bundle: Bundle to add the sources to.
    198     sources, dict: Mapping from input path to SourceWithPaths objects.
    199     top_level, list: List of top-level input paths to calculate dependencies
    200       for.
    201   '''
    202   providers = [s for s in sources.itervalues() if len(s.provides) > 0]
    203   deps = depstree.DepsTree(providers)
    204   namespaces = []
    205   for path in top_level:
    206     namespaces.extend(sources[path].requires)
    207   # base.js is an implicit dependency that always goes first.
    208   bundle.Add(_GetBase(sources))
    209   bundle.Add(deps.GetDependencies(namespaces))
    210 
    211 
    212 def _MarkAsCompiled(sources):
    213   '''Sets COMPILED to true in the Closure base.js source.
    214 
    215   Args:
    216     sources: Dictionary with input paths names as keys and SourcWithPaths
    217       objects as values.
    218   '''
    219   base = _GetBase(sources)
    220   new_content, count = re.subn('^var COMPILED = false;$',
    221                                'var COMPILED = true;',
    222                                base.GetSource(),
    223                                count=1,
    224                                flags=re.MULTILINE)
    225   if count != 1:
    226     Die('COMPILED var assignment not found in %s' % base.GetInPath())
    227   sources[base.GetInPath()] = SourceWithPaths(
    228       new_content,
    229       base.GetInPath(),
    230       base.GetOutPath())
    231 
    232 def LinkOrCopyFiles(sources, dest_dir):
    233   '''Copies a list of sources to a destination directory.'''
    234 
    235   def LinkOrCopyOneFile(src, dst):
    236     if not os.path.exists(os.path.dirname(dst)):
    237       os.makedirs(os.path.dirname(dst))
    238     if os.path.exists(dst):
    239       # Avoid clobbering the inode if source and destination refer to the
    240       # same file already.
    241       if os.path.samefile(src, dst):
    242         return
    243       os.unlink(dst)
    244     try:
    245       os.link(src, dst)
    246     except:
    247       shutil.copy(src, dst)
    248 
    249   for source in sources:
    250     LinkOrCopyOneFile(source.GetInPath(),
    251                       os.path.join(dest_dir, source.GetOutPath()))
    252 
    253 
    254 def WriteOutput(bundle, format, out_file, dest_dir):
    255   '''Writes output in the specified format.
    256 
    257   Args:
    258     bundle: The ordered bundle iwth all sources already added.
    259     format: Output format, one of list, html, bundle, compressed_bundle.
    260     out_file: File object to receive the output.
    261     dest_dir: Prepended to each path mentioned in the output, if applicable.
    262   '''
    263   if format == 'list':
    264     paths = bundle.GetOutPaths()
    265     if dest_dir:
    266       paths = (os.path.join(dest_dir, p) for p in paths)
    267     paths = (os.path.normpath(p) for p in paths)
    268     out_file.write('\n'.join(paths))
    269   elif format == 'html':
    270     HTML_TEMPLATE = '<script src=\'%s\'>'
    271     script_lines = (HTML_TEMPLATE % p for p in bundle.GetOutPaths())
    272     out_file.write('\n'.join(script_lines))
    273   elif format == 'bundle':
    274     out_file.write(bundle.GetUncompressedSource())
    275   elif format == 'compressed_bundle':
    276     out_file.write(bundle.GetCompressedSource())
    277   out_file.write('\n')
    278 
    279 
    280 def CreateOptionParser():
    281   parser = optparse.OptionParser(description=__doc__)
    282   parser.usage = '%prog [options] <top_level_file>...'
    283   parser.add_option('-d', '--dest_dir', action='store', metavar='DIR',
    284                     help=('Destination directory.  Used when translating ' +
    285                           'input paths to output paths and when copying '
    286                           'files.'))
    287   parser.add_option('-o', '--output_file', action='store', metavar='FILE',
    288                     help=('File to output result to for modes that output '
    289                           'a single file.'))
    290   parser.add_option('-r', '--root', dest='roots', action='append', default=[],
    291                     metavar='ROOT',
    292                     help='Roots of directory trees to scan for sources.')
    293   parser.add_option('-w', '--rewrite_prefix', action='append', default=[],
    294                     dest='prefix_map', metavar='SPEC',
    295                     help=('Two path prefixes, separated by colons ' +
    296                           'specifying that a file whose (relative) path ' +
    297                           'name starts with the first prefix should have ' +
    298                           'that prefix replaced by the second prefix to ' +
    299                           'form a path relative to the output directory.'))
    300   parser.add_option('-m', '--mode', type='choice', action='store',
    301                     choices=['list', 'html', 'bundle',
    302                              'compressed_bundle', 'copy'],
    303                     default='list', metavar='MODE',
    304                     help=("Otput mode. One of 'list', 'html', 'bundle', " +
    305                           "'compressed_bundle' or 'copy'."))
    306   parser.add_option('-x', '--exclude', action='append', default=[],
    307                     help=('Exclude files whose full path contains a match for '
    308                           'the given regular expression.  Does not apply to '
    309                           'filenames given as arguments.'))
    310   return parser
    311 
    312 
    313 def main():
    314   options, args = CreateOptionParser().parse_args()
    315   if len(args) < 1:
    316     Die('At least one top-level source file must be specified.')
    317   will_output_source_text = options.mode in ('bundle', 'compressed_bundle')
    318   path_rewriter = PathRewriter(options.prefix_map)
    319   exclude = [re.compile(r) for r in options.exclude]
    320   sources = ReadSources(options.roots, args, will_output_source_text,
    321                         path_rewriter, exclude)
    322   if will_output_source_text:
    323     _MarkAsCompiled(sources)
    324   bundle = Bundle()
    325   if len(options.roots) > 0:
    326     CalcDeps(bundle, sources, args)
    327   bundle.Add((sources[name] for name in args))
    328   if options.mode == 'copy':
    329     if options.dest_dir is None:
    330       Die('Must specify --dest_dir when copying.')
    331     LinkOrCopyFiles(bundle.GetSources(), options.dest_dir)
    332   else:
    333     if options.output_file:
    334       out_file = open(options.output_file, 'w')
    335     else:
    336       out_file = sys.stdout
    337     try:
    338       WriteOutput(bundle, options.mode, out_file, options.dest_dir)
    339     finally:
    340       if options.output_file:
    341         out_file.close()
    342 
    343 
    344 if __name__ == '__main__':
    345   main()
    346