Home | History | Annotate | Download | only in tools 
      1 #!/usr/bin/env python
      2 #
      3 # Copyright 2012 the V8 project authors. All rights reserved.
      4 # Redistribution and use in source and binary forms, with or without
      5 # modification, are permitted provided that the following conditions are
      6 # met:
      7 #
      8 #     * Redistributions of source code must retain the above copyright
      9 #       notice, this list of conditions and the following disclaimer.
     10 #     * Redistributions in binary form must reproduce the above
     11 #       copyright notice, this list of conditions and the following
     12 #       disclaimer in the documentation and/or other materials provided
     13 #       with the distribution.
     14 #     * Neither the name of Google Inc. nor the names of its
     15 #       contributors may be used to endorse or promote products derived
     16 #       from this software without specific prior written permission.
     17 #
     18 # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
     19 # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
     20 # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
     21 # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
     22 # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
     23 # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
     24 # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
     25 # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
     26 # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
     27 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
     28 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
     29 
     30 try:
     31   import hashlib
     32   md5er = hashlib.md5
     33 except ImportError, e:
     34   import md5
     35   md5er = md5.new
     36 
     37 
     38 import optparse
     39 import os
     40 from os.path import abspath, join, dirname, basename, exists
     41 import pickle
     42 import re
     43 import sys
     44 import subprocess
     45 import multiprocessing
     46 from subprocess import PIPE
     47 
     48 # Disabled LINT rules and reason.
     49 # build/include_what_you_use: Started giving false positives for variables
     50 #  named "string" and "map" assuming that you needed to include STL headers.
     51 
     52 ENABLED_LINT_RULES = """
     53 build/class
     54 build/deprecated
     55 build/endif_comment
     56 build/forward_decl
     57 build/include_order
     58 build/printf_format
     59 build/storage_class
     60 legal/copyright
     61 readability/boost
     62 readability/braces
     63 readability/casting
     64 readability/check
     65 readability/constructors
     66 readability/fn_size
     67 readability/function
     68 readability/multiline_comment
     69 readability/multiline_string
     70 readability/streams
     71 readability/todo
     72 readability/utf8
     73 runtime/arrays
     74 runtime/casting
     75 runtime/deprecated_fn
     76 runtime/explicit
     77 runtime/int
     78 runtime/memset
     79 runtime/mutex
     80 runtime/nonconf
     81 runtime/printf
     82 runtime/printf_format
     83 runtime/references
     84 runtime/rtti
     85 runtime/sizeof
     86 runtime/string
     87 runtime/virtual
     88 runtime/vlog
     89 whitespace/blank_line
     90 whitespace/braces
     91 whitespace/comma
     92 whitespace/comments
     93 whitespace/ending_newline
     94 whitespace/indent
     95 whitespace/labels
     96 whitespace/line_length
     97 whitespace/newline
     98 whitespace/operators
     99 whitespace/parens
    100 whitespace/tab
    101 whitespace/todo
    102 """.split()
    103 
    104 
    105 LINT_OUTPUT_PATTERN = re.compile(r'^.+[:(]\d+[:)]|^Done processing')
    106 
    107 
    108 def CppLintWorker(command):
    109   try:
    110     process = subprocess.Popen(command, stderr=subprocess.PIPE)
    111     process.wait()
    112     out_lines = ""
    113     error_count = -1
    114     while True:
    115       out_line = process.stderr.readline()
    116       if out_line == '' and process.poll() != None:
    117         break
    118       m = LINT_OUTPUT_PATTERN.match(out_line)
    119       if m:
    120         out_lines += out_line
    121         error_count += 1
    122     sys.stderr.write(out_lines)
    123     return error_count
    124   except KeyboardInterrupt:
    125     process.kill()
    126   except:
    127     print('Error running cpplint.py. Please make sure you have depot_tools' +
    128           ' in your $PATH. Lint check skipped.')
    129     process.kill()
    130 
    131 
    132 class FileContentsCache(object):
    133 
    134   def __init__(self, sums_file_name):
    135     self.sums = {}
    136     self.sums_file_name = sums_file_name
    137 
    138   def Load(self):
    139     try:
    140       sums_file = None
    141       try:
    142         sums_file = open(self.sums_file_name, 'r')
    143         self.sums = pickle.load(sums_file)
    144       except IOError:
    145         # File might not exist, this is OK.
    146         pass
    147     finally:
    148       if sums_file:
    149         sums_file.close()
    150 
    151   def Save(self):
    152     try:
    153       sums_file = open(self.sums_file_name, 'w')
    154       pickle.dump(self.sums, sums_file)
    155     finally:
    156       sums_file.close()
    157 
    158   def FilterUnchangedFiles(self, files):
    159     changed_or_new = []
    160     for file in files:
    161       try:
    162         handle = open(file, "r")
    163         file_sum = md5er(handle.read()).digest()
    164         if not file in self.sums or self.sums[file] != file_sum:
    165           changed_or_new.append(file)
    166           self.sums[file] = file_sum
    167       finally:
    168         handle.close()
    169     return changed_or_new
    170 
    171   def RemoveFile(self, file):
    172     if file in self.sums:
    173       self.sums.pop(file)
    174 
    175 
    176 class SourceFileProcessor(object):
    177   """
    178   Utility class that can run through a directory structure, find all relevant
    179   files and invoke a custom check on the files.
    180   """
    181 
    182   def Run(self, path):
    183     all_files = []
    184     for file in self.GetPathsToSearch():
    185       all_files += self.FindFilesIn(join(path, file))
    186     if not self.ProcessFiles(all_files, path):
    187       return False
    188     return True
    189 
    190   def IgnoreDir(self, name):
    191     return name.startswith('.') or name == 'data' or name == 'sputniktests'
    192 
    193   def IgnoreFile(self, name):
    194     return name.startswith('.')
    195 
    196   def FindFilesIn(self, path):
    197     result = []
    198     for (root, dirs, files) in os.walk(path):
    199       for ignored in [x for x in dirs if self.IgnoreDir(x)]:
    200         dirs.remove(ignored)
    201       for file in files:
    202         if not self.IgnoreFile(file) and self.IsRelevant(file):
    203           result.append(join(root, file))
    204     return result
    205 
    206 
    207 class CppLintProcessor(SourceFileProcessor):
    208   """
    209   Lint files to check that they follow the google code style.
    210   """
    211 
    212   def IsRelevant(self, name):
    213     return name.endswith('.cc') or name.endswith('.h')
    214 
    215   def IgnoreDir(self, name):
    216     return (super(CppLintProcessor, self).IgnoreDir(name)
    217               or (name == 'third_party'))
    218 
    219   IGNORE_LINT = ['flag-definitions.h']
    220 
    221   def IgnoreFile(self, name):
    222     return (super(CppLintProcessor, self).IgnoreFile(name)
    223               or (name in CppLintProcessor.IGNORE_LINT))
    224 
    225   def GetPathsToSearch(self):
    226     return ['src', 'preparser', 'include', 'samples', join('test', 'cctest')]
    227 
    228   def ProcessFiles(self, files, path):
    229     good_files_cache = FileContentsCache('.cpplint-cache')
    230     good_files_cache.Load()
    231     files = good_files_cache.FilterUnchangedFiles(files)
    232     if len(files) == 0:
    233       print 'No changes in files detected. Skipping cpplint check.'
    234       return True
    235 
    236     filt = '-,' + ",".join(['+' + n for n in ENABLED_LINT_RULES])
    237     command = ['cpplint.py', '--filter', filt]
    238     local_cpplint = join(path, "tools", "cpplint.py")
    239     if exists(local_cpplint):
    240       command = ['python', local_cpplint, '--filter', filt]
    241 
    242     commands = join([command + [file] for file in files])
    243     count = multiprocessing.cpu_count()
    244     pool = multiprocessing.Pool(count)
    245     try:
    246       results = pool.map_async(CppLintWorker, commands).get(999999)
    247     except KeyboardInterrupt:
    248       print "\nCaught KeyboardInterrupt, terminating workers."
    249       sys.exit(1)
    250 
    251     for i in range(len(files)):
    252       if results[i] > 0:
    253         good_files_cache.RemoveFile(files[i])
    254 
    255     total_errors = sum(results)
    256     print "Total errors found: %d" % total_errors
    257     good_files_cache.Save()
    258     return total_errors == 0
    259 
    260 
    261 COPYRIGHT_HEADER_PATTERN = re.compile(
    262     r'Copyright [\d-]*20[0-1][0-9] the V8 project authors. All rights reserved.')
    263 
    264 class SourceProcessor(SourceFileProcessor):
    265   """
    266   Check that all files include a copyright notice and no trailing whitespaces.
    267   """
    268 
    269   RELEVANT_EXTENSIONS = ['.js', '.cc', '.h', '.py', '.c', 'SConscript',
    270       'SConstruct', '.status', '.gyp', '.gypi']
    271 
    272   # Overwriting the one in the parent class.
    273   def FindFilesIn(self, path):
    274     if os.path.exists(path+'/.git'):
    275       output = subprocess.Popen('git ls-files --full-name',
    276                                 stdout=PIPE, cwd=path, shell=True)
    277       result = []
    278       for file in output.stdout.read().split():
    279         for dir_part in os.path.dirname(file).split(os.sep):
    280           if self.IgnoreDir(dir_part):
    281             break
    282         else:
    283           if self.IsRelevant(file) and not self.IgnoreFile(file):
    284             result.append(join(path, file))
    285       if output.wait() == 0:
    286         return result
    287     return super(SourceProcessor, self).FindFilesIn(path)
    288 
    289   def IsRelevant(self, name):
    290     for ext in SourceProcessor.RELEVANT_EXTENSIONS:
    291       if name.endswith(ext):
    292         return True
    293     return False
    294 
    295   def GetPathsToSearch(self):
    296     return ['.']
    297 
    298   def IgnoreDir(self, name):
    299     return (super(SourceProcessor, self).IgnoreDir(name)
    300               or (name == 'third_party')
    301               or (name == 'gyp')
    302               or (name == 'out')
    303               or (name == 'obj'))
    304 
    305   IGNORE_COPYRIGHTS = ['cpplint.py',
    306                        'earley-boyer.js',
    307                        'raytrace.js',
    308                        'crypto.js',
    309                        'libraries.cc',
    310                        'libraries-empty.cc',
    311                        'jsmin.py',
    312                        'regexp-pcre.js']
    313   IGNORE_TABS = IGNORE_COPYRIGHTS + ['unicode-test.js', 'html-comments.js']
    314 
    315   def ProcessContents(self, name, contents):
    316     result = True
    317     base = basename(name)
    318     if not base in SourceProcessor.IGNORE_TABS:
    319       if '\t' in contents:
    320         print "%s contains tabs" % name
    321         result = False
    322     if not base in SourceProcessor.IGNORE_COPYRIGHTS:
    323       if not COPYRIGHT_HEADER_PATTERN.search(contents):
    324         print "%s is missing a correct copyright header." % name
    325         result = False
    326     ext = base.split('.').pop()
    327     if ' \n' in contents or contents.endswith(' '):
    328       line = 0
    329       lines = []
    330       parts = contents.split(' \n')
    331       if not contents.endswith(' '):
    332         parts.pop()
    333       for part in parts:
    334         line += part.count('\n') + 1
    335         lines.append(str(line))
    336       linenumbers = ', '.join(lines)
    337       if len(lines) > 1:
    338         print "%s has trailing whitespaces in lines %s." % (name, linenumbers)
    339       else:
    340         print "%s has trailing whitespaces in line %s." % (name, linenumbers)
    341       result = False
    342     return result
    343 
    344   def ProcessFiles(self, files, path):
    345     success = True
    346     violations = 0
    347     for file in files:
    348       try:
    349         handle = open(file)
    350         contents = handle.read()
    351         if not self.ProcessContents(file, contents):
    352           success = False
    353           violations += 1
    354       finally:
    355         handle.close()
    356     print "Total violating files: %s" % violations
    357     return success
    358 
    359 
    360 def GetOptions():
    361   result = optparse.OptionParser()
    362   result.add_option('--no-lint', help="Do not run cpplint", default=False,
    363                     action="store_true")
    364   return result
    365 
    366 
    367 def Main():
    368   workspace = abspath(join(dirname(sys.argv[0]), '..'))
    369   parser = GetOptions()
    370   (options, args) = parser.parse_args()
    371   success = True
    372   print "Running C++ lint check..."
    373   if not options.no_lint:
    374     success = CppLintProcessor().Run(workspace) and success
    375   print "Running copyright header and trailing whitespaces check..."
    376   success = SourceProcessor().Run(workspace) and success
    377   if success:
    378     return 0
    379   else:
    380     return 1
    381 
    382 
    383 if __name__ == '__main__':
    384   sys.exit(Main())
    385