Home | History | Annotate | Download | only in tools 
      1 #!/usr/bin/env python
      2 #
      3 # Copyright 2012 the V8 project authors. All rights reserved.
      4 # Redistribution and use in source and binary forms, with or without
      5 # modification, are permitted provided that the following conditions are
      6 # met:
      7 #
      8 #     * Redistributions of source code must retain the above copyright
      9 #       notice, this list of conditions and the following disclaimer.
     10 #     * Redistributions in binary form must reproduce the above
     11 #       copyright notice, this list of conditions and the following
     12 #       disclaimer in the documentation and/or other materials provided
     13 #       with the distribution.
     14 #     * Neither the name of Google Inc. nor the names of its
     15 #       contributors may be used to endorse or promote products derived
     16 #       from this software without specific prior written permission.
     17 #
     18 # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
     19 # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
     20 # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
     21 # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
     22 # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
     23 # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
     24 # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
     25 # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
     26 # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
     27 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
     28 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
     29 
     30 try:
     31   import hashlib
     32   md5er = hashlib.md5
     33 except ImportError, e:
     34   import md5
     35   md5er = md5.new
     36 
     37 
     38 import optparse
     39 import os
     40 from os.path import abspath, join, dirname, basename, exists
     41 import pickle
     42 import re
     43 import sys
     44 import subprocess
     45 import multiprocessing
     46 from subprocess import PIPE
     47 
     48 # Disabled LINT rules and reason.
     49 # build/include_what_you_use: Started giving false positives for variables
     50 #  named "string" and "map" assuming that you needed to include STL headers.
     51 
     52 ENABLED_LINT_RULES = """
     53 build/class
     54 build/deprecated
     55 build/endif_comment
     56 build/forward_decl
     57 build/include_alpha
     58 build/include_order
     59 build/printf_format
     60 build/storage_class
     61 legal/copyright
     62 readability/boost
     63 readability/braces
     64 readability/casting
     65 readability/constructors
     66 readability/fn_size
     67 readability/function
     68 readability/multiline_comment
     69 readability/multiline_string
     70 readability/streams
     71 readability/todo
     72 readability/utf8
     73 runtime/arrays
     74 runtime/casting
     75 runtime/deprecated_fn
     76 runtime/explicit
     77 runtime/int
     78 runtime/memset
     79 runtime/mutex
     80 runtime/nonconf
     81 runtime/printf
     82 runtime/printf_format
     83 runtime/rtti
     84 runtime/sizeof
     85 runtime/string
     86 runtime/virtual
     87 runtime/vlog
     88 whitespace/blank_line
     89 whitespace/braces
     90 whitespace/comma
     91 whitespace/comments
     92 whitespace/ending_newline
     93 whitespace/indent
     94 whitespace/labels
     95 whitespace/line_length
     96 whitespace/newline
     97 whitespace/operators
     98 whitespace/parens
     99 whitespace/tab
    100 whitespace/todo
    101 """.split()
    102 
    103 # TODO(bmeurer): Fix and re-enable readability/check
    104 
    105 LINT_OUTPUT_PATTERN = re.compile(r'^.+[:(]\d+[:)]|^Done processing')
    106 
    107 
    108 def CppLintWorker(command):
    109   try:
    110     process = subprocess.Popen(command, stderr=subprocess.PIPE)
    111     process.wait()
    112     out_lines = ""
    113     error_count = -1
    114     while True:
    115       out_line = process.stderr.readline()
    116       if out_line == '' and process.poll() != None:
    117         if error_count == -1:
    118           print "Failed to process %s" % command.pop()
    119           return 1
    120         break
    121       m = LINT_OUTPUT_PATTERN.match(out_line)
    122       if m:
    123         out_lines += out_line
    124         error_count += 1
    125     sys.stdout.write(out_lines)
    126     return error_count
    127   except KeyboardInterrupt:
    128     process.kill()
    129   except:
    130     print('Error running cpplint.py. Please make sure you have depot_tools' +
    131           ' in your $PATH. Lint check skipped.')
    132     process.kill()
    133 
    134 
    135 class FileContentsCache(object):
    136 
    137   def __init__(self, sums_file_name):
    138     self.sums = {}
    139     self.sums_file_name = sums_file_name
    140 
    141   def Load(self):
    142     try:
    143       sums_file = None
    144       try:
    145         sums_file = open(self.sums_file_name, 'r')
    146         self.sums = pickle.load(sums_file)
    147       except:
    148         # Cannot parse pickle for any reason. Not much we can do about it.
    149         pass
    150     finally:
    151       if sums_file:
    152         sums_file.close()
    153 
    154   def Save(self):
    155     try:
    156       sums_file = open(self.sums_file_name, 'w')
    157       pickle.dump(self.sums, sums_file)
    158     except:
    159       # Failed to write pickle. Try to clean-up behind us.
    160       if sums_file:
    161         sums_file.close()
    162       try:
    163         os.unlink(self.sums_file_name)
    164       except:
    165         pass
    166     finally:
    167       sums_file.close()
    168 
    169   def FilterUnchangedFiles(self, files):
    170     changed_or_new = []
    171     for file in files:
    172       try:
    173         handle = open(file, "r")
    174         file_sum = md5er(handle.read()).digest()
    175         if not file in self.sums or self.sums[file] != file_sum:
    176           changed_or_new.append(file)
    177           self.sums[file] = file_sum
    178       finally:
    179         handle.close()
    180     return changed_or_new
    181 
    182   def RemoveFile(self, file):
    183     if file in self.sums:
    184       self.sums.pop(file)
    185 
    186 
    187 class SourceFileProcessor(object):
    188   """
    189   Utility class that can run through a directory structure, find all relevant
    190   files and invoke a custom check on the files.
    191   """
    192 
    193   def Run(self, path):
    194     all_files = []
    195     for file in self.GetPathsToSearch():
    196       all_files += self.FindFilesIn(join(path, file))
    197     if not self.ProcessFiles(all_files, path):
    198       return False
    199     return True
    200 
    201   def IgnoreDir(self, name):
    202     return (name.startswith('.') or
    203             name in ('buildtools', 'data', 'gmock', 'gtest', 'kraken',
    204                      'octane', 'sunspider'))
    205 
    206   def IgnoreFile(self, name):
    207     return name.startswith('.')
    208 
    209   def FindFilesIn(self, path):
    210     result = []
    211     for (root, dirs, files) in os.walk(path):
    212       for ignored in [x for x in dirs if self.IgnoreDir(x)]:
    213         dirs.remove(ignored)
    214       for file in files:
    215         if not self.IgnoreFile(file) and self.IsRelevant(file):
    216           result.append(join(root, file))
    217     return result
    218 
    219 
    220 class CppLintProcessor(SourceFileProcessor):
    221   """
    222   Lint files to check that they follow the google code style.
    223   """
    224 
    225   def IsRelevant(self, name):
    226     return name.endswith('.cc') or name.endswith('.h')
    227 
    228   def IgnoreDir(self, name):
    229     return (super(CppLintProcessor, self).IgnoreDir(name)
    230               or (name == 'third_party'))
    231 
    232   IGNORE_LINT = ['flag-definitions.h']
    233 
    234   def IgnoreFile(self, name):
    235     return (super(CppLintProcessor, self).IgnoreFile(name)
    236               or (name in CppLintProcessor.IGNORE_LINT))
    237 
    238   def GetPathsToSearch(self):
    239     return ['src', 'include', 'samples', join('test', 'cctest')]
    240 
    241   def GetCpplintScript(self, prio_path):
    242     for path in [prio_path] + os.environ["PATH"].split(os.pathsep):
    243       path = path.strip('"')
    244       cpplint = os.path.join(path, "cpplint.py")
    245       if os.path.isfile(cpplint):
    246         return cpplint
    247 
    248     return None
    249 
    250   def ProcessFiles(self, files, path):
    251     good_files_cache = FileContentsCache('.cpplint-cache')
    252     good_files_cache.Load()
    253     files = good_files_cache.FilterUnchangedFiles(files)
    254     if len(files) == 0:
    255       print 'No changes in files detected. Skipping cpplint check.'
    256       return True
    257 
    258     filt = '-,' + ",".join(['+' + n for n in ENABLED_LINT_RULES])
    259     command = [sys.executable, 'cpplint.py', '--filter', filt]
    260     cpplint = self.GetCpplintScript(join(path, "tools"))
    261     if cpplint is None:
    262       print('Could not find cpplint.py. Make sure '
    263             'depot_tools is installed and in the path.')
    264       sys.exit(1)
    265 
    266     command = [sys.executable, cpplint, '--filter', filt]
    267 
    268     commands = join([command + [file] for file in files])
    269     count = multiprocessing.cpu_count()
    270     pool = multiprocessing.Pool(count)
    271     try:
    272       results = pool.map_async(CppLintWorker, commands).get(999999)
    273     except KeyboardInterrupt:
    274       print "\nCaught KeyboardInterrupt, terminating workers."
    275       sys.exit(1)
    276 
    277     for i in range(len(files)):
    278       if results[i] > 0:
    279         good_files_cache.RemoveFile(files[i])
    280 
    281     total_errors = sum(results)
    282     print "Total errors found: %d" % total_errors
    283     good_files_cache.Save()
    284     return total_errors == 0
    285 
    286 
    287 COPYRIGHT_HEADER_PATTERN = re.compile(
    288     r'Copyright [\d-]*20[0-1][0-9] the V8 project authors. All rights reserved.')
    289 
    290 class SourceProcessor(SourceFileProcessor):
    291   """
    292   Check that all files include a copyright notice and no trailing whitespaces.
    293   """
    294 
    295   RELEVANT_EXTENSIONS = ['.js', '.cc', '.h', '.py', '.c',
    296                          '.status', '.gyp', '.gypi']
    297 
    298   # Overwriting the one in the parent class.
    299   def FindFilesIn(self, path):
    300     if os.path.exists(path+'/.git'):
    301       output = subprocess.Popen('git ls-files --full-name',
    302                                 stdout=PIPE, cwd=path, shell=True)
    303       result = []
    304       for file in output.stdout.read().split():
    305         for dir_part in os.path.dirname(file).replace(os.sep, '/').split('/'):
    306           if self.IgnoreDir(dir_part):
    307             break
    308         else:
    309           if (self.IsRelevant(file) and os.path.exists(file)
    310               and not self.IgnoreFile(file)):
    311             result.append(join(path, file))
    312       if output.wait() == 0:
    313         return result
    314     return super(SourceProcessor, self).FindFilesIn(path)
    315 
    316   def IsRelevant(self, name):
    317     for ext in SourceProcessor.RELEVANT_EXTENSIONS:
    318       if name.endswith(ext):
    319         return True
    320     return False
    321 
    322   def GetPathsToSearch(self):
    323     return ['.']
    324 
    325   def IgnoreDir(self, name):
    326     return (super(SourceProcessor, self).IgnoreDir(name) or
    327             name in ('third_party', 'gyp', 'out', 'obj', 'DerivedSources'))
    328 
    329   IGNORE_COPYRIGHTS = ['cpplint.py',
    330                        'daemon.py',
    331                        'earley-boyer.js',
    332                        'raytrace.js',
    333                        'crypto.js',
    334                        'libraries.cc',
    335                        'libraries-empty.cc',
    336                        'jsmin.py',
    337                        'regexp-pcre.js',
    338                        'gnuplot-4.6.3-emscripten.js']
    339   IGNORE_TABS = IGNORE_COPYRIGHTS + ['unicode-test.js', 'html-comments.js']
    340 
    341   def EndOfDeclaration(self, line):
    342     return line == "}" or line == "};"
    343 
    344   def StartOfDeclaration(self, line):
    345     return line.find("//") == 0 or \
    346            line.find("/*") == 0 or \
    347            line.find(") {") != -1
    348 
    349   def ProcessContents(self, name, contents):
    350     result = True
    351     base = basename(name)
    352     if not base in SourceProcessor.IGNORE_TABS:
    353       if '\t' in contents:
    354         print "%s contains tabs" % name
    355         result = False
    356     if not base in SourceProcessor.IGNORE_COPYRIGHTS:
    357       if not COPYRIGHT_HEADER_PATTERN.search(contents):
    358         print "%s is missing a correct copyright header." % name
    359         result = False
    360     if ' \n' in contents or contents.endswith(' '):
    361       line = 0
    362       lines = []
    363       parts = contents.split(' \n')
    364       if not contents.endswith(' '):
    365         parts.pop()
    366       for part in parts:
    367         line += part.count('\n') + 1
    368         lines.append(str(line))
    369       linenumbers = ', '.join(lines)
    370       if len(lines) > 1:
    371         print "%s has trailing whitespaces in lines %s." % (name, linenumbers)
    372       else:
    373         print "%s has trailing whitespaces in line %s." % (name, linenumbers)
    374       result = False
    375     if not contents.endswith('\n') or contents.endswith('\n\n'):
    376       print "%s does not end with a single new line." % name
    377       result = False
    378     # Check two empty lines between declarations.
    379     if name.endswith(".cc"):
    380       line = 0
    381       lines = []
    382       parts = contents.split('\n')
    383       while line < len(parts) - 2:
    384         if self.EndOfDeclaration(parts[line]):
    385           if self.StartOfDeclaration(parts[line + 1]):
    386             lines.append(str(line + 1))
    387             line += 1
    388           elif parts[line + 1] == "" and \
    389                self.StartOfDeclaration(parts[line + 2]):
    390             lines.append(str(line + 1))
    391             line += 2
    392         line += 1
    393       if len(lines) >= 1:
    394         linenumbers = ', '.join(lines)
    395         if len(lines) > 1:
    396           print "%s does not have two empty lines between declarations " \
    397                 "in lines %s." % (name, linenumbers)
    398         else:
    399           print "%s does not have two empty lines between declarations " \
    400                 "in line %s." % (name, linenumbers)
    401         result = False
    402     return result
    403 
    404   def ProcessFiles(self, files, path):
    405     success = True
    406     violations = 0
    407     for file in files:
    408       try:
    409         handle = open(file)
    410         contents = handle.read()
    411         if not self.ProcessContents(file, contents):
    412           success = False
    413           violations += 1
    414       finally:
    415         handle.close()
    416     print "Total violating files: %s" % violations
    417     return success
    418 
    419 
    420 def CheckRuntimeVsNativesNameClashes(workspace):
    421   code = subprocess.call(
    422       [sys.executable, join(workspace, "tools", "check-name-clashes.py")])
    423   return code == 0
    424 
    425 
    426 def CheckExternalReferenceRegistration(workspace):
    427   code = subprocess.call(
    428       [sys.executable, join(workspace, "tools", "external-reference-check.py")])
    429   return code == 0
    430 
    431 
    432 def GetOptions():
    433   result = optparse.OptionParser()
    434   result.add_option('--no-lint', help="Do not run cpplint", default=False,
    435                     action="store_true")
    436   return result
    437 
    438 
    439 def Main():
    440   workspace = abspath(join(dirname(sys.argv[0]), '..'))
    441   parser = GetOptions()
    442   (options, args) = parser.parse_args()
    443   success = True
    444   print "Running C++ lint check..."
    445   if not options.no_lint:
    446     success = CppLintProcessor().Run(workspace) and success
    447   print "Running copyright header, trailing whitespaces and " \
    448         "two empty lines between declarations check..."
    449   success = SourceProcessor().Run(workspace) and success
    450   success = CheckRuntimeVsNativesNameClashes(workspace) and success
    451   success = CheckExternalReferenceRegistration(workspace) and success
    452   if success:
    453     return 0
    454   else:
    455     return 1
    456 
    457 
    458 if __name__ == '__main__':
    459   sys.exit(Main())
    460