Home | History | Annotate | Download | only in tools 
      1 #!/usr/bin/env python
      2 #
      3 # Copyright 2012 the V8 project authors. All rights reserved.
      4 # Redistribution and use in source and binary forms, with or without
      5 # modification, are permitted provided that the following conditions are
      6 # met:
      7 #
      8 #     * Redistributions of source code must retain the above copyright
      9 #       notice, this list of conditions and the following disclaimer.
     10 #     * Redistributions in binary form must reproduce the above
     11 #       copyright notice, this list of conditions and the following
     12 #       disclaimer in the documentation and/or other materials provided
     13 #       with the distribution.
     14 #     * Neither the name of Google Inc. nor the names of its
     15 #       contributors may be used to endorse or promote products derived
     16 #       from this software without specific prior written permission.
     17 #
     18 # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
     19 # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
     20 # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
     21 # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
     22 # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
     23 # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
     24 # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
     25 # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
     26 # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
     27 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
     28 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
     29 
     30 try:
     31   import hashlib
     32   md5er = hashlib.md5
     33 except ImportError, e:
     34   import md5
     35   md5er = md5.new
     36 
     37 
     38 import optparse
     39 import os
     40 from os.path import abspath, join, dirname, basename, exists
     41 import pickle
     42 import re
     43 import sys
     44 import subprocess
     45 import multiprocessing
     46 from subprocess import PIPE
     47 
     48 # Disabled LINT rules and reason.
     49 # build/include_what_you_use: Started giving false positives for variables
     50 #  named "string" and "map" assuming that you needed to include STL headers.
     51 
     52 ENABLED_LINT_RULES = """
     53 build/class
     54 build/deprecated
     55 build/endif_comment
     56 build/forward_decl
     57 build/include_order
     58 build/printf_format
     59 build/storage_class
     60 legal/copyright
     61 readability/boost
     62 readability/braces
     63 readability/casting
     64 readability/check
     65 readability/constructors
     66 readability/fn_size
     67 readability/function
     68 readability/multiline_comment
     69 readability/multiline_string
     70 readability/streams
     71 readability/todo
     72 readability/utf8
     73 runtime/arrays
     74 runtime/casting
     75 runtime/deprecated_fn
     76 runtime/explicit
     77 runtime/int
     78 runtime/memset
     79 runtime/mutex
     80 runtime/nonconf
     81 runtime/printf
     82 runtime/printf_format
     83 runtime/references
     84 runtime/rtti
     85 runtime/sizeof
     86 runtime/string
     87 runtime/virtual
     88 runtime/vlog
     89 whitespace/blank_line
     90 whitespace/braces
     91 whitespace/comma
     92 whitespace/comments
     93 whitespace/ending_newline
     94 whitespace/indent
     95 whitespace/labels
     96 whitespace/line_length
     97 whitespace/newline
     98 whitespace/operators
     99 whitespace/parens
    100 whitespace/tab
    101 whitespace/todo
    102 """.split()
    103 
    104 
    105 LINT_OUTPUT_PATTERN = re.compile(r'^.+[:(]\d+[:)]|^Done processing')
    106 
    107 
    108 def CppLintWorker(command):
    109   try:
    110     process = subprocess.Popen(command, stderr=subprocess.PIPE)
    111     process.wait()
    112     out_lines = ""
    113     error_count = -1
    114     while True:
    115       out_line = process.stderr.readline()
    116       if out_line == '' and process.poll() != None:
    117         if error_count == -1:
    118           print "Failed to process %s" % command.pop()
    119           return 1
    120         break
    121       m = LINT_OUTPUT_PATTERN.match(out_line)
    122       if m:
    123         out_lines += out_line
    124         error_count += 1
    125     sys.stdout.write(out_lines)
    126     return error_count
    127   except KeyboardInterrupt:
    128     process.kill()
    129   except:
    130     print('Error running cpplint.py. Please make sure you have depot_tools' +
    131           ' in your $PATH. Lint check skipped.')
    132     process.kill()
    133 
    134 
    135 class FileContentsCache(object):
    136 
    137   def __init__(self, sums_file_name):
    138     self.sums = {}
    139     self.sums_file_name = sums_file_name
    140 
    141   def Load(self):
    142     try:
    143       sums_file = None
    144       try:
    145         sums_file = open(self.sums_file_name, 'r')
    146         self.sums = pickle.load(sums_file)
    147       except:
    148         # Cannot parse pickle for any reason. Not much we can do about it.
    149         pass
    150     finally:
    151       if sums_file:
    152         sums_file.close()
    153 
    154   def Save(self):
    155     try:
    156       sums_file = open(self.sums_file_name, 'w')
    157       pickle.dump(self.sums, sums_file)
    158     except:
    159       # Failed to write pickle. Try to clean-up behind us.
    160       if sums_file:
    161         sums_file.close()
    162       try:
    163         os.unlink(self.sums_file_name)
    164       except:
    165         pass
    166     finally:
    167       sums_file.close()
    168 
    169   def FilterUnchangedFiles(self, files):
    170     changed_or_new = []
    171     for file in files:
    172       try:
    173         handle = open(file, "r")
    174         file_sum = md5er(handle.read()).digest()
    175         if not file in self.sums or self.sums[file] != file_sum:
    176           changed_or_new.append(file)
    177           self.sums[file] = file_sum
    178       finally:
    179         handle.close()
    180     return changed_or_new
    181 
    182   def RemoveFile(self, file):
    183     if file in self.sums:
    184       self.sums.pop(file)
    185 
    186 
    187 class SourceFileProcessor(object):
    188   """
    189   Utility class that can run through a directory structure, find all relevant
    190   files and invoke a custom check on the files.
    191   """
    192 
    193   def Run(self, path):
    194     all_files = []
    195     for file in self.GetPathsToSearch():
    196       all_files += self.FindFilesIn(join(path, file))
    197     if not self.ProcessFiles(all_files, path):
    198       return False
    199     return True
    200 
    201   def IgnoreDir(self, name):
    202     return (name.startswith('.') or
    203             name in ('buildtools', 'data', 'kraken', 'octane', 'sunspider'))
    204 
    205   def IgnoreFile(self, name):
    206     return name.startswith('.')
    207 
    208   def FindFilesIn(self, path):
    209     result = []
    210     for (root, dirs, files) in os.walk(path):
    211       for ignored in [x for x in dirs if self.IgnoreDir(x)]:
    212         dirs.remove(ignored)
    213       for file in files:
    214         if not self.IgnoreFile(file) and self.IsRelevant(file):
    215           result.append(join(root, file))
    216     return result
    217 
    218 
    219 class CppLintProcessor(SourceFileProcessor):
    220   """
    221   Lint files to check that they follow the google code style.
    222   """
    223 
    224   def IsRelevant(self, name):
    225     return name.endswith('.cc') or name.endswith('.h')
    226 
    227   def IgnoreDir(self, name):
    228     return (super(CppLintProcessor, self).IgnoreDir(name)
    229               or (name == 'third_party'))
    230 
    231   IGNORE_LINT = ['flag-definitions.h']
    232 
    233   def IgnoreFile(self, name):
    234     return (super(CppLintProcessor, self).IgnoreFile(name)
    235               or (name in CppLintProcessor.IGNORE_LINT))
    236 
    237   def GetPathsToSearch(self):
    238     return ['src', 'include', 'samples', join('test', 'cctest')]
    239 
    240   def GetCpplintScript(self, prio_path):
    241     for path in [prio_path] + os.environ["PATH"].split(os.pathsep):
    242       path = path.strip('"')
    243       cpplint = os.path.join(path, "cpplint.py")
    244       if os.path.isfile(cpplint):
    245         return cpplint
    246 
    247     return None
    248 
    249   def ProcessFiles(self, files, path):
    250     good_files_cache = FileContentsCache('.cpplint-cache')
    251     good_files_cache.Load()
    252     files = good_files_cache.FilterUnchangedFiles(files)
    253     if len(files) == 0:
    254       print 'No changes in files detected. Skipping cpplint check.'
    255       return True
    256 
    257     filt = '-,' + ",".join(['+' + n for n in ENABLED_LINT_RULES])
    258     command = [sys.executable, 'cpplint.py', '--filter', filt]
    259     cpplint = self.GetCpplintScript(join(path, "tools"))
    260     if cpplint is None:
    261       print('Could not find cpplint.py. Make sure '
    262             'depot_tools is installed and in the path.')
    263       sys.exit(1)
    264 
    265     command = [sys.executable, cpplint, '--filter', filt]
    266 
    267     commands = join([command + [file] for file in files])
    268     count = multiprocessing.cpu_count()
    269     pool = multiprocessing.Pool(count)
    270     try:
    271       results = pool.map_async(CppLintWorker, commands).get(999999)
    272     except KeyboardInterrupt:
    273       print "\nCaught KeyboardInterrupt, terminating workers."
    274       sys.exit(1)
    275 
    276     for i in range(len(files)):
    277       if results[i] > 0:
    278         good_files_cache.RemoveFile(files[i])
    279 
    280     total_errors = sum(results)
    281     print "Total errors found: %d" % total_errors
    282     good_files_cache.Save()
    283     return total_errors == 0
    284 
    285 
    286 COPYRIGHT_HEADER_PATTERN = re.compile(
    287     r'Copyright [\d-]*20[0-1][0-9] the V8 project authors. All rights reserved.')
    288 
    289 class SourceProcessor(SourceFileProcessor):
    290   """
    291   Check that all files include a copyright notice and no trailing whitespaces.
    292   """
    293 
    294   RELEVANT_EXTENSIONS = ['.js', '.cc', '.h', '.py', '.c',
    295                          '.status', '.gyp', '.gypi']
    296 
    297   # Overwriting the one in the parent class.
    298   def FindFilesIn(self, path):
    299     if os.path.exists(path+'/.git'):
    300       output = subprocess.Popen('git ls-files --full-name',
    301                                 stdout=PIPE, cwd=path, shell=True)
    302       result = []
    303       for file in output.stdout.read().split():
    304         for dir_part in os.path.dirname(file).replace(os.sep, '/').split('/'):
    305           if self.IgnoreDir(dir_part):
    306             break
    307         else:
    308           if (self.IsRelevant(file) and os.path.exists(file)
    309               and not self.IgnoreFile(file)):
    310             result.append(join(path, file))
    311       if output.wait() == 0:
    312         return result
    313     return super(SourceProcessor, self).FindFilesIn(path)
    314 
    315   def IsRelevant(self, name):
    316     for ext in SourceProcessor.RELEVANT_EXTENSIONS:
    317       if name.endswith(ext):
    318         return True
    319     return False
    320 
    321   def GetPathsToSearch(self):
    322     return ['.']
    323 
    324   def IgnoreDir(self, name):
    325     return (super(SourceProcessor, self).IgnoreDir(name) or
    326             name in ('third_party', 'gyp', 'out', 'obj', 'DerivedSources'))
    327 
    328   IGNORE_COPYRIGHTS = ['cpplint.py',
    329                        'daemon.py',
    330                        'earley-boyer.js',
    331                        'raytrace.js',
    332                        'crypto.js',
    333                        'libraries.cc',
    334                        'libraries-empty.cc',
    335                        'jsmin.py',
    336                        'regexp-pcre.js',
    337                        'gnuplot-4.6.3-emscripten.js']
    338   IGNORE_TABS = IGNORE_COPYRIGHTS + ['unicode-test.js', 'html-comments.js']
    339 
    340   def EndOfDeclaration(self, line):
    341     return line == "}" or line == "};"
    342 
    343   def StartOfDeclaration(self, line):
    344     return line.find("//") == 0 or \
    345            line.find("/*") == 0 or \
    346            line.find(") {") != -1
    347 
    348   def ProcessContents(self, name, contents):
    349     result = True
    350     base = basename(name)
    351     if not base in SourceProcessor.IGNORE_TABS:
    352       if '\t' in contents:
    353         print "%s contains tabs" % name
    354         result = False
    355     if not base in SourceProcessor.IGNORE_COPYRIGHTS:
    356       if not COPYRIGHT_HEADER_PATTERN.search(contents):
    357         print "%s is missing a correct copyright header." % name
    358         result = False
    359     if ' \n' in contents or contents.endswith(' '):
    360       line = 0
    361       lines = []
    362       parts = contents.split(' \n')
    363       if not contents.endswith(' '):
    364         parts.pop()
    365       for part in parts:
    366         line += part.count('\n') + 1
    367         lines.append(str(line))
    368       linenumbers = ', '.join(lines)
    369       if len(lines) > 1:
    370         print "%s has trailing whitespaces in lines %s." % (name, linenumbers)
    371       else:
    372         print "%s has trailing whitespaces in line %s." % (name, linenumbers)
    373       result = False
    374     if not contents.endswith('\n') or contents.endswith('\n\n'):
    375       print "%s does not end with a single new line." % name
    376       result = False
    377     # Check two empty lines between declarations.
    378     if name.endswith(".cc"):
    379       line = 0
    380       lines = []
    381       parts = contents.split('\n')
    382       while line < len(parts) - 2:
    383         if self.EndOfDeclaration(parts[line]):
    384           if self.StartOfDeclaration(parts[line + 1]):
    385             lines.append(str(line + 1))
    386             line += 1
    387           elif parts[line + 1] == "" and \
    388                self.StartOfDeclaration(parts[line + 2]):
    389             lines.append(str(line + 1))
    390             line += 2
    391         line += 1
    392       if len(lines) >= 1:
    393         linenumbers = ', '.join(lines)
    394         if len(lines) > 1:
    395           print "%s does not have two empty lines between declarations " \
    396                 "in lines %s." % (name, linenumbers)
    397         else:
    398           print "%s does not have two empty lines between declarations " \
    399                 "in line %s." % (name, linenumbers)
    400         result = False
    401     return result
    402 
    403   def ProcessFiles(self, files, path):
    404     success = True
    405     violations = 0
    406     for file in files:
    407       try:
    408         handle = open(file)
    409         contents = handle.read()
    410         if not self.ProcessContents(file, contents):
    411           success = False
    412           violations += 1
    413       finally:
    414         handle.close()
    415     print "Total violating files: %s" % violations
    416     return success
    417 
    418 
    419 def CheckGeneratedRuntimeTests(workspace):
    420   code = subprocess.call(
    421       [sys.executable, join(workspace, "tools", "generate-runtime-tests.py"),
    422        "check"])
    423   return code == 0
    424 
    425 
    426 def GetOptions():
    427   result = optparse.OptionParser()
    428   result.add_option('--no-lint', help="Do not run cpplint", default=False,
    429                     action="store_true")
    430   return result
    431 
    432 
    433 def Main():
    434   workspace = abspath(join(dirname(sys.argv[0]), '..'))
    435   parser = GetOptions()
    436   (options, args) = parser.parse_args()
    437   success = True
    438   print "Running C++ lint check..."
    439   if not options.no_lint:
    440     success = CppLintProcessor().Run(workspace) and success
    441   print "Running copyright header, trailing whitespaces and " \
    442         "two empty lines between declarations check..."
    443   success = SourceProcessor().Run(workspace) and success
    444   success = CheckGeneratedRuntimeTests(workspace) and success
    445   if success:
    446     return 0
    447   else:
    448     return 1
    449 
    450 
    451 if __name__ == '__main__':
    452   sys.exit(Main())
    453