1 #!/usr/bin/env python 2 # 3 # Copyright 2012 the V8 project authors. All rights reserved. 4 # Redistribution and use in source and binary forms, with or without 5 # modification, are permitted provided that the following conditions are 6 # met: 7 # 8 # * Redistributions of source code must retain the above copyright 9 # notice, this list of conditions and the following disclaimer. 10 # * Redistributions in binary form must reproduce the above 11 # copyright notice, this list of conditions and the following 12 # disclaimer in the documentation and/or other materials provided 13 # with the distribution. 14 # * Neither the name of Google Inc. nor the names of its 15 # contributors may be used to endorse or promote products derived 16 # from this software without specific prior written permission. 17 # 18 # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 19 # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 20 # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 21 # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 22 # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 23 # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 24 # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 25 # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 26 # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 27 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 28 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 29 30 try: 31 import hashlib 32 md5er = hashlib.md5 33 except ImportError, e: 34 import md5 35 md5er = md5.new 36 37 38 import optparse 39 import os 40 from os.path import abspath, join, dirname, basename, exists 41 import pickle 42 import re 43 import sys 44 import subprocess 45 import multiprocessing 46 from subprocess import PIPE 47 48 # Disabled LINT rules and reason. 49 # build/include_what_you_use: Started giving false positives for variables 50 # named "string" and "map" assuming that you needed to include STL headers. 51 52 ENABLED_LINT_RULES = """ 53 build/class 54 build/deprecated 55 build/endif_comment 56 build/forward_decl 57 build/include_order 58 build/printf_format 59 build/storage_class 60 legal/copyright 61 readability/boost 62 readability/braces 63 readability/casting 64 readability/check 65 readability/constructors 66 readability/fn_size 67 readability/function 68 readability/multiline_comment 69 readability/multiline_string 70 readability/streams 71 readability/todo 72 readability/utf8 73 runtime/arrays 74 runtime/casting 75 runtime/deprecated_fn 76 runtime/explicit 77 runtime/int 78 runtime/memset 79 runtime/mutex 80 runtime/nonconf 81 runtime/printf 82 runtime/printf_format 83 runtime/references 84 runtime/rtti 85 runtime/sizeof 86 runtime/string 87 runtime/virtual 88 runtime/vlog 89 whitespace/blank_line 90 whitespace/braces 91 whitespace/comma 92 whitespace/comments 93 whitespace/ending_newline 94 whitespace/indent 95 whitespace/labels 96 whitespace/line_length 97 whitespace/newline 98 whitespace/operators 99 whitespace/parens 100 whitespace/tab 101 whitespace/todo 102 """.split() 103 104 105 LINT_OUTPUT_PATTERN = re.compile(r'^.+[:(]\d+[:)]|^Done processing') 106 107 108 def CppLintWorker(command): 109 try: 110 process = subprocess.Popen(command, stderr=subprocess.PIPE) 111 process.wait() 112 out_lines = "" 113 error_count = -1 114 while True: 115 out_line = process.stderr.readline() 116 if out_line == '' and process.poll() != None: 117 if error_count == -1: 118 print "Failed to process %s" % command.pop() 119 return 1 120 break 121 m = LINT_OUTPUT_PATTERN.match(out_line) 122 if m: 123 out_lines += out_line 124 error_count += 1 125 sys.stdout.write(out_lines) 126 return error_count 127 except KeyboardInterrupt: 128 process.kill() 129 except: 130 print('Error running cpplint.py. Please make sure you have depot_tools' + 131 ' in your $PATH. Lint check skipped.') 132 process.kill() 133 134 135 class FileContentsCache(object): 136 137 def __init__(self, sums_file_name): 138 self.sums = {} 139 self.sums_file_name = sums_file_name 140 141 def Load(self): 142 try: 143 sums_file = None 144 try: 145 sums_file = open(self.sums_file_name, 'r') 146 self.sums = pickle.load(sums_file) 147 except: 148 # Cannot parse pickle for any reason. Not much we can do about it. 149 pass 150 finally: 151 if sums_file: 152 sums_file.close() 153 154 def Save(self): 155 try: 156 sums_file = open(self.sums_file_name, 'w') 157 pickle.dump(self.sums, sums_file) 158 except: 159 # Failed to write pickle. Try to clean-up behind us. 160 if sums_file: 161 sums_file.close() 162 try: 163 os.unlink(self.sums_file_name) 164 except: 165 pass 166 finally: 167 sums_file.close() 168 169 def FilterUnchangedFiles(self, files): 170 changed_or_new = [] 171 for file in files: 172 try: 173 handle = open(file, "r") 174 file_sum = md5er(handle.read()).digest() 175 if not file in self.sums or self.sums[file] != file_sum: 176 changed_or_new.append(file) 177 self.sums[file] = file_sum 178 finally: 179 handle.close() 180 return changed_or_new 181 182 def RemoveFile(self, file): 183 if file in self.sums: 184 self.sums.pop(file) 185 186 187 class SourceFileProcessor(object): 188 """ 189 Utility class that can run through a directory structure, find all relevant 190 files and invoke a custom check on the files. 191 """ 192 193 def Run(self, path): 194 all_files = [] 195 for file in self.GetPathsToSearch(): 196 all_files += self.FindFilesIn(join(path, file)) 197 if not self.ProcessFiles(all_files, path): 198 return False 199 return True 200 201 def IgnoreDir(self, name): 202 return (name.startswith('.') or 203 name in ('buildtools', 'data', 'kraken', 'octane', 'sunspider')) 204 205 def IgnoreFile(self, name): 206 return name.startswith('.') 207 208 def FindFilesIn(self, path): 209 result = [] 210 for (root, dirs, files) in os.walk(path): 211 for ignored in [x for x in dirs if self.IgnoreDir(x)]: 212 dirs.remove(ignored) 213 for file in files: 214 if not self.IgnoreFile(file) and self.IsRelevant(file): 215 result.append(join(root, file)) 216 return result 217 218 219 class CppLintProcessor(SourceFileProcessor): 220 """ 221 Lint files to check that they follow the google code style. 222 """ 223 224 def IsRelevant(self, name): 225 return name.endswith('.cc') or name.endswith('.h') 226 227 def IgnoreDir(self, name): 228 return (super(CppLintProcessor, self).IgnoreDir(name) 229 or (name == 'third_party')) 230 231 IGNORE_LINT = ['flag-definitions.h'] 232 233 def IgnoreFile(self, name): 234 return (super(CppLintProcessor, self).IgnoreFile(name) 235 or (name in CppLintProcessor.IGNORE_LINT)) 236 237 def GetPathsToSearch(self): 238 return ['src', 'include', 'samples', join('test', 'cctest')] 239 240 def GetCpplintScript(self, prio_path): 241 for path in [prio_path] + os.environ["PATH"].split(os.pathsep): 242 path = path.strip('"') 243 cpplint = os.path.join(path, "cpplint.py") 244 if os.path.isfile(cpplint): 245 return cpplint 246 247 return None 248 249 def ProcessFiles(self, files, path): 250 good_files_cache = FileContentsCache('.cpplint-cache') 251 good_files_cache.Load() 252 files = good_files_cache.FilterUnchangedFiles(files) 253 if len(files) == 0: 254 print 'No changes in files detected. Skipping cpplint check.' 255 return True 256 257 filt = '-,' + ",".join(['+' + n for n in ENABLED_LINT_RULES]) 258 command = [sys.executable, 'cpplint.py', '--filter', filt] 259 cpplint = self.GetCpplintScript(join(path, "tools")) 260 if cpplint is None: 261 print('Could not find cpplint.py. Make sure ' 262 'depot_tools is installed and in the path.') 263 sys.exit(1) 264 265 command = [sys.executable, cpplint, '--filter', filt] 266 267 commands = join([command + [file] for file in files]) 268 count = multiprocessing.cpu_count() 269 pool = multiprocessing.Pool(count) 270 try: 271 results = pool.map_async(CppLintWorker, commands).get(999999) 272 except KeyboardInterrupt: 273 print "\nCaught KeyboardInterrupt, terminating workers." 274 sys.exit(1) 275 276 for i in range(len(files)): 277 if results[i] > 0: 278 good_files_cache.RemoveFile(files[i]) 279 280 total_errors = sum(results) 281 print "Total errors found: %d" % total_errors 282 good_files_cache.Save() 283 return total_errors == 0 284 285 286 COPYRIGHT_HEADER_PATTERN = re.compile( 287 r'Copyright [\d-]*20[0-1][0-9] the V8 project authors. All rights reserved.') 288 289 class SourceProcessor(SourceFileProcessor): 290 """ 291 Check that all files include a copyright notice and no trailing whitespaces. 292 """ 293 294 RELEVANT_EXTENSIONS = ['.js', '.cc', '.h', '.py', '.c', 295 '.status', '.gyp', '.gypi'] 296 297 # Overwriting the one in the parent class. 298 def FindFilesIn(self, path): 299 if os.path.exists(path+'/.git'): 300 output = subprocess.Popen('git ls-files --full-name', 301 stdout=PIPE, cwd=path, shell=True) 302 result = [] 303 for file in output.stdout.read().split(): 304 for dir_part in os.path.dirname(file).replace(os.sep, '/').split('/'): 305 if self.IgnoreDir(dir_part): 306 break 307 else: 308 if (self.IsRelevant(file) and os.path.exists(file) 309 and not self.IgnoreFile(file)): 310 result.append(join(path, file)) 311 if output.wait() == 0: 312 return result 313 return super(SourceProcessor, self).FindFilesIn(path) 314 315 def IsRelevant(self, name): 316 for ext in SourceProcessor.RELEVANT_EXTENSIONS: 317 if name.endswith(ext): 318 return True 319 return False 320 321 def GetPathsToSearch(self): 322 return ['.'] 323 324 def IgnoreDir(self, name): 325 return (super(SourceProcessor, self).IgnoreDir(name) or 326 name in ('third_party', 'gyp', 'out', 'obj', 'DerivedSources')) 327 328 IGNORE_COPYRIGHTS = ['cpplint.py', 329 'daemon.py', 330 'earley-boyer.js', 331 'raytrace.js', 332 'crypto.js', 333 'libraries.cc', 334 'libraries-empty.cc', 335 'jsmin.py', 336 'regexp-pcre.js', 337 'gnuplot-4.6.3-emscripten.js'] 338 IGNORE_TABS = IGNORE_COPYRIGHTS + ['unicode-test.js', 'html-comments.js'] 339 340 def EndOfDeclaration(self, line): 341 return line == "}" or line == "};" 342 343 def StartOfDeclaration(self, line): 344 return line.find("//") == 0 or \ 345 line.find("/*") == 0 or \ 346 line.find(") {") != -1 347 348 def ProcessContents(self, name, contents): 349 result = True 350 base = basename(name) 351 if not base in SourceProcessor.IGNORE_TABS: 352 if '\t' in contents: 353 print "%s contains tabs" % name 354 result = False 355 if not base in SourceProcessor.IGNORE_COPYRIGHTS: 356 if not COPYRIGHT_HEADER_PATTERN.search(contents): 357 print "%s is missing a correct copyright header." % name 358 result = False 359 if ' \n' in contents or contents.endswith(' '): 360 line = 0 361 lines = [] 362 parts = contents.split(' \n') 363 if not contents.endswith(' '): 364 parts.pop() 365 for part in parts: 366 line += part.count('\n') + 1 367 lines.append(str(line)) 368 linenumbers = ', '.join(lines) 369 if len(lines) > 1: 370 print "%s has trailing whitespaces in lines %s." % (name, linenumbers) 371 else: 372 print "%s has trailing whitespaces in line %s." % (name, linenumbers) 373 result = False 374 if not contents.endswith('\n') or contents.endswith('\n\n'): 375 print "%s does not end with a single new line." % name 376 result = False 377 # Check two empty lines between declarations. 378 if name.endswith(".cc"): 379 line = 0 380 lines = [] 381 parts = contents.split('\n') 382 while line < len(parts) - 2: 383 if self.EndOfDeclaration(parts[line]): 384 if self.StartOfDeclaration(parts[line + 1]): 385 lines.append(str(line + 1)) 386 line += 1 387 elif parts[line + 1] == "" and \ 388 self.StartOfDeclaration(parts[line + 2]): 389 lines.append(str(line + 1)) 390 line += 2 391 line += 1 392 if len(lines) >= 1: 393 linenumbers = ', '.join(lines) 394 if len(lines) > 1: 395 print "%s does not have two empty lines between declarations " \ 396 "in lines %s." % (name, linenumbers) 397 else: 398 print "%s does not have two empty lines between declarations " \ 399 "in line %s." % (name, linenumbers) 400 result = False 401 return result 402 403 def ProcessFiles(self, files, path): 404 success = True 405 violations = 0 406 for file in files: 407 try: 408 handle = open(file) 409 contents = handle.read() 410 if not self.ProcessContents(file, contents): 411 success = False 412 violations += 1 413 finally: 414 handle.close() 415 print "Total violating files: %s" % violations 416 return success 417 418 419 def CheckGeneratedRuntimeTests(workspace): 420 code = subprocess.call( 421 [sys.executable, join(workspace, "tools", "generate-runtime-tests.py"), 422 "check"]) 423 return code == 0 424 425 426 def GetOptions(): 427 result = optparse.OptionParser() 428 result.add_option('--no-lint', help="Do not run cpplint", default=False, 429 action="store_true") 430 return result 431 432 433 def Main(): 434 workspace = abspath(join(dirname(sys.argv[0]), '..')) 435 parser = GetOptions() 436 (options, args) = parser.parse_args() 437 success = True 438 print "Running C++ lint check..." 439 if not options.no_lint: 440 success = CppLintProcessor().Run(workspace) and success 441 print "Running copyright header, trailing whitespaces and " \ 442 "two empty lines between declarations check..." 443 success = SourceProcessor().Run(workspace) and success 444 success = CheckGeneratedRuntimeTests(workspace) and success 445 if success: 446 return 0 447 else: 448 return 1 449 450 451 if __name__ == '__main__': 452 sys.exit(Main()) 453