1 #!/usr/bin/env python 2 # 3 # Copyright 2012 the V8 project authors. All rights reserved. 4 # Redistribution and use in source and binary forms, with or without 5 # modification, are permitted provided that the following conditions are 6 # met: 7 # 8 # * Redistributions of source code must retain the above copyright 9 # notice, this list of conditions and the following disclaimer. 10 # * Redistributions in binary form must reproduce the above 11 # copyright notice, this list of conditions and the following 12 # disclaimer in the documentation and/or other materials provided 13 # with the distribution. 14 # * Neither the name of Google Inc. nor the names of its 15 # contributors may be used to endorse or promote products derived 16 # from this software without specific prior written permission. 17 # 18 # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 19 # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 20 # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 21 # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 22 # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 23 # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 24 # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 25 # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 26 # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 27 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 28 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 29 30 try: 31 import hashlib 32 md5er = hashlib.md5 33 except ImportError, e: 34 import md5 35 md5er = md5.new 36 37 38 import json 39 import optparse 40 import os 41 from os.path import abspath, join, dirname, basename, exists 42 import pickle 43 import re 44 import sys 45 import subprocess 46 import multiprocessing 47 from subprocess import PIPE 48 49 from testrunner.local import statusfile 50 from testrunner.local import testsuite 51 from testrunner.local import utils 52 53 # Special LINT rules diverging from default and reason. 54 # build/header_guard: Our guards have the form "V8_FOO_H_", not "SRC_FOO_H_". 55 # build/include_what_you_use: Started giving false positives for variables 56 # named "string" and "map" assuming that you needed to include STL headers. 57 # TODO(bmeurer): Fix and re-enable readability/check 58 # TODO(epertoso): Maybe re-enable readability/fn_size after 59 # http://crrev.com/2199323003 relands. 60 61 LINT_RULES = """ 62 -build/header_guard 63 -build/include_what_you_use 64 -build/namespaces 65 -readability/check 66 -readability/fn_size 67 +readability/streams 68 -runtime/references 69 """.split() 70 71 LINT_OUTPUT_PATTERN = re.compile(r'^.+[:(]\d+[:)]|^Done processing') 72 FLAGS_LINE = re.compile("//\s*Flags:.*--([A-z0-9-])+_[A-z0-9].*\n") 73 74 def CppLintWorker(command): 75 try: 76 process = subprocess.Popen(command, stderr=subprocess.PIPE) 77 process.wait() 78 out_lines = "" 79 error_count = -1 80 while True: 81 out_line = process.stderr.readline() 82 if out_line == '' and process.poll() != None: 83 if error_count == -1: 84 print "Failed to process %s" % command.pop() 85 return 1 86 break 87 m = LINT_OUTPUT_PATTERN.match(out_line) 88 if m: 89 out_lines += out_line 90 error_count += 1 91 sys.stdout.write(out_lines) 92 return error_count 93 except KeyboardInterrupt: 94 process.kill() 95 except: 96 print('Error running cpplint.py. Please make sure you have depot_tools' + 97 ' in your $PATH. Lint check skipped.') 98 process.kill() 99 100 101 class FileContentsCache(object): 102 103 def __init__(self, sums_file_name): 104 self.sums = {} 105 self.sums_file_name = sums_file_name 106 107 def Load(self): 108 try: 109 sums_file = None 110 try: 111 sums_file = open(self.sums_file_name, 'r') 112 self.sums = pickle.load(sums_file) 113 except: 114 # Cannot parse pickle for any reason. Not much we can do about it. 115 pass 116 finally: 117 if sums_file: 118 sums_file.close() 119 120 def Save(self): 121 try: 122 sums_file = open(self.sums_file_name, 'w') 123 pickle.dump(self.sums, sums_file) 124 except: 125 # Failed to write pickle. Try to clean-up behind us. 126 if sums_file: 127 sums_file.close() 128 try: 129 os.unlink(self.sums_file_name) 130 except: 131 pass 132 finally: 133 sums_file.close() 134 135 def FilterUnchangedFiles(self, files): 136 changed_or_new = [] 137 for file in files: 138 try: 139 handle = open(file, "r") 140 file_sum = md5er(handle.read()).digest() 141 if not file in self.sums or self.sums[file] != file_sum: 142 changed_or_new.append(file) 143 self.sums[file] = file_sum 144 finally: 145 handle.close() 146 return changed_or_new 147 148 def RemoveFile(self, file): 149 if file in self.sums: 150 self.sums.pop(file) 151 152 153 class SourceFileProcessor(object): 154 """ 155 Utility class that can run through a directory structure, find all relevant 156 files and invoke a custom check on the files. 157 """ 158 159 def Run(self, path): 160 all_files = [] 161 for file in self.GetPathsToSearch(): 162 all_files += self.FindFilesIn(join(path, file)) 163 if not self.ProcessFiles(all_files, path): 164 return False 165 return True 166 167 def IgnoreDir(self, name): 168 return (name.startswith('.') or 169 name in ('buildtools', 'data', 'gmock', 'gtest', 'kraken', 170 'octane', 'sunspider')) 171 172 def IgnoreFile(self, name): 173 return name.startswith('.') 174 175 def FindFilesIn(self, path): 176 result = [] 177 for (root, dirs, files) in os.walk(path): 178 for ignored in [x for x in dirs if self.IgnoreDir(x)]: 179 dirs.remove(ignored) 180 for file in files: 181 if not self.IgnoreFile(file) and self.IsRelevant(file): 182 result.append(join(root, file)) 183 return result 184 185 186 class CppLintProcessor(SourceFileProcessor): 187 """ 188 Lint files to check that they follow the google code style. 189 """ 190 191 def IsRelevant(self, name): 192 return name.endswith('.cc') or name.endswith('.h') 193 194 def IgnoreDir(self, name): 195 return (super(CppLintProcessor, self).IgnoreDir(name) 196 or (name == 'third_party')) 197 198 IGNORE_LINT = ['flag-definitions.h'] 199 200 def IgnoreFile(self, name): 201 return (super(CppLintProcessor, self).IgnoreFile(name) 202 or (name in CppLintProcessor.IGNORE_LINT)) 203 204 def GetPathsToSearch(self): 205 return ['src', 'include', 'samples', join('test', 'cctest'), 206 join('test', 'unittests'), join('test', 'inspector')] 207 208 def GetCpplintScript(self, prio_path): 209 for path in [prio_path] + os.environ["PATH"].split(os.pathsep): 210 path = path.strip('"') 211 cpplint = os.path.join(path, "cpplint.py") 212 if os.path.isfile(cpplint): 213 return cpplint 214 215 return None 216 217 def ProcessFiles(self, files, path): 218 good_files_cache = FileContentsCache('.cpplint-cache') 219 good_files_cache.Load() 220 files = good_files_cache.FilterUnchangedFiles(files) 221 if len(files) == 0: 222 print 'No changes in files detected. Skipping cpplint check.' 223 return True 224 225 filters = ",".join([n for n in LINT_RULES]) 226 command = [sys.executable, 'cpplint.py', '--filter', filters] 227 cpplint = self.GetCpplintScript(join(path, "tools")) 228 if cpplint is None: 229 print('Could not find cpplint.py. Make sure ' 230 'depot_tools is installed and in the path.') 231 sys.exit(1) 232 233 command = [sys.executable, cpplint, '--filter', filters] 234 235 commands = join([command + [file] for file in files]) 236 count = multiprocessing.cpu_count() 237 pool = multiprocessing.Pool(count) 238 try: 239 results = pool.map_async(CppLintWorker, commands).get(999999) 240 except KeyboardInterrupt: 241 print "\nCaught KeyboardInterrupt, terminating workers." 242 sys.exit(1) 243 244 for i in range(len(files)): 245 if results[i] > 0: 246 good_files_cache.RemoveFile(files[i]) 247 248 total_errors = sum(results) 249 print "Total errors found: %d" % total_errors 250 good_files_cache.Save() 251 return total_errors == 0 252 253 254 COPYRIGHT_HEADER_PATTERN = re.compile( 255 r'Copyright [\d-]*20[0-1][0-9] the V8 project authors. All rights reserved.') 256 257 class SourceProcessor(SourceFileProcessor): 258 """ 259 Check that all files include a copyright notice and no trailing whitespaces. 260 """ 261 262 RELEVANT_EXTENSIONS = ['.js', '.cc', '.h', '.py', '.c', 263 '.status', '.gyp', '.gypi'] 264 265 # Overwriting the one in the parent class. 266 def FindFilesIn(self, path): 267 if os.path.exists(path+'/.git'): 268 output = subprocess.Popen('git ls-files --full-name', 269 stdout=PIPE, cwd=path, shell=True) 270 result = [] 271 for file in output.stdout.read().split(): 272 for dir_part in os.path.dirname(file).replace(os.sep, '/').split('/'): 273 if self.IgnoreDir(dir_part): 274 break 275 else: 276 if (self.IsRelevant(file) and os.path.exists(file) 277 and not self.IgnoreFile(file)): 278 result.append(join(path, file)) 279 if output.wait() == 0: 280 return result 281 return super(SourceProcessor, self).FindFilesIn(path) 282 283 def IsRelevant(self, name): 284 for ext in SourceProcessor.RELEVANT_EXTENSIONS: 285 if name.endswith(ext): 286 return True 287 return False 288 289 def GetPathsToSearch(self): 290 return ['.'] 291 292 def IgnoreDir(self, name): 293 return (super(SourceProcessor, self).IgnoreDir(name) or 294 name in ('third_party', 'gyp', 'out', 'obj', 'DerivedSources')) 295 296 IGNORE_COPYRIGHTS = ['box2d.js', 297 'cpplint.py', 298 'check_injected_script_source.py', 299 'copy.js', 300 'corrections.js', 301 'crypto.js', 302 'daemon.py', 303 'debugger-script.js', 304 'earley-boyer.js', 305 'fannkuch.js', 306 'fasta.js', 307 'generate_protocol_externs.py', 308 'injected-script.cc', 309 'injected-script.h', 310 'injected-script-source.js', 311 'java-script-call-frame.cc', 312 'java-script-call-frame.h', 313 'jsmin.py', 314 'libraries.cc', 315 'libraries-empty.cc', 316 'lua_binarytrees.js', 317 'memops.js', 318 'poppler.js', 319 'primes.js', 320 'raytrace.js', 321 'regexp-pcre.js', 322 'rjsmin.py', 323 'script-breakpoint.h', 324 'sqlite.js', 325 'sqlite-change-heap.js', 326 'sqlite-pointer-masking.js', 327 'sqlite-safe-heap.js', 328 'v8-debugger-script.h', 329 'v8-function-call.cc', 330 'v8-function-call.h', 331 'v8-inspector-impl.cc', 332 'v8-inspector-impl.h', 333 'v8-runtime-agent-impl.cc', 334 'v8-runtime-agent-impl.h', 335 'gnuplot-4.6.3-emscripten.js', 336 'zlib.js'] 337 IGNORE_TABS = IGNORE_COPYRIGHTS + ['unicode-test.js', 'html-comments.js'] 338 339 def EndOfDeclaration(self, line): 340 return line == "}" or line == "};" 341 342 def StartOfDeclaration(self, line): 343 return line.find("//") == 0 or \ 344 line.find("/*") == 0 or \ 345 line.find(") {") != -1 346 347 def ProcessContents(self, name, contents): 348 result = True 349 base = basename(name) 350 if not base in SourceProcessor.IGNORE_TABS: 351 if '\t' in contents: 352 print "%s contains tabs" % name 353 result = False 354 if not base in SourceProcessor.IGNORE_COPYRIGHTS: 355 if not COPYRIGHT_HEADER_PATTERN.search(contents): 356 print "%s is missing a correct copyright header." % name 357 result = False 358 if ' \n' in contents or contents.endswith(' '): 359 line = 0 360 lines = [] 361 parts = contents.split(' \n') 362 if not contents.endswith(' '): 363 parts.pop() 364 for part in parts: 365 line += part.count('\n') + 1 366 lines.append(str(line)) 367 linenumbers = ', '.join(lines) 368 if len(lines) > 1: 369 print "%s has trailing whitespaces in lines %s." % (name, linenumbers) 370 else: 371 print "%s has trailing whitespaces in line %s." % (name, linenumbers) 372 result = False 373 if not contents.endswith('\n') or contents.endswith('\n\n'): 374 print "%s does not end with a single new line." % name 375 result = False 376 # Sanitize flags for fuzzer. 377 if "mjsunit" in name: 378 match = FLAGS_LINE.search(contents) 379 if match: 380 print "%s Flags should use '-' (not '_')" % name 381 result = False 382 return result 383 384 def ProcessFiles(self, files, path): 385 success = True 386 violations = 0 387 for file in files: 388 try: 389 handle = open(file) 390 contents = handle.read() 391 if not self.ProcessContents(file, contents): 392 success = False 393 violations += 1 394 finally: 395 handle.close() 396 print "Total violating files: %s" % violations 397 return success 398 399 def _CheckStatusFileForDuplicateKeys(filepath): 400 comma_space_bracket = re.compile(", *]") 401 lines = [] 402 with open(filepath) as f: 403 for line in f.readlines(): 404 # Skip all-comment lines. 405 if line.lstrip().startswith("#"): continue 406 # Strip away comments at the end of the line. 407 comment_start = line.find("#") 408 if comment_start != -1: 409 line = line[:comment_start] 410 line = line.strip() 411 # Strip away trailing commas within the line. 412 line = comma_space_bracket.sub("]", line) 413 if len(line) > 0: 414 lines.append(line) 415 416 # Strip away trailing commas at line ends. Ugh. 417 for i in range(len(lines) - 1): 418 if (lines[i].endswith(",") and len(lines[i + 1]) > 0 and 419 lines[i + 1][0] in ("}", "]")): 420 lines[i] = lines[i][:-1] 421 422 contents = "\n".join(lines) 423 # JSON wants double-quotes. 424 contents = contents.replace("'", '"') 425 # Fill in keywords (like PASS, SKIP). 426 for key in statusfile.KEYWORDS: 427 contents = re.sub(r"\b%s\b" % key, "\"%s\"" % key, contents) 428 429 status = {"success": True} 430 def check_pairs(pairs): 431 keys = {} 432 for key, value in pairs: 433 if key in keys: 434 print("%s: Error: duplicate key %s" % (filepath, key)) 435 status["success"] = False 436 keys[key] = True 437 438 json.loads(contents, object_pairs_hook=check_pairs) 439 return status["success"] 440 441 def CheckStatusFiles(workspace): 442 success = True 443 suite_paths = utils.GetSuitePaths(join(workspace, "test")) 444 for root in suite_paths: 445 suite_path = join(workspace, "test", root) 446 status_file_path = join(suite_path, root + ".status") 447 suite = testsuite.TestSuite.LoadTestSuite(suite_path) 448 if suite and exists(status_file_path): 449 success &= statusfile.PresubmitCheck(status_file_path) 450 success &= _CheckStatusFileForDuplicateKeys(status_file_path) 451 return success 452 453 def CheckAuthorizedAuthor(input_api, output_api): 454 """For non-googler/chromites committers, verify the author's email address is 455 in AUTHORS. 456 """ 457 # TODO(maruel): Add it to input_api? 458 import fnmatch 459 460 author = input_api.change.author_email 461 if not author: 462 input_api.logging.info('No author, skipping AUTHOR check') 463 return [] 464 authors_path = input_api.os_path.join( 465 input_api.PresubmitLocalPath(), 'AUTHORS') 466 valid_authors = ( 467 input_api.re.match(r'[^#]+\s+\<(.+?)\>\s*$', line) 468 for line in open(authors_path)) 469 valid_authors = [item.group(1).lower() for item in valid_authors if item] 470 if not any(fnmatch.fnmatch(author.lower(), valid) for valid in valid_authors): 471 input_api.logging.info('Valid authors are %s', ', '.join(valid_authors)) 472 return [output_api.PresubmitPromptWarning( 473 ('%s is not in AUTHORS file. If you are a new contributor, please visit' 474 '\n' 475 'http://www.chromium.org/developers/contributing-code and read the ' 476 '"Legal" section\n' 477 'If you are a chromite, verify the contributor signed the CLA.') % 478 author)] 479 return [] 480 481 def GetOptions(): 482 result = optparse.OptionParser() 483 result.add_option('--no-lint', help="Do not run cpplint", default=False, 484 action="store_true") 485 return result 486 487 488 def Main(): 489 workspace = abspath(join(dirname(sys.argv[0]), '..')) 490 parser = GetOptions() 491 (options, args) = parser.parse_args() 492 success = True 493 print "Running C++ lint check..." 494 if not options.no_lint: 495 success &= CppLintProcessor().Run(workspace) 496 print "Running copyright header, trailing whitespaces and " \ 497 "two empty lines between declarations check..." 498 success &= SourceProcessor().Run(workspace) 499 success &= CheckStatusFiles(workspace) 500 if success: 501 return 0 502 else: 503 return 1 504 505 506 if __name__ == '__main__': 507 sys.exit(Main()) 508