1 # Copyright (c) 2013 The Chromium Authors. All rights reserved. 2 # Use of this source code is governed by a BSD-style license that can be 3 # found in the LICENSE file. 4 5 6 """Top-level presubmit script for Skia. 7 8 See http://dev.chromium.org/developers/how-tos/depottools/presubmit-scripts 9 for more details about the presubmit API built into gcl. 10 """ 11 12 import collections 13 import csv 14 import fnmatch 15 import os 16 import re 17 import subprocess 18 import sys 19 import traceback 20 21 22 REVERT_CL_SUBJECT_PREFIX = 'Revert ' 23 24 SKIA_TREE_STATUS_URL = 'http://skia-tree-status.appspot.com' 25 26 # Please add the complete email address here (and not just 'xyz@' or 'xyz'). 27 PUBLIC_API_OWNERS = ( 28 'reed (at] chromium.org', 29 'reed (at] google.com', 30 'bsalomon (at] chromium.org', 31 'bsalomon (at] google.com', 32 'djsollen (at] chromium.org', 33 'djsollen (at] google.com', 34 'hcm (at] chromium.org', 35 'hcm (at] google.com', 36 ) 37 38 AUTHORS_FILE_NAME = 'AUTHORS' 39 40 DOCS_PREVIEW_URL = 'https://skia.org/?cl=' 41 GOLD_TRYBOT_URL = 'https://gold.skia.org/search?issue=' 42 43 SERVICE_ACCOUNT_SUFFIX = [ 44 '@%s.iam.gserviceaccount.com' % project for project in [ 45 'skia-buildbots.google.com', 'skia-swarming-bots', 'skia-public', 46 'skia-corp.google.com']] 47 48 49 def _CheckChangeHasEol(input_api, output_api, source_file_filter=None): 50 """Checks that files end with atleast one \n (LF).""" 51 eof_files = [] 52 for f in input_api.AffectedSourceFiles(source_file_filter): 53 contents = input_api.ReadFile(f, 'rb') 54 # Check that the file ends in atleast one newline character. 55 if len(contents) > 1 and contents[-1:] != '\n': 56 eof_files.append(f.LocalPath()) 57 58 if eof_files: 59 return [output_api.PresubmitPromptWarning( 60 'These files should end in a newline character:', 61 items=eof_files)] 62 return [] 63 64 65 def _JsonChecks(input_api, output_api): 66 """Run checks on any modified json files.""" 67 failing_files = [] 68 for affected_file in input_api.AffectedFiles(None): 69 affected_file_path = affected_file.LocalPath() 70 is_json = affected_file_path.endswith('.json') 71 is_metadata = (affected_file_path.startswith('site/') and 72 affected_file_path.endswith('/METADATA')) 73 if is_json or is_metadata: 74 try: 75 input_api.json.load(open(affected_file_path, 'r')) 76 except ValueError: 77 failing_files.append(affected_file_path) 78 79 results = [] 80 if failing_files: 81 results.append( 82 output_api.PresubmitError( 83 'The following files contain invalid json:\n%s\n\n' % 84 '\n'.join(failing_files))) 85 return results 86 87 88 def _IfDefChecks(input_api, output_api): 89 """Ensures if/ifdef are not before includes. See skbug/3362 for details.""" 90 comment_block_start_pattern = re.compile('^\s*\/\*.*$') 91 comment_block_middle_pattern = re.compile('^\s+\*.*') 92 comment_block_end_pattern = re.compile('^\s+\*\/.*$') 93 single_line_comment_pattern = re.compile('^\s*//.*$') 94 def is_comment(line): 95 return (comment_block_start_pattern.match(line) or 96 comment_block_middle_pattern.match(line) or 97 comment_block_end_pattern.match(line) or 98 single_line_comment_pattern.match(line)) 99 100 empty_line_pattern = re.compile('^\s*$') 101 def is_empty_line(line): 102 return empty_line_pattern.match(line) 103 104 failing_files = [] 105 for affected_file in input_api.AffectedSourceFiles(None): 106 affected_file_path = affected_file.LocalPath() 107 if affected_file_path.endswith('.cpp') or affected_file_path.endswith('.h'): 108 f = open(affected_file_path) 109 for line in f.xreadlines(): 110 if is_comment(line) or is_empty_line(line): 111 continue 112 # The below will be the first real line after comments and newlines. 113 if line.startswith('#if 0 '): 114 pass 115 elif line.startswith('#if ') or line.startswith('#ifdef '): 116 failing_files.append(affected_file_path) 117 break 118 119 results = [] 120 if failing_files: 121 results.append( 122 output_api.PresubmitError( 123 'The following files have #if or #ifdef before includes:\n%s\n\n' 124 'See https://bug.skia.org/3362 for why this should be fixed.' % 125 '\n'.join(failing_files))) 126 return results 127 128 129 def _CopyrightChecks(input_api, output_api, source_file_filter=None): 130 results = [] 131 year_pattern = r'\d{4}' 132 year_range_pattern = r'%s(-%s)?' % (year_pattern, year_pattern) 133 years_pattern = r'%s(,%s)*,?' % (year_range_pattern, year_range_pattern) 134 copyright_pattern = ( 135 r'Copyright (\([cC]\) )?%s \w+' % years_pattern) 136 137 for affected_file in input_api.AffectedSourceFiles(source_file_filter): 138 if 'third_party' in affected_file.LocalPath(): 139 continue 140 contents = input_api.ReadFile(affected_file, 'rb') 141 if not re.search(copyright_pattern, contents): 142 results.append(output_api.PresubmitError( 143 '%s is missing a correct copyright header.' % affected_file)) 144 return results 145 146 147 def _ToolFlags(input_api, output_api): 148 """Make sure `{dm,nanobench}_flags.py test` passes if modified.""" 149 results = [] 150 sources = lambda x: ('dm_flags.py' in x.LocalPath() or 151 'nanobench_flags.py' in x.LocalPath()) 152 for f in input_api.AffectedSourceFiles(sources): 153 if 0 != subprocess.call(['python', f.LocalPath(), 'test']): 154 results.append(output_api.PresubmitError('`python %s test` failed' % f)) 155 return results 156 157 158 def _InfraTests(input_api, output_api): 159 """Run the infra tests.""" 160 results = [] 161 if not any(f.LocalPath().startswith('infra') 162 for f in input_api.AffectedFiles()): 163 return results 164 165 cmd = ['python', os.path.join('infra', 'bots', 'infra_tests.py')] 166 try: 167 subprocess.check_output(cmd) 168 except subprocess.CalledProcessError as e: 169 results.append(output_api.PresubmitError( 170 '`%s` failed:\n%s' % (' '.join(cmd), e.output))) 171 return results 172 173 174 def _CheckGNFormatted(input_api, output_api): 175 """Make sure any .gn files we're changing have been formatted.""" 176 results = [] 177 for f in input_api.AffectedFiles(): 178 if (not f.LocalPath().endswith('.gn') and 179 not f.LocalPath().endswith('.gni')): 180 continue 181 182 gn = 'gn.bat' if 'win32' in sys.platform else 'gn' 183 cmd = [gn, 'format', '--dry-run', f.LocalPath()] 184 try: 185 subprocess.check_output(cmd) 186 except subprocess.CalledProcessError: 187 fix = 'gn format ' + f.LocalPath() 188 results.append(output_api.PresubmitError( 189 '`%s` failed, try\n\t%s' % (' '.join(cmd), fix))) 190 return results 191 192 193 class _WarningsAsErrors(): 194 def __init__(self, output_api): 195 self.output_api = output_api 196 self.old_warning = None 197 def __enter__(self): 198 self.old_warning = self.output_api.PresubmitPromptWarning 199 self.output_api.PresubmitPromptWarning = self.output_api.PresubmitError 200 return self.output_api 201 def __exit__(self, ex_type, ex_value, ex_traceback): 202 self.output_api.PresubmitPromptWarning = self.old_warning 203 204 205 def _CommonChecks(input_api, output_api): 206 """Presubmit checks common to upload and commit.""" 207 results = [] 208 sources = lambda x: (x.LocalPath().endswith('.h') or 209 x.LocalPath().endswith('.py') or 210 x.LocalPath().endswith('.sh') or 211 x.LocalPath().endswith('.m') or 212 x.LocalPath().endswith('.mm') or 213 x.LocalPath().endswith('.go') or 214 x.LocalPath().endswith('.c') or 215 x.LocalPath().endswith('.cc') or 216 x.LocalPath().endswith('.cpp')) 217 results.extend(_CheckChangeHasEol( 218 input_api, output_api, source_file_filter=sources)) 219 with _WarningsAsErrors(output_api): 220 results.extend(input_api.canned_checks.CheckChangeHasNoCR( 221 input_api, output_api, source_file_filter=sources)) 222 results.extend(input_api.canned_checks.CheckChangeHasNoStrayWhitespace( 223 input_api, output_api, source_file_filter=sources)) 224 results.extend(_JsonChecks(input_api, output_api)) 225 results.extend(_IfDefChecks(input_api, output_api)) 226 results.extend(_CopyrightChecks(input_api, output_api, 227 source_file_filter=sources)) 228 results.extend(_ToolFlags(input_api, output_api)) 229 return results 230 231 232 def CheckChangeOnUpload(input_api, output_api): 233 """Presubmit checks for the change on upload. 234 235 The following are the presubmit checks: 236 * Check change has one and only one EOL. 237 """ 238 results = [] 239 results.extend(_CommonChecks(input_api, output_api)) 240 # Run on upload, not commit, since the presubmit bot apparently doesn't have 241 # coverage or Go installed. 242 results.extend(_InfraTests(input_api, output_api)) 243 244 results.extend(_CheckGNFormatted(input_api, output_api)) 245 return results 246 247 248 def _CheckTreeStatus(input_api, output_api, json_url): 249 """Check whether to allow commit. 250 251 Args: 252 input_api: input related apis. 253 output_api: output related apis. 254 json_url: url to download json style status. 255 """ 256 tree_status_results = input_api.canned_checks.CheckTreeIsOpen( 257 input_api, output_api, json_url=json_url) 258 if not tree_status_results: 259 # Check for caution state only if tree is not closed. 260 connection = input_api.urllib2.urlopen(json_url) 261 status = input_api.json.loads(connection.read()) 262 connection.close() 263 if ('caution' in status['message'].lower() and 264 os.isatty(sys.stdout.fileno())): 265 # Display a prompt only if we are in an interactive shell. Without this 266 # check the commit queue behaves incorrectly because it considers 267 # prompts to be failures. 268 short_text = 'Tree state is: ' + status['general_state'] 269 long_text = status['message'] + '\n' + json_url 270 tree_status_results.append( 271 output_api.PresubmitPromptWarning( 272 message=short_text, long_text=long_text)) 273 else: 274 # Tree status is closed. Put in message about contacting sheriff. 275 connection = input_api.urllib2.urlopen( 276 SKIA_TREE_STATUS_URL + '/current-sheriff') 277 sheriff_details = input_api.json.loads(connection.read()) 278 if sheriff_details: 279 tree_status_results[0]._message += ( 280 '\n\nPlease contact the current Skia sheriff (%s) if you are trying ' 281 'to submit a build fix\nand do not know how to submit because the ' 282 'tree is closed') % sheriff_details['username'] 283 return tree_status_results 284 285 286 class CodeReview(object): 287 """Abstracts which codereview tool is used for the specified issue.""" 288 289 def __init__(self, input_api): 290 self._issue = input_api.change.issue 291 self._gerrit = input_api.gerrit 292 293 def GetOwnerEmail(self): 294 return self._gerrit.GetChangeOwner(self._issue) 295 296 def GetSubject(self): 297 return self._gerrit.GetChangeInfo(self._issue)['subject'] 298 299 def GetDescription(self): 300 return self._gerrit.GetChangeDescription(self._issue) 301 302 def IsDryRun(self): 303 return self._gerrit.GetChangeInfo( 304 self._issue)['labels']['Commit-Queue'].get('value', 0) == 1 305 306 def GetReviewers(self): 307 code_review_label = ( 308 self._gerrit.GetChangeInfo(self._issue)['labels']['Code-Review']) 309 return [r['email'] for r in code_review_label.get('all', [])] 310 311 def GetApprovers(self): 312 approvers = [] 313 code_review_label = ( 314 self._gerrit.GetChangeInfo(self._issue)['labels']['Code-Review']) 315 for m in code_review_label.get('all', []): 316 if m.get("value") == 1: 317 approvers.append(m["email"]) 318 return approvers 319 320 321 def _CheckOwnerIsInAuthorsFile(input_api, output_api): 322 results = [] 323 if input_api.change.issue: 324 cr = CodeReview(input_api) 325 326 owner_email = cr.GetOwnerEmail() 327 328 # Service accounts don't need to be in AUTHORS. 329 for suffix in SERVICE_ACCOUNT_SUFFIX: 330 if owner_email.endswith(suffix): 331 return results 332 333 try: 334 authors_content = '' 335 for line in open(AUTHORS_FILE_NAME): 336 if not line.startswith('#'): 337 authors_content += line 338 email_fnmatches = re.findall('<(.*)>', authors_content) 339 for email_fnmatch in email_fnmatches: 340 if fnmatch.fnmatch(owner_email, email_fnmatch): 341 # Found a match, the user is in the AUTHORS file break out of the loop 342 break 343 else: 344 results.append( 345 output_api.PresubmitError( 346 'The email %s is not in Skia\'s AUTHORS file.\n' 347 'Issue owner, this CL must include an addition to the Skia AUTHORS ' 348 'file.' 349 % owner_email)) 350 except IOError: 351 # Do not fail if authors file cannot be found. 352 traceback.print_exc() 353 input_api.logging.error('AUTHORS file not found!') 354 355 return results 356 357 358 def _CheckLGTMsForPublicAPI(input_api, output_api): 359 """Check LGTMs for public API changes. 360 361 For public API files make sure there is an LGTM from the list of owners in 362 PUBLIC_API_OWNERS. 363 """ 364 results = [] 365 requires_owner_check = False 366 for affected_file in input_api.AffectedFiles(): 367 affected_file_path = affected_file.LocalPath() 368 file_path, file_ext = os.path.splitext(affected_file_path) 369 # We only care about files that end in .h and are under the top-level 370 # include dir, but not include/private. 371 if (file_ext == '.h' and 372 'include' == file_path.split(os.path.sep)[0] and 373 'private' not in file_path): 374 requires_owner_check = True 375 376 if not requires_owner_check: 377 return results 378 379 lgtm_from_owner = False 380 if input_api.change.issue: 381 cr = CodeReview(input_api) 382 383 if re.match(REVERT_CL_SUBJECT_PREFIX, cr.GetSubject(), re.I): 384 # It is a revert CL, ignore the public api owners check. 385 return results 386 387 if cr.IsDryRun(): 388 # Ignore public api owners check for dry run CLs since they are not 389 # going to be committed. 390 return results 391 392 if input_api.gerrit: 393 for reviewer in cr.GetReviewers(): 394 if reviewer in PUBLIC_API_OWNERS: 395 # If an owner is specified as an reviewer in Gerrit then ignore the 396 # public api owners check. 397 return results 398 else: 399 match = re.search(r'^TBR=(.*)$', cr.GetDescription(), re.M) 400 if match: 401 tbr_section = match.group(1).strip().split(' ')[0] 402 tbr_entries = tbr_section.split(',') 403 for owner in PUBLIC_API_OWNERS: 404 if owner in tbr_entries or owner.split('@')[0] in tbr_entries: 405 # If an owner is specified in the TBR= line then ignore the public 406 # api owners check. 407 return results 408 409 if cr.GetOwnerEmail() in PUBLIC_API_OWNERS: 410 # An owner created the CL that is an automatic LGTM. 411 lgtm_from_owner = True 412 413 for approver in cr.GetApprovers(): 414 if approver in PUBLIC_API_OWNERS: 415 # Found an lgtm in a message from an owner. 416 lgtm_from_owner = True 417 break 418 419 if not lgtm_from_owner: 420 results.append( 421 output_api.PresubmitError( 422 "If this CL adds to or changes Skia's public API, you need an LGTM " 423 "from any of %s. If this CL only removes from or doesn't change " 424 "Skia's public API, please add a short note to the CL saying so. " 425 "Add one of the owners as a reviewer to your CL as well as to the " 426 "TBR= line. If you don't know if this CL affects Skia's public " 427 "API, treat it like it does." % str(PUBLIC_API_OWNERS))) 428 return results 429 430 431 def _FooterExists(footers, key, value): 432 for k, v in footers: 433 if k == key and v == value: 434 return True 435 return False 436 437 438 def PostUploadHook(cl, change, output_api): 439 """git cl upload will call this hook after the issue is created/modified. 440 441 This hook does the following: 442 * Adds a link to preview docs changes if there are any docs changes in the CL. 443 * Adds 'No-Try: true' if the CL contains only docs changes. 444 """ 445 446 results = [] 447 atleast_one_docs_change = False 448 all_docs_changes = True 449 for affected_file in change.AffectedFiles(): 450 affected_file_path = affected_file.LocalPath() 451 file_path, _ = os.path.splitext(affected_file_path) 452 if 'site' == file_path.split(os.path.sep)[0]: 453 atleast_one_docs_change = True 454 else: 455 all_docs_changes = False 456 if atleast_one_docs_change and not all_docs_changes: 457 break 458 459 issue = cl.issue 460 if issue: 461 # Skip PostUploadHooks for all auto-commit service account bots. New 462 # patchsets (caused due to PostUploadHooks) invalidates the CQ+2 vote from 463 # the "--use-commit-queue" flag to "git cl upload". 464 for suffix in SERVICE_ACCOUNT_SUFFIX: 465 if cl.GetIssueOwner().endswith(suffix): 466 return results 467 468 original_description_lines, footers = cl.GetDescriptionFooters() 469 new_description_lines = list(original_description_lines) 470 471 # If the change includes only doc changes then add No-Try: true in the 472 # CL's description if it does not exist yet. 473 if all_docs_changes and not _FooterExists(footers, 'No-Try', 'true'): 474 new_description_lines.append('No-Try: true') 475 results.append( 476 output_api.PresubmitNotifyResult( 477 'This change has only doc changes. Automatically added ' 478 '\'No-Try: true\' to the CL\'s description')) 479 480 # If there is atleast one docs change then add preview link in the CL's 481 # description if it does not already exist there. 482 docs_preview_link = '%s%s' % (DOCS_PREVIEW_URL, issue) 483 docs_preview_line = 'Docs-Preview: %s' % docs_preview_link 484 if (atleast_one_docs_change and 485 not _FooterExists(footers, 'Docs-Preview', docs_preview_link)): 486 # Automatically add a link to where the docs can be previewed. 487 new_description_lines.append(docs_preview_line) 488 results.append( 489 output_api.PresubmitNotifyResult( 490 'Automatically added a link to preview the docs changes to the ' 491 'CL\'s description')) 492 493 # If the description has changed update it. 494 if new_description_lines != original_description_lines: 495 # Add a new line separating the new contents from the old contents. 496 new_description_lines.insert(len(original_description_lines), '') 497 cl.UpdateDescriptionFooters(new_description_lines, footers) 498 499 return results 500 501 502 def CheckChangeOnCommit(input_api, output_api): 503 """Presubmit checks for the change on commit. 504 505 The following are the presubmit checks: 506 * Check change has one and only one EOL. 507 * Ensures that the Skia tree is open in 508 http://skia-tree-status.appspot.com/. Shows a warning if it is in 'Caution' 509 state and an error if it is in 'Closed' state. 510 """ 511 results = [] 512 results.extend(_CommonChecks(input_api, output_api)) 513 results.extend( 514 _CheckTreeStatus(input_api, output_api, json_url=( 515 SKIA_TREE_STATUS_URL + '/banner-status?format=json'))) 516 results.extend(_CheckLGTMsForPublicAPI(input_api, output_api)) 517 results.extend(_CheckOwnerIsInAuthorsFile(input_api, output_api)) 518 # Checks for the presence of 'DO NOT''SUBMIT' in CL description and in 519 # content of files. 520 results.extend( 521 input_api.canned_checks.CheckDoNotSubmit(input_api, output_api)) 522 return results 523
