1 # Copyright (c) 2013 The Chromium Authors. All rights reserved. 2 # Use of this source code is governed by a BSD-style license that can be 3 # found in the LICENSE file. 4 5 6 """Top-level presubmit script for Skia. 7 8 See http://dev.chromium.org/developers/how-tos/depottools/presubmit-scripts 9 for more details about the presubmit API built into gcl. 10 """ 11 12 import collections 13 import csv 14 import fnmatch 15 import os 16 import re 17 import subprocess 18 import sys 19 import traceback 20 21 22 REVERT_CL_SUBJECT_PREFIX = 'Revert ' 23 24 SKIA_TREE_STATUS_URL = 'http://skia-tree-status.appspot.com' 25 26 # Please add the complete email address here (and not just 'xyz@' or 'xyz'). 27 PUBLIC_API_OWNERS = ( 28 'reed (at] chromium.org', 29 'reed (at] google.com', 30 'bsalomon (at] chromium.org', 31 'bsalomon (at] google.com', 32 'djsollen (at] chromium.org', 33 'djsollen (at] google.com', 34 ) 35 36 AUTHORS_FILE_NAME = 'AUTHORS' 37 38 DOCS_PREVIEW_URL = 'https://skia.org/?cl=' 39 GOLD_TRYBOT_URL = ('https://gold.skia.org/search2?unt=true' 40 '&query=source_type%3Dgm&master=false&issue=') 41 42 # Path to CQ bots feature is described in https://bug.skia.org/4364 43 PATH_PREFIX_TO_EXTRA_TRYBOTS = { 44 # pylint: disable=line-too-long 45 'cmake/': 'client.skia.compile:Build-Mac-Clang-x86_64-Release-CMake-Trybot,Build-Ubuntu-GCC-x86_64-Release-CMake-Trybot', 46 # pylint: disable=line-too-long 47 'src/opts/': 'client.skia:Test-Ubuntu-GCC-GCE-CPU-AVX2-x86_64-Release-SKNX_NO_SIMD-Trybot', 48 49 'include/private/SkAtomics.h': ('client.skia:' 50 'Test-Ubuntu-GCC-GCE-CPU-AVX2-x86_64-Release-TSAN-Trybot,' 51 'Test-Ubuntu-GCC-Golo-GPU-GT610-x86_64-Release-TSAN-Trybot' 52 ), 53 54 # Below are examples to show what is possible with this feature. 55 # 'src/svg/': 'master1:abc;master2:def', 56 # 'src/svg/parser/': 'master3:ghi,jkl;master4:mno', 57 # 'src/image/SkImage_Base.h': 'master5:pqr,stu;master1:abc1;master2:def', 58 } 59 60 61 def _CheckChangeHasEol(input_api, output_api, source_file_filter=None): 62 """Checks that files end with atleast one \n (LF).""" 63 eof_files = [] 64 for f in input_api.AffectedSourceFiles(source_file_filter): 65 contents = input_api.ReadFile(f, 'rb') 66 # Check that the file ends in atleast one newline character. 67 if len(contents) > 1 and contents[-1:] != '\n': 68 eof_files.append(f.LocalPath()) 69 70 if eof_files: 71 return [output_api.PresubmitPromptWarning( 72 'These files should end in a newline character:', 73 items=eof_files)] 74 return [] 75 76 77 def _PythonChecks(input_api, output_api): 78 """Run checks on any modified Python files.""" 79 pylint_disabled_warnings = ( 80 'F0401', # Unable to import. 81 'E0611', # No name in module. 82 'W0232', # Class has no __init__ method. 83 'E1002', # Use of super on an old style class. 84 'W0403', # Relative import used. 85 'R0201', # Method could be a function. 86 'E1003', # Using class name in super. 87 'W0613', # Unused argument. 88 'W0105', # String statement has no effect. 89 ) 90 # Run Pylint on only the modified python files. Unfortunately it still runs 91 # Pylint on the whole file instead of just the modified lines. 92 affected_python_files = [] 93 for affected_file in input_api.AffectedSourceFiles(None): 94 affected_file_path = affected_file.LocalPath() 95 if affected_file_path.endswith('.py'): 96 affected_python_files.append(affected_file_path) 97 return input_api.canned_checks.RunPylint( 98 input_api, output_api, 99 disabled_warnings=pylint_disabled_warnings, 100 white_list=affected_python_files) 101 102 103 def _IfDefChecks(input_api, output_api): 104 """Ensures if/ifdef are not before includes. See skbug/3362 for details.""" 105 comment_block_start_pattern = re.compile('^\s*\/\*.*$') 106 comment_block_middle_pattern = re.compile('^\s+\*.*') 107 comment_block_end_pattern = re.compile('^\s+\*\/.*$') 108 single_line_comment_pattern = re.compile('^\s*//.*$') 109 def is_comment(line): 110 return (comment_block_start_pattern.match(line) or 111 comment_block_middle_pattern.match(line) or 112 comment_block_end_pattern.match(line) or 113 single_line_comment_pattern.match(line)) 114 115 empty_line_pattern = re.compile('^\s*$') 116 def is_empty_line(line): 117 return empty_line_pattern.match(line) 118 119 failing_files = [] 120 for affected_file in input_api.AffectedSourceFiles(None): 121 affected_file_path = affected_file.LocalPath() 122 if affected_file_path.endswith('.cpp') or affected_file_path.endswith('.h'): 123 f = open(affected_file_path) 124 for line in f.xreadlines(): 125 if is_comment(line) or is_empty_line(line): 126 continue 127 # The below will be the first real line after comments and newlines. 128 if line.startswith('#if 0 '): 129 pass 130 elif line.startswith('#if ') or line.startswith('#ifdef '): 131 failing_files.append(affected_file_path) 132 break 133 134 results = [] 135 if failing_files: 136 results.append( 137 output_api.PresubmitError( 138 'The following files have #if or #ifdef before includes:\n%s\n\n' 139 'See https://bug.skia.org/3362 for why this should be fixed.' % 140 '\n'.join(failing_files))) 141 return results 142 143 144 def _CopyrightChecks(input_api, output_api, source_file_filter=None): 145 results = [] 146 year_pattern = r'\d{4}' 147 year_range_pattern = r'%s(-%s)?' % (year_pattern, year_pattern) 148 years_pattern = r'%s(,%s)*,?' % (year_range_pattern, year_range_pattern) 149 copyright_pattern = ( 150 r'Copyright (\([cC]\) )?%s \w+' % years_pattern) 151 152 for affected_file in input_api.AffectedSourceFiles(source_file_filter): 153 if 'third_party' in affected_file.LocalPath(): 154 continue 155 contents = input_api.ReadFile(affected_file, 'rb') 156 if not re.search(copyright_pattern, contents): 157 results.append(output_api.PresubmitError( 158 '%s is missing a correct copyright header.' % affected_file)) 159 return results 160 161 162 def _ToolFlags(input_api, output_api): 163 """Make sure `{dm,nanobench}_flags.py test` passes if modified.""" 164 results = [] 165 sources = lambda x: ('dm_flags.py' in x.LocalPath() or 166 'nanobench_flags.py' in x.LocalPath()) 167 for f in input_api.AffectedSourceFiles(sources): 168 if 0 != subprocess.call(['python', f.LocalPath(), 'test']): 169 results.append(output_api.PresubmitError('`python %s test` failed' % f)) 170 return results 171 172 173 def _CommonChecks(input_api, output_api): 174 """Presubmit checks common to upload and commit.""" 175 results = [] 176 sources = lambda x: (x.LocalPath().endswith('.h') or 177 x.LocalPath().endswith('.gypi') or 178 x.LocalPath().endswith('.gyp') or 179 x.LocalPath().endswith('.py') or 180 x.LocalPath().endswith('.sh') or 181 x.LocalPath().endswith('.m') or 182 x.LocalPath().endswith('.mm') or 183 x.LocalPath().endswith('.go') or 184 x.LocalPath().endswith('.c') or 185 x.LocalPath().endswith('.cc') or 186 x.LocalPath().endswith('.cpp')) 187 results.extend( 188 _CheckChangeHasEol( 189 input_api, output_api, source_file_filter=sources)) 190 results.extend(_PythonChecks(input_api, output_api)) 191 results.extend(_IfDefChecks(input_api, output_api)) 192 results.extend(_CopyrightChecks(input_api, output_api, 193 source_file_filter=sources)) 194 results.extend(_ToolFlags(input_api, output_api)) 195 return results 196 197 198 def CheckChangeOnUpload(input_api, output_api): 199 """Presubmit checks for the change on upload. 200 201 The following are the presubmit checks: 202 * Check change has one and only one EOL. 203 """ 204 results = [] 205 results.extend(_CommonChecks(input_api, output_api)) 206 return results 207 208 209 def _CheckTreeStatus(input_api, output_api, json_url): 210 """Check whether to allow commit. 211 212 Args: 213 input_api: input related apis. 214 output_api: output related apis. 215 json_url: url to download json style status. 216 """ 217 tree_status_results = input_api.canned_checks.CheckTreeIsOpen( 218 input_api, output_api, json_url=json_url) 219 if not tree_status_results: 220 # Check for caution state only if tree is not closed. 221 connection = input_api.urllib2.urlopen(json_url) 222 status = input_api.json.loads(connection.read()) 223 connection.close() 224 if ('caution' in status['message'].lower() and 225 os.isatty(sys.stdout.fileno())): 226 # Display a prompt only if we are in an interactive shell. Without this 227 # check the commit queue behaves incorrectly because it considers 228 # prompts to be failures. 229 short_text = 'Tree state is: ' + status['general_state'] 230 long_text = status['message'] + '\n' + json_url 231 tree_status_results.append( 232 output_api.PresubmitPromptWarning( 233 message=short_text, long_text=long_text)) 234 else: 235 # Tree status is closed. Put in message about contacting sheriff. 236 connection = input_api.urllib2.urlopen( 237 SKIA_TREE_STATUS_URL + '/current-sheriff') 238 sheriff_details = input_api.json.loads(connection.read()) 239 if sheriff_details: 240 tree_status_results[0]._message += ( 241 '\n\nPlease contact the current Skia sheriff (%s) if you are trying ' 242 'to submit a build fix\nand do not know how to submit because the ' 243 'tree is closed') % sheriff_details['username'] 244 return tree_status_results 245 246 247 def _CheckOwnerIsInAuthorsFile(input_api, output_api): 248 results = [] 249 issue = input_api.change.issue 250 if issue and input_api.rietveld: 251 issue_properties = input_api.rietveld.get_issue_properties( 252 issue=int(issue), messages=False) 253 owner_email = issue_properties['owner_email'] 254 255 try: 256 authors_content = '' 257 for line in open(AUTHORS_FILE_NAME): 258 if not line.startswith('#'): 259 authors_content += line 260 email_fnmatches = re.findall('<(.*)>', authors_content) 261 for email_fnmatch in email_fnmatches: 262 if fnmatch.fnmatch(owner_email, email_fnmatch): 263 # Found a match, the user is in the AUTHORS file break out of the loop 264 break 265 else: 266 results.append( 267 output_api.PresubmitError( 268 'The email %s is not in Skia\'s AUTHORS file.\n' 269 'Issue owner, this CL must include an addition to the Skia AUTHORS ' 270 'file.' 271 % owner_email)) 272 except IOError: 273 # Do not fail if authors file cannot be found. 274 traceback.print_exc() 275 input_api.logging.error('AUTHORS file not found!') 276 277 return results 278 279 280 def _CheckLGTMsForPublicAPI(input_api, output_api): 281 """Check LGTMs for public API changes. 282 283 For public API files make sure there is an LGTM from the list of owners in 284 PUBLIC_API_OWNERS. 285 """ 286 results = [] 287 requires_owner_check = False 288 for affected_file in input_api.AffectedFiles(): 289 affected_file_path = affected_file.LocalPath() 290 file_path, file_ext = os.path.splitext(affected_file_path) 291 # We only care about files that end in .h and are under the top-level 292 # include dir, but not include/private. 293 if (file_ext == '.h' and 294 'include' == file_path.split(os.path.sep)[0] and 295 'private' not in file_path): 296 requires_owner_check = True 297 298 if not requires_owner_check: 299 return results 300 301 lgtm_from_owner = False 302 issue = input_api.change.issue 303 if issue and input_api.rietveld: 304 issue_properties = input_api.rietveld.get_issue_properties( 305 issue=int(issue), messages=True) 306 if re.match(REVERT_CL_SUBJECT_PREFIX, issue_properties['subject'], re.I): 307 # It is a revert CL, ignore the public api owners check. 308 return results 309 310 if issue_properties['cq_dry_run']: 311 # Ignore public api owners check for dry run CLs since they are not 312 # going to be committed. 313 return results 314 315 match = re.search(r'^TBR=(.*)$', issue_properties['description'], re.M) 316 if match: 317 tbr_entries = match.group(1).strip().split(',') 318 for owner in PUBLIC_API_OWNERS: 319 if owner in tbr_entries or owner.split('@')[0] in tbr_entries: 320 # If an owner is specified in the TBR= line then ignore the public 321 # api owners check. 322 return results 323 324 if issue_properties['owner_email'] in PUBLIC_API_OWNERS: 325 # An owner created the CL that is an automatic LGTM. 326 lgtm_from_owner = True 327 328 messages = issue_properties.get('messages') 329 if messages: 330 for message in messages: 331 if (message['sender'] in PUBLIC_API_OWNERS and 332 'lgtm' in message['text'].lower()): 333 # Found an lgtm in a message from an owner. 334 lgtm_from_owner = True 335 break 336 337 if not lgtm_from_owner: 338 results.append( 339 output_api.PresubmitError( 340 "If this CL adds to or changes Skia's public API, you need an LGTM " 341 "from any of %s. If this CL only removes from or doesn't change " 342 "Skia's public API, please add a short note to the CL saying so " 343 "and add one of those reviewers on a TBR= line. If you don't know " 344 "if this CL affects Skia's public API, treat it like it does." 345 % str(PUBLIC_API_OWNERS))) 346 return results 347 348 349 def PostUploadHook(cl, change, output_api): 350 """git cl upload will call this hook after the issue is created/modified. 351 352 This hook does the following: 353 * Adds a link to the CL's Gold trybot results. 354 * Adds a link to preview docs changes if there are any docs changes in the CL. 355 * Adds 'NOTRY=true' if the CL contains only docs changes. 356 * Adds 'NOTREECHECKS=true' for non master branch changes since they do not 357 need to be gated on the master branch's tree. 358 * Adds 'NOTRY=true' for non master branch changes since trybots do not yet 359 work on them. 360 * Adds 'NOPRESUBMIT=true' for non master branch changes since those don't 361 run the presubmit checks. 362 * Adds extra trybots for the paths defined in PATH_TO_EXTRA_TRYBOTS. 363 """ 364 365 results = [] 366 atleast_one_docs_change = False 367 all_docs_changes = True 368 for affected_file in change.AffectedFiles(): 369 affected_file_path = affected_file.LocalPath() 370 file_path, _ = os.path.splitext(affected_file_path) 371 if 'site' == file_path.split(os.path.sep)[0]: 372 atleast_one_docs_change = True 373 else: 374 all_docs_changes = False 375 if atleast_one_docs_change and not all_docs_changes: 376 break 377 378 issue = cl.issue 379 rietveld_obj = cl.RpcServer() 380 if issue and rietveld_obj: 381 original_description = rietveld_obj.get_description(issue) 382 new_description = original_description 383 384 # Add GOLD_TRYBOT_URL if it does not exist yet. 385 if not re.search(r'^GOLD_TRYBOT_URL=', new_description, re.M | re.I): 386 new_description += '\nGOLD_TRYBOT_URL= %s%s' % (GOLD_TRYBOT_URL, issue) 387 results.append( 388 output_api.PresubmitNotifyResult( 389 'Added link to Gold trybot runs to the CL\'s description.\n' 390 'Note: Results may take sometime to be populated after trybots ' 391 'complete.')) 392 393 # If the change includes only doc changes then add NOTRY=true in the 394 # CL's description if it does not exist yet. 395 if all_docs_changes and not re.search( 396 r'^NOTRY=true$', new_description, re.M | re.I): 397 new_description += '\nNOTRY=true' 398 results.append( 399 output_api.PresubmitNotifyResult( 400 'This change has only doc changes. Automatically added ' 401 '\'NOTRY=true\' to the CL\'s description')) 402 403 # If there is atleast one docs change then add preview link in the CL's 404 # description if it does not already exist there. 405 if atleast_one_docs_change and not re.search( 406 r'^DOCS_PREVIEW=.*', new_description, re.M | re.I): 407 # Automatically add a link to where the docs can be previewed. 408 new_description += '\nDOCS_PREVIEW= %s%s' % (DOCS_PREVIEW_URL, issue) 409 results.append( 410 output_api.PresubmitNotifyResult( 411 'Automatically added a link to preview the docs changes to the ' 412 'CL\'s description')) 413 414 # If the target ref is not master then add NOTREECHECKS=true and NOTRY=true 415 # to the CL's description if it does not already exist there. 416 target_ref = rietveld_obj.get_issue_properties(issue, False).get( 417 'target_ref', '') 418 if target_ref != 'refs/heads/master': 419 if not re.search( 420 r'^NOTREECHECKS=true$', new_description, re.M | re.I): 421 new_description += "\nNOTREECHECKS=true" 422 results.append( 423 output_api.PresubmitNotifyResult( 424 'Branch changes do not need to rely on the master branch\'s ' 425 'tree status. Automatically added \'NOTREECHECKS=true\' to the ' 426 'CL\'s description')) 427 if not re.search( 428 r'^NOTRY=true$', new_description, re.M | re.I): 429 new_description += "\nNOTRY=true" 430 results.append( 431 output_api.PresubmitNotifyResult( 432 'Trybots do not yet work for non-master branches. ' 433 'Automatically added \'NOTRY=true\' to the CL\'s description')) 434 if not re.search( 435 r'^NOPRESUBMIT=true$', new_description, re.M | re.I): 436 new_description += "\nNOPRESUBMIT=true" 437 results.append( 438 output_api.PresubmitNotifyResult( 439 'Branch changes do not run the presubmit checks.')) 440 441 # Automatically set CQ_EXTRA_TRYBOTS if any of the changed files here begin 442 # with the paths of interest. 443 cq_master_to_trybots = collections.defaultdict(set) 444 for affected_file in change.AffectedFiles(): 445 affected_file_path = affected_file.LocalPath() 446 for path_prefix, extra_bots in PATH_PREFIX_TO_EXTRA_TRYBOTS.iteritems(): 447 if affected_file_path.startswith(path_prefix): 448 results.append( 449 output_api.PresubmitNotifyResult( 450 'Your CL modifies the path %s.\nAutomatically adding %s to ' 451 'the CL description.' % (affected_file_path, extra_bots))) 452 _MergeCQExtraTrybotsMaps( 453 cq_master_to_trybots, _GetCQExtraTrybotsMap(extra_bots)) 454 if cq_master_to_trybots: 455 new_description = _AddCQExtraTrybotsToDesc( 456 cq_master_to_trybots, new_description) 457 458 # If the description has changed update it. 459 if new_description != original_description: 460 rietveld_obj.update_description(issue, new_description) 461 462 return results 463 464 465 def _AddCQExtraTrybotsToDesc(cq_master_to_trybots, description): 466 """Adds the specified master and trybots to the CQ_EXTRA_TRYBOTS keyword. 467 468 If the keyword already exists in the description then it appends to it only 469 if the specified values do not already exist. 470 If the keyword does not exist then it creates a new section in the 471 description. 472 """ 473 match = re.search(r'^CQ_EXTRA_TRYBOTS=(.*)$', description, re.M | re.I) 474 if match: 475 original_trybots_map = _GetCQExtraTrybotsMap(match.group(1)) 476 _MergeCQExtraTrybotsMaps(cq_master_to_trybots, original_trybots_map) 477 new_description = description.replace( 478 match.group(0), _GetCQExtraTrybotsStr(cq_master_to_trybots)) 479 else: 480 new_description = description + "\n%s" % ( 481 _GetCQExtraTrybotsStr(cq_master_to_trybots)) 482 return new_description 483 484 485 def _MergeCQExtraTrybotsMaps(dest_map, map_to_be_consumed): 486 """Merges two maps of masters to trybots into one.""" 487 for master, trybots in map_to_be_consumed.iteritems(): 488 dest_map[master].update(trybots) 489 return dest_map 490 491 492 def _GetCQExtraTrybotsMap(cq_extra_trybots_str): 493 """Parses the CQ_EXTRA_TRYBOTS str and returns a map of masters to trybots.""" 494 cq_master_to_trybots = collections.defaultdict(set) 495 for section in cq_extra_trybots_str.split(';'): 496 if section: 497 master, bots = section.split(':') 498 cq_master_to_trybots[master].update(bots.split(',')) 499 return cq_master_to_trybots 500 501 502 def _GetCQExtraTrybotsStr(cq_master_to_trybots): 503 """Constructs the CQ_EXTRA_TRYBOTS str from a map of masters to trybots.""" 504 sections = [] 505 for master, trybots in cq_master_to_trybots.iteritems(): 506 sections.append('%s:%s' % (master, ','.join(trybots))) 507 return 'CQ_EXTRA_TRYBOTS=%s' % ';'.join(sections) 508 509 510 def CheckChangeOnCommit(input_api, output_api): 511 """Presubmit checks for the change on commit. 512 513 The following are the presubmit checks: 514 * Check change has one and only one EOL. 515 * Ensures that the Skia tree is open in 516 http://skia-tree-status.appspot.com/. Shows a warning if it is in 'Caution' 517 state and an error if it is in 'Closed' state. 518 """ 519 results = [] 520 results.extend(_CommonChecks(input_api, output_api)) 521 results.extend( 522 _CheckTreeStatus(input_api, output_api, json_url=( 523 SKIA_TREE_STATUS_URL + '/banner-status?format=json'))) 524 results.extend(_CheckLGTMsForPublicAPI(input_api, output_api)) 525 results.extend(_CheckOwnerIsInAuthorsFile(input_api, output_api)) 526 return results 527
