1 # Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. 2 # 3 # Use of this source code is governed by a BSD-style license 4 # that can be found in the LICENSE file in the root of the source 5 # tree. An additional intellectual property rights grant can be found 6 # in the file PATENTS. All contributing project authors may 7 # be found in the AUTHORS file in the root of the source tree. 8 9 import json 10 import os 11 import platform 12 import re 13 import subprocess 14 import sys 15 16 17 # Directories that will be scanned by cpplint by the presubmit script. 18 CPPLINT_DIRS = [ 19 'webrtc/audio', 20 'webrtc/call', 21 'webrtc/common_video', 22 'webrtc/examples', 23 'webrtc/modules/remote_bitrate_estimator', 24 'webrtc/modules/rtp_rtcp', 25 'webrtc/modules/video_coding', 26 'webrtc/modules/video_processing', 27 'webrtc/sound', 28 'webrtc/tools', 29 'webrtc/video', 30 ] 31 32 # List of directories of "supported" native APIs. That means changes to headers 33 # will be done in a compatible way following this scheme: 34 # 1. Non-breaking changes are made. 35 # 2. The old APIs as marked as deprecated (with comments). 36 # 3. Deprecation is announced to discuss-webrtc (at] googlegroups.com and 37 # webrtc-users (at] google.com (internal list). 38 # 4. (later) The deprecated APIs are removed. 39 # Directories marked as DEPRECATED should not be used. They're only present in 40 # the list to support legacy downstream code. 41 NATIVE_API_DIRS = ( 42 'talk/app/webrtc', 43 'webrtc', 44 'webrtc/base', # DEPRECATED. 45 'webrtc/common_audio/include', # DEPRECATED. 46 'webrtc/modules/audio_coding/include', 47 'webrtc/modules/audio_conference_mixer/include', # DEPRECATED. 48 'webrtc/modules/audio_device/include', 49 'webrtc/modules/audio_processing/include', 50 'webrtc/modules/bitrate_controller/include', 51 'webrtc/modules/include', 52 'webrtc/modules/remote_bitrate_estimator/include', 53 'webrtc/modules/rtp_rtcp/include', 54 'webrtc/modules/rtp_rtcp/source', # DEPRECATED. 55 'webrtc/modules/utility/include', 56 'webrtc/modules/video_coding/codecs/h264/include', 57 'webrtc/modules/video_coding/codecs/i420/include', 58 'webrtc/modules/video_coding/codecs/vp8/include', 59 'webrtc/modules/video_coding/codecs/vp9/include', 60 'webrtc/modules/video_coding/include', 61 'webrtc/system_wrappers/include', # DEPRECATED. 62 'webrtc/voice_engine/include', 63 ) 64 65 66 def _VerifyNativeApiHeadersListIsValid(input_api, output_api): 67 """Ensures the list of native API header directories is up to date.""" 68 non_existing_paths = [] 69 native_api_full_paths = [ 70 input_api.os_path.join(input_api.PresubmitLocalPath(), 71 *path.split('/')) for path in NATIVE_API_DIRS] 72 for path in native_api_full_paths: 73 if not os.path.isdir(path): 74 non_existing_paths.append(path) 75 if non_existing_paths: 76 return [output_api.PresubmitError( 77 'Directories to native API headers have changed which has made the ' 78 'list in PRESUBMIT.py outdated.\nPlease update it to the current ' 79 'location of our native APIs.', 80 non_existing_paths)] 81 return [] 82 83 84 def _CheckNativeApiHeaderChanges(input_api, output_api): 85 """Checks to remind proper changing of native APIs.""" 86 files = [] 87 for f in input_api.AffectedSourceFiles(input_api.FilterSourceFile): 88 if f.LocalPath().endswith('.h'): 89 for path in NATIVE_API_DIRS: 90 if os.path.dirname(f.LocalPath()) == path: 91 files.append(f) 92 93 if files: 94 return [output_api.PresubmitNotifyResult( 95 'You seem to be changing native API header files. Please make sure ' 96 'you:\n' 97 ' 1. Make compatible changes that don\'t break existing clients.\n' 98 ' 2. Mark the old APIs as deprecated.\n' 99 ' 3. Create a timeline and plan for when the deprecated method will ' 100 'be removed (preferably 3 months or so).\n' 101 ' 4. Update/inform existing downstream code owners to stop using the ' 102 'deprecated APIs: \n' 103 'send announcement to discuss-webrtc (at] googlegroups.com and ' 104 'webrtc-users (at] google.com.\n' 105 ' 5. (after ~3 months) remove the deprecated API.\n' 106 'Related files:', 107 files)] 108 return [] 109 110 111 def _CheckNoIOStreamInHeaders(input_api, output_api): 112 """Checks to make sure no .h files include <iostream>.""" 113 files = [] 114 pattern = input_api.re.compile(r'^#include\s*<iostream>', 115 input_api.re.MULTILINE) 116 for f in input_api.AffectedSourceFiles(input_api.FilterSourceFile): 117 if not f.LocalPath().endswith('.h'): 118 continue 119 contents = input_api.ReadFile(f) 120 if pattern.search(contents): 121 files.append(f) 122 123 if len(files): 124 return [output_api.PresubmitError( 125 'Do not #include <iostream> in header files, since it inserts static ' + 126 'initialization into every file including the header. Instead, ' + 127 '#include <ostream>. See http://crbug.com/94794', 128 files)] 129 return [] 130 131 132 def _CheckNoFRIEND_TEST(input_api, output_api): 133 """Make sure that gtest's FRIEND_TEST() macro is not used, the 134 FRIEND_TEST_ALL_PREFIXES() macro from testsupport/gtest_prod_util.h should be 135 used instead since that allows for FLAKY_, FAILS_ and DISABLED_ prefixes.""" 136 problems = [] 137 138 file_filter = lambda f: f.LocalPath().endswith(('.cc', '.h')) 139 for f in input_api.AffectedFiles(file_filter=file_filter): 140 for line_num, line in f.ChangedContents(): 141 if 'FRIEND_TEST(' in line: 142 problems.append(' %s:%d' % (f.LocalPath(), line_num)) 143 144 if not problems: 145 return [] 146 return [output_api.PresubmitPromptWarning('WebRTC\'s code should not use ' 147 'gtest\'s FRIEND_TEST() macro. Include testsupport/gtest_prod_util.h and ' 148 'use FRIEND_TEST_ALL_PREFIXES() instead.\n' + '\n'.join(problems))] 149 150 151 def _IsLintWhitelisted(whitelist_dirs, file_path): 152 """ Checks if a file is whitelisted for lint check.""" 153 for path in whitelist_dirs: 154 if os.path.dirname(file_path).startswith(path): 155 return True 156 return False 157 158 159 def _CheckApprovedFilesLintClean(input_api, output_api, 160 source_file_filter=None): 161 """Checks that all new or whitelisted .cc and .h files pass cpplint.py. 162 This check is based on _CheckChangeLintsClean in 163 depot_tools/presubmit_canned_checks.py but has less filters and only checks 164 added files.""" 165 result = [] 166 167 # Initialize cpplint. 168 import cpplint 169 # Access to a protected member _XX of a client class 170 # pylint: disable=W0212 171 cpplint._cpplint_state.ResetErrorCounts() 172 173 # Create a platform independent whitelist for the CPPLINT_DIRS. 174 whitelist_dirs = [input_api.os_path.join(*path.split('/')) 175 for path in CPPLINT_DIRS] 176 177 # Use the strictest verbosity level for cpplint.py (level 1) which is the 178 # default when running cpplint.py from command line. 179 # To make it possible to work with not-yet-converted code, we're only applying 180 # it to new (or moved/renamed) files and files listed in LINT_FOLDERS. 181 verbosity_level = 1 182 files = [] 183 for f in input_api.AffectedSourceFiles(source_file_filter): 184 # Note that moved/renamed files also count as added. 185 if f.Action() == 'A' or _IsLintWhitelisted(whitelist_dirs, f.LocalPath()): 186 files.append(f.AbsoluteLocalPath()) 187 188 for file_name in files: 189 cpplint.ProcessFile(file_name, verbosity_level) 190 191 if cpplint._cpplint_state.error_count > 0: 192 if input_api.is_committing: 193 # TODO(kjellander): Change back to PresubmitError below when we're 194 # confident with the lint settings. 195 res_type = output_api.PresubmitPromptWarning 196 else: 197 res_type = output_api.PresubmitPromptWarning 198 result = [res_type('Changelist failed cpplint.py check.')] 199 200 return result 201 202 def _CheckNoRtcBaseDeps(input_api, gyp_files, output_api): 203 pattern = input_api.re.compile(r"base.gyp:rtc_base\s*'") 204 violating_files = [] 205 for f in gyp_files: 206 gyp_exceptions = ( 207 'base_tests.gyp', 208 'desktop_capture.gypi', 209 'libjingle.gyp', 210 'libjingle_tests.gyp', 211 'p2p.gyp', 212 'sound.gyp', 213 'webrtc_test_common.gyp', 214 'webrtc_tests.gypi', 215 ) 216 if f.LocalPath().endswith(gyp_exceptions): 217 continue 218 contents = input_api.ReadFile(f) 219 if pattern.search(contents): 220 violating_files.append(f) 221 if violating_files: 222 return [output_api.PresubmitError( 223 'Depending on rtc_base is not allowed. Change your dependency to ' 224 'rtc_base_approved and possibly sanitize and move the desired source ' 225 'file(s) to rtc_base_approved.\nChanged GYP files:', 226 items=violating_files)] 227 return [] 228 229 def _CheckNoSourcesAboveGyp(input_api, gyp_files, output_api): 230 # Disallow referencing source files with paths above the GYP file location. 231 source_pattern = input_api.re.compile(r'sources.*?\[(.*?)\]', 232 re.MULTILINE | re.DOTALL) 233 file_pattern = input_api.re.compile(r"'((\.\./.*?)|(<\(webrtc_root\).*?))'") 234 violating_gyp_files = set() 235 violating_source_entries = [] 236 for gyp_file in gyp_files: 237 contents = input_api.ReadFile(gyp_file) 238 for source_block_match in source_pattern.finditer(contents): 239 # Find all source list entries starting with ../ in the source block 240 # (exclude overrides entries). 241 for file_list_match in file_pattern.finditer(source_block_match.group(0)): 242 source_file = file_list_match.group(0) 243 if 'overrides/' not in source_file: 244 violating_source_entries.append(source_file) 245 violating_gyp_files.add(gyp_file) 246 if violating_gyp_files: 247 return [output_api.PresubmitError( 248 'Referencing source files above the directory of the GYP file is not ' 249 'allowed. Please introduce new GYP targets and/or GYP files in the ' 250 'proper location instead.\n' 251 'Invalid source entries:\n' 252 '%s\n' 253 'Violating GYP files:' % '\n'.join(violating_source_entries), 254 items=violating_gyp_files)] 255 return [] 256 257 def _CheckGypChanges(input_api, output_api): 258 source_file_filter = lambda x: input_api.FilterSourceFile( 259 x, white_list=(r'.+\.(gyp|gypi)$',)) 260 261 gyp_files = [] 262 for f in input_api.AffectedSourceFiles(source_file_filter): 263 if f.LocalPath().startswith('webrtc'): 264 gyp_files.append(f) 265 266 result = [] 267 if gyp_files: 268 result.append(output_api.PresubmitNotifyResult( 269 'As you\'re changing GYP files: please make sure corresponding ' 270 'BUILD.gn files are also updated.\nChanged GYP files:', 271 items=gyp_files)) 272 result.extend(_CheckNoRtcBaseDeps(input_api, gyp_files, output_api)) 273 result.extend(_CheckNoSourcesAboveGyp(input_api, gyp_files, output_api)) 274 return result 275 276 def _CheckUnwantedDependencies(input_api, output_api): 277 """Runs checkdeps on #include statements added in this 278 change. Breaking - rules is an error, breaking ! rules is a 279 warning. 280 """ 281 # Copied from Chromium's src/PRESUBMIT.py. 282 283 # We need to wait until we have an input_api object and use this 284 # roundabout construct to import checkdeps because this file is 285 # eval-ed and thus doesn't have __file__. 286 original_sys_path = sys.path 287 try: 288 checkdeps_path = input_api.os_path.join(input_api.PresubmitLocalPath(), 289 'buildtools', 'checkdeps') 290 if not os.path.exists(checkdeps_path): 291 return [output_api.PresubmitError( 292 'Cannot find checkdeps at %s\nHave you run "gclient sync" to ' 293 'download Chromium and setup the symlinks?' % checkdeps_path)] 294 sys.path.append(checkdeps_path) 295 import checkdeps 296 from cpp_checker import CppChecker 297 from rules import Rule 298 finally: 299 # Restore sys.path to what it was before. 300 sys.path = original_sys_path 301 302 added_includes = [] 303 for f in input_api.AffectedFiles(): 304 if not CppChecker.IsCppFile(f.LocalPath()): 305 continue 306 307 changed_lines = [line for _, line in f.ChangedContents()] 308 added_includes.append([f.LocalPath(), changed_lines]) 309 310 deps_checker = checkdeps.DepsChecker(input_api.PresubmitLocalPath()) 311 312 error_descriptions = [] 313 warning_descriptions = [] 314 for path, rule_type, rule_description in deps_checker.CheckAddedCppIncludes( 315 added_includes): 316 description_with_path = '%s\n %s' % (path, rule_description) 317 if rule_type == Rule.DISALLOW: 318 error_descriptions.append(description_with_path) 319 else: 320 warning_descriptions.append(description_with_path) 321 322 results = [] 323 if error_descriptions: 324 results.append(output_api.PresubmitError( 325 'You added one or more #includes that violate checkdeps rules.', 326 error_descriptions)) 327 if warning_descriptions: 328 results.append(output_api.PresubmitPromptOrNotify( 329 'You added one or more #includes of files that are temporarily\n' 330 'allowed but being removed. Can you avoid introducing the\n' 331 '#include? See relevant DEPS file(s) for details and contacts.', 332 warning_descriptions)) 333 return results 334 335 336 def _RunPythonTests(input_api, output_api): 337 def join(*args): 338 return input_api.os_path.join(input_api.PresubmitLocalPath(), *args) 339 340 test_directories = [ 341 join('tools', 'autoroller', 'unittests'), 342 ] 343 344 tests = [] 345 for directory in test_directories: 346 tests.extend( 347 input_api.canned_checks.GetUnitTestsInDirectory( 348 input_api, 349 output_api, 350 directory, 351 whitelist=[r'.+_test\.py$'])) 352 return input_api.RunTests(tests, parallel=True) 353 354 355 def _CommonChecks(input_api, output_api): 356 """Checks common to both upload and commit.""" 357 results = [] 358 # Filter out files that are in objc or ios dirs from being cpplint-ed since 359 # they do not follow C++ lint rules. 360 black_list = input_api.DEFAULT_BLACK_LIST + ( 361 r".*\bobjc[\\\/].*", 362 ) 363 source_file_filter = lambda x: input_api.FilterSourceFile(x, None, black_list) 364 results.extend(_CheckApprovedFilesLintClean( 365 input_api, output_api, source_file_filter)) 366 results.extend(input_api.canned_checks.RunPylint(input_api, output_api, 367 black_list=(r'^.*gviz_api\.py$', 368 r'^.*gaeunit\.py$', 369 # Embedded shell-script fakes out pylint. 370 r'^build[\\\/].*\.py$', 371 r'^buildtools[\\\/].*\.py$', 372 r'^chromium[\\\/].*\.py$', 373 r'^google_apis[\\\/].*\.py$', 374 r'^net.*[\\\/].*\.py$', 375 r'^out.*[\\\/].*\.py$', 376 r'^testing[\\\/].*\.py$', 377 r'^third_party[\\\/].*\.py$', 378 r'^tools[\\\/]find_depot_tools.py$', 379 r'^tools[\\\/]clang[\\\/].*\.py$', 380 r'^tools[\\\/]generate_library_loader[\\\/].*\.py$', 381 r'^tools[\\\/]gn[\\\/].*\.py$', 382 r'^tools[\\\/]gyp[\\\/].*\.py$', 383 r'^tools[\\\/]isolate_driver.py$', 384 r'^tools[\\\/]protoc_wrapper[\\\/].*\.py$', 385 r'^tools[\\\/]python[\\\/].*\.py$', 386 r'^tools[\\\/]python_charts[\\\/]data[\\\/].*\.py$', 387 r'^tools[\\\/]refactoring[\\\/].*\.py$', 388 r'^tools[\\\/]swarming_client[\\\/].*\.py$', 389 r'^tools[\\\/]vim[\\\/].*\.py$', 390 # TODO(phoglund): should arguably be checked. 391 r'^tools[\\\/]valgrind-webrtc[\\\/].*\.py$', 392 r'^tools[\\\/]valgrind[\\\/].*\.py$', 393 r'^tools[\\\/]win[\\\/].*\.py$', 394 r'^xcodebuild.*[\\\/].*\.py$',), 395 disabled_warnings=['F0401', # Failed to import x 396 'E0611', # No package y in x 397 'W0232', # Class has no __init__ method 398 ], 399 pylintrc='pylintrc')) 400 # WebRTC can't use the presubmit_canned_checks.PanProjectChecks function since 401 # we need to have different license checks in talk/ and webrtc/ directories. 402 # Instead, hand-picked checks are included below. 403 404 # Skip long-lines check for DEPS, GN and GYP files. 405 long_lines_sources = lambda x: input_api.FilterSourceFile(x, 406 black_list=(r'.+\.gyp$', r'.+\.gypi$', r'.+\.gn$', r'.+\.gni$', 'DEPS')) 407 results.extend(input_api.canned_checks.CheckLongLines( 408 input_api, output_api, maxlen=80, source_file_filter=long_lines_sources)) 409 results.extend(input_api.canned_checks.CheckChangeHasNoTabs( 410 input_api, output_api)) 411 results.extend(input_api.canned_checks.CheckChangeHasNoStrayWhitespace( 412 input_api, output_api)) 413 results.extend(input_api.canned_checks.CheckChangeTodoHasOwner( 414 input_api, output_api)) 415 results.extend(_CheckNativeApiHeaderChanges(input_api, output_api)) 416 results.extend(_CheckNoIOStreamInHeaders(input_api, output_api)) 417 results.extend(_CheckNoFRIEND_TEST(input_api, output_api)) 418 results.extend(_CheckGypChanges(input_api, output_api)) 419 results.extend(_CheckUnwantedDependencies(input_api, output_api)) 420 results.extend(_RunPythonTests(input_api, output_api)) 421 return results 422 423 424 def CheckChangeOnUpload(input_api, output_api): 425 results = [] 426 results.extend(_CommonChecks(input_api, output_api)) 427 results.extend( 428 input_api.canned_checks.CheckGNFormatted(input_api, output_api)) 429 return results 430 431 432 def CheckChangeOnCommit(input_api, output_api): 433 results = [] 434 results.extend(_CommonChecks(input_api, output_api)) 435 results.extend(_VerifyNativeApiHeadersListIsValid(input_api, output_api)) 436 results.extend(input_api.canned_checks.CheckOwners(input_api, output_api)) 437 results.extend(input_api.canned_checks.CheckChangeWasUploaded( 438 input_api, output_api)) 439 results.extend(input_api.canned_checks.CheckChangeHasDescription( 440 input_api, output_api)) 441 results.extend(input_api.canned_checks.CheckChangeHasBugField( 442 input_api, output_api)) 443 results.extend(input_api.canned_checks.CheckChangeHasTestField( 444 input_api, output_api)) 445 results.extend(input_api.canned_checks.CheckTreeIsOpen( 446 input_api, output_api, 447 json_url='http://webrtc-status.appspot.com/current?format=json')) 448 return results 449 450 451 # pylint: disable=W0613 452 def GetPreferredTryMasters(project, change): 453 cq_config_path = os.path.join( 454 change.RepositoryRoot(), 'infra', 'config', 'cq.cfg') 455 # commit_queue.py below is a script in depot_tools directory, which has a 456 # 'builders' command to retrieve a list of CQ builders from the CQ config. 457 is_win = platform.system() == 'Windows' 458 masters = json.loads(subprocess.check_output( 459 ['commit_queue', 'builders', cq_config_path], shell=is_win)) 460 461 try_config = {} 462 for master in masters: 463 try_config.setdefault(master, {}) 464 for builder in masters[master]: 465 if 'presubmit' in builder: 466 # Do not trigger presubmit builders, since they're likely to fail 467 # (e.g. OWNERS checks before finished code review), and we're running 468 # local presubmit anyway. 469 pass 470 else: 471 try_config[master][builder] = ['defaulttests'] 472 473 return try_config 474
