1 #!/usr/bin/env python 2 # Copyright (c) 2012 The Chromium Authors. All rights reserved. 3 # Use of this source code is governed by a BSD-style license that can be 4 # found in the LICENSE file. 5 6 """Makes sure files have the right permissions. 7 8 Some developers have broken SCM configurations that flip the svn:executable 9 permission on for no good reason. Unix developers who run ls --color will then 10 see .cc files in green and get confused. 11 12 - For file extensions that must be executable, add it to EXECUTABLE_EXTENSIONS. 13 - For file extensions that must not be executable, add it to 14 NOT_EXECUTABLE_EXTENSIONS. 15 - To ignore all the files inside a directory, add it to IGNORED_PATHS. 16 - For file base name with ambiguous state and that should not be checked for 17 shebang, add it to IGNORED_FILENAMES. 18 19 Any file not matching the above will be opened and looked if it has a shebang 20 or an ELF header. If this does not match the executable bit on the file, the 21 file will be flagged. 22 23 Note that all directory separators must be slashes (Unix-style) and not 24 backslashes. All directories should be relative to the source root and all 25 file paths should be only lowercase. 26 """ 27 28 import json 29 import logging 30 import optparse 31 import os 32 import stat 33 import string 34 import subprocess 35 import sys 36 37 #### USER EDITABLE SECTION STARTS HERE #### 38 39 # Files with these extensions must have executable bit set. 40 # 41 # Case-sensitive. 42 EXECUTABLE_EXTENSIONS = ( 43 'bat', 44 'dll', 45 'dylib', 46 'exe', 47 ) 48 49 # These files must have executable bit set. 50 # 51 # Case-insensitive, lower-case only. 52 EXECUTABLE_PATHS = ( 53 'chrome/test/data/app_shim/app_shim_32_bit.app/contents/' 54 'macos/app_mode_loader', 55 'chrome/test/data/extensions/uitest/plugins/plugin.plugin/contents/' 56 'macos/testnetscapeplugin', 57 'chrome/test/data/extensions/uitest/plugins_private/plugin.plugin/contents/' 58 'macos/testnetscapeplugin', 59 ) 60 61 # These files must not have the executable bit set. This is mainly a performance 62 # optimization as these files are not checked for shebang. The list was 63 # partially generated from: 64 # git ls-files | grep "\\." | sed 's/.*\.//' | sort | uniq -c | sort -b -g 65 # 66 # Case-sensitive. 67 NON_EXECUTABLE_EXTENSIONS = ( 68 '1', 69 '3ds', 70 'S', 71 'am', 72 'applescript', 73 'asm', 74 'c', 75 'cc', 76 'cfg', 77 'chromium', 78 'cpp', 79 'crx', 80 'cs', 81 'css', 82 'cur', 83 'def', 84 'der', 85 'expected', 86 'gif', 87 'grd', 88 'gyp', 89 'gypi', 90 'h', 91 'hh', 92 'htm', 93 'html', 94 'hyph', 95 'ico', 96 'idl', 97 'java', 98 'jpg', 99 'js', 100 'json', 101 'm', 102 'm4', 103 'mm', 104 'mms', 105 'mock-http-headers', 106 'nexe', 107 'nmf', 108 'onc', 109 'pat', 110 'patch', 111 'pdf', 112 'pem', 113 'plist', 114 'png', 115 'proto', 116 'rc', 117 'rfx', 118 'rgs', 119 'rules', 120 'spec', 121 'sql', 122 'srpc', 123 'svg', 124 'tcl', 125 'test', 126 'tga', 127 'txt', 128 'vcproj', 129 'vsprops', 130 'webm', 131 'word', 132 'xib', 133 'xml', 134 'xtb', 135 'zip', 136 ) 137 138 # These files must not have executable bit set. 139 # 140 # Case-insensitive, lower-case only. 141 NON_EXECUTABLE_PATHS = ( 142 'build/android/tests/symbolize/liba.so', 143 'build/android/tests/symbolize/libb.so', 144 'chrome/installer/mac/sign_app.sh.in', 145 'chrome/installer/mac/sign_versioned_dir.sh.in', 146 'chrome/test/data/components/ihfokbkgjpifnbbojhneepfflplebdkc/' 147 'ihfokbkgjpifnbbojhneepfflplebdkc_1/a_changing_binary_file', 148 'chrome/test/data/components/ihfokbkgjpifnbbojhneepfflplebdkc/' 149 'ihfokbkgjpifnbbojhneepfflplebdkc_2/a_changing_binary_file', 150 'chrome/test/data/extensions/uitest/plugins/plugin32.so', 151 'chrome/test/data/extensions/uitest/plugins/plugin64.so', 152 'chrome/test/data/extensions/uitest/plugins_private/plugin32.so', 153 'chrome/test/data/extensions/uitest/plugins_private/plugin64.so', 154 'courgette/testdata/elf-32-1', 155 'courgette/testdata/elf-32-2', 156 'courgette/testdata/elf-64', 157 ) 158 159 # File names that are always whitelisted. (These are mostly autoconf spew.) 160 # 161 # Case-sensitive. 162 IGNORED_FILENAMES = ( 163 'config.guess', 164 'config.sub', 165 'configure', 166 'depcomp', 167 'install-sh', 168 'missing', 169 'mkinstalldirs', 170 'naclsdk', 171 'scons', 172 ) 173 174 # File paths starting with one of these will be ignored as well. 175 # Please consider fixing your file permissions, rather than adding to this list. 176 # 177 # Case-insensitive, lower-case only. 178 IGNORED_PATHS = ( 179 'native_client_sdk/src/build_tools/sdk_tools/third_party/fancy_urllib/' 180 '__init__.py', 181 'out/', 182 # TODO(maruel): Fix these. 183 'third_party/android_testrunner/', 184 'third_party/bintrees/', 185 'third_party/closure_linter/', 186 'third_party/devscripts/licensecheck.pl.vanilla', 187 'third_party/hyphen/', 188 'third_party/jemalloc/', 189 'third_party/lcov-1.9/contrib/galaxy/conglomerate_functions.pl', 190 'third_party/lcov-1.9/contrib/galaxy/gen_makefile.sh', 191 'third_party/lcov/contrib/galaxy/conglomerate_functions.pl', 192 'third_party/lcov/contrib/galaxy/gen_makefile.sh', 193 'third_party/libevent/autogen.sh', 194 'third_party/libevent/test/test.sh', 195 'third_party/libxml/linux/xml2-config', 196 'third_party/libxml/src/ltmain.sh', 197 'third_party/mesa/', 198 'third_party/protobuf/', 199 'third_party/python_gflags/gflags.py', 200 'third_party/sqlite/', 201 'third_party/talloc/script/mksyms.sh', 202 'third_party/tcmalloc/', 203 'third_party/tlslite/setup.py', 204 ) 205 206 #### USER EDITABLE SECTION ENDS HERE #### 207 208 assert set(EXECUTABLE_EXTENSIONS) & set(NON_EXECUTABLE_EXTENSIONS) == set() 209 assert set(EXECUTABLE_PATHS) & set(NON_EXECUTABLE_PATHS) == set() 210 211 VALID_CHARS = set(string.ascii_lowercase + string.digits + '/-_.') 212 for paths in (EXECUTABLE_PATHS, NON_EXECUTABLE_PATHS, IGNORED_PATHS): 213 assert all([set(path).issubset(VALID_CHARS) for path in paths]) 214 215 216 def capture(cmd, cwd): 217 """Returns the output of a command. 218 219 Ignores the error code or stderr. 220 """ 221 logging.debug('%s; cwd=%s' % (' '.join(cmd), cwd)) 222 env = os.environ.copy() 223 env['LANGUAGE'] = 'en_US.UTF-8' 224 p = subprocess.Popen( 225 cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, cwd=cwd, env=env) 226 return p.communicate()[0] 227 228 229 def get_svn_info(dir_path): 230 """Returns svn meta-data for a svn checkout.""" 231 if not os.path.isdir(dir_path): 232 return {} 233 out = capture(['svn', 'info', '.', '--non-interactive'], dir_path) 234 return dict(l.split(': ', 1) for l in out.splitlines() if l) 235 236 237 def get_svn_url(dir_path): 238 return get_svn_info(dir_path).get('URL') 239 240 241 def get_svn_root(dir_path): 242 """Returns the svn checkout root or None.""" 243 svn_url = get_svn_url(dir_path) 244 if not svn_url: 245 return None 246 logging.info('svn url: %s' % svn_url) 247 while True: 248 parent = os.path.dirname(dir_path) 249 if parent == dir_path: 250 return None 251 svn_url = svn_url.rsplit('/', 1)[0] 252 if svn_url != get_svn_url(parent): 253 return dir_path 254 dir_path = parent 255 256 257 def get_git_root(dir_path): 258 """Returns the git checkout root or None.""" 259 root = capture(['git', 'rev-parse', '--show-toplevel'], dir_path).strip() 260 if root: 261 return root 262 263 264 def is_ignored(rel_path): 265 """Returns True if rel_path is in our whitelist of files to ignore.""" 266 rel_path = rel_path.lower() 267 return ( 268 os.path.basename(rel_path) in IGNORED_FILENAMES or 269 rel_path.lower().startswith(IGNORED_PATHS)) 270 271 272 def must_be_executable(rel_path): 273 """The file name represents a file type that must have the executable bit 274 set. 275 """ 276 return (os.path.splitext(rel_path)[1][1:] in EXECUTABLE_EXTENSIONS or 277 rel_path.lower() in EXECUTABLE_PATHS) 278 279 280 def must_not_be_executable(rel_path): 281 """The file name represents a file type that must not have the executable 282 bit set. 283 """ 284 return (os.path.splitext(rel_path)[1][1:] in NON_EXECUTABLE_EXTENSIONS or 285 rel_path.lower() in NON_EXECUTABLE_PATHS) 286 287 288 def has_executable_bit(full_path): 289 """Returns if any executable bit is set.""" 290 permission = stat.S_IXUSR | stat.S_IXGRP | stat.S_IXOTH 291 return bool(permission & os.stat(full_path).st_mode) 292 293 294 def has_shebang_or_is_elf(full_path): 295 """Returns if the file starts with #!/ or is an ELF binary. 296 297 full_path is the absolute path to the file. 298 """ 299 with open(full_path, 'rb') as f: 300 data = f.read(4) 301 return (data[:3] == '#!/', data == '\x7fELF') 302 303 304 def check_file(root_path, rel_path): 305 """Checks the permissions of the file whose path is root_path + rel_path and 306 returns an error if it is inconsistent. Returns None on success. 307 308 It is assumed that the file is not ignored by is_ignored(). 309 310 If the file name is matched with must_be_executable() or 311 must_not_be_executable(), only its executable bit is checked. 312 Otherwise, the first few bytes of the file are read to verify if it has a 313 shebang or ELF header and compares this with the executable bit on the file. 314 """ 315 full_path = os.path.join(root_path, rel_path) 316 def result_dict(error): 317 return { 318 'error': error, 319 'full_path': full_path, 320 'rel_path': rel_path, 321 } 322 try: 323 bit = has_executable_bit(full_path) 324 except OSError: 325 # It's faster to catch exception than call os.path.islink(). Chromium 326 # tree happens to have invalid symlinks under 327 # third_party/openssl/openssl/test/. 328 return None 329 330 if must_be_executable(rel_path): 331 if not bit: 332 return result_dict('Must have executable bit set') 333 return 334 if must_not_be_executable(rel_path): 335 if bit: 336 return result_dict('Must not have executable bit set') 337 return 338 339 # For the others, it depends on the file header. 340 (shebang, elf) = has_shebang_or_is_elf(full_path) 341 if bit != (shebang or elf): 342 if bit: 343 return result_dict('Has executable bit but not shebang or ELF header') 344 if shebang: 345 return result_dict('Has shebang but not executable bit') 346 return result_dict('Has ELF header but not executable bit') 347 348 349 def check_files(root, files): 350 gen = (check_file(root, f) for f in files if not is_ignored(f)) 351 return filter(None, gen) 352 353 354 class ApiBase(object): 355 def __init__(self, root_dir, bare_output): 356 self.root_dir = root_dir 357 self.bare_output = bare_output 358 self.count = 0 359 self.count_read_header = 0 360 361 def check_file(self, rel_path): 362 logging.debug('check_file(%s)' % rel_path) 363 self.count += 1 364 365 if (not must_be_executable(rel_path) and 366 not must_not_be_executable(rel_path)): 367 self.count_read_header += 1 368 369 return check_file(self.root_dir, rel_path) 370 371 def check_dir(self, rel_path): 372 return self.check(rel_path) 373 374 def check(self, start_dir): 375 """Check the files in start_dir, recursively check its subdirectories.""" 376 errors = [] 377 items = self.list_dir(start_dir) 378 logging.info('check(%s) -> %d' % (start_dir, len(items))) 379 for item in items: 380 full_path = os.path.join(self.root_dir, start_dir, item) 381 rel_path = full_path[len(self.root_dir) + 1:] 382 if is_ignored(rel_path): 383 continue 384 if os.path.isdir(full_path): 385 # Depth first. 386 errors.extend(self.check_dir(rel_path)) 387 else: 388 error = self.check_file(rel_path) 389 if error: 390 errors.append(error) 391 return errors 392 393 def list_dir(self, start_dir): 394 """Lists all the files and directory inside start_dir.""" 395 return sorted( 396 x for x in os.listdir(os.path.join(self.root_dir, start_dir)) 397 if not x.startswith('.') 398 ) 399 400 401 class ApiSvnQuick(ApiBase): 402 """Returns all files in svn-versioned directories, independent of the fact if 403 they are versionned. 404 405 Uses svn info in each directory to determine which directories should be 406 crawled. 407 """ 408 def __init__(self, *args): 409 super(ApiSvnQuick, self).__init__(*args) 410 self.url = get_svn_url(self.root_dir) 411 412 def check_dir(self, rel_path): 413 url = self.url + '/' + rel_path 414 if get_svn_url(os.path.join(self.root_dir, rel_path)) != url: 415 return [] 416 return super(ApiSvnQuick, self).check_dir(rel_path) 417 418 419 class ApiAllFilesAtOnceBase(ApiBase): 420 _files = None 421 422 def list_dir(self, start_dir): 423 """Lists all the files and directory inside start_dir.""" 424 if self._files is None: 425 self._files = sorted(self._get_all_files()) 426 if not self.bare_output: 427 print 'Found %s files' % len(self._files) 428 start_dir = start_dir[len(self.root_dir) + 1:] 429 return [ 430 x[len(start_dir):] for x in self._files if x.startswith(start_dir) 431 ] 432 433 def _get_all_files(self): 434 """Lists all the files and directory inside self._root_dir.""" 435 raise NotImplementedError() 436 437 438 class ApiSvn(ApiAllFilesAtOnceBase): 439 """Returns all the subversion controlled files. 440 441 Warning: svn ls is abnormally slow. 442 """ 443 def _get_all_files(self): 444 cmd = ['svn', 'ls', '--non-interactive', '--recursive'] 445 return ( 446 x for x in capture(cmd, self.root_dir).splitlines() 447 if not x.endswith(os.path.sep)) 448 449 450 class ApiGit(ApiAllFilesAtOnceBase): 451 def _get_all_files(self): 452 return capture(['git', 'ls-files'], cwd=self.root_dir).splitlines() 453 454 455 def get_scm(dir_path, bare): 456 """Returns a properly configured ApiBase instance.""" 457 cwd = os.getcwd() 458 root = get_svn_root(dir_path or cwd) 459 if root: 460 if not bare: 461 print('Found subversion checkout at %s' % root) 462 return ApiSvnQuick(dir_path or root, bare) 463 root = get_git_root(dir_path or cwd) 464 if root: 465 if not bare: 466 print('Found git repository at %s' % root) 467 return ApiGit(dir_path or root, bare) 468 469 # Returns a non-scm aware checker. 470 if not bare: 471 print('Failed to determine the SCM for %s' % dir_path) 472 return ApiBase(dir_path or cwd, bare) 473 474 475 def main(): 476 usage = """Usage: python %prog [--root <root>] [tocheck] 477 tocheck Specifies the directory, relative to root, to check. This defaults 478 to "." so it checks everything. 479 480 Examples: 481 python %prog 482 python %prog --root /path/to/source chrome""" 483 484 parser = optparse.OptionParser(usage=usage) 485 parser.add_option( 486 '--root', 487 help='Specifies the repository root. This defaults ' 488 'to the checkout repository root') 489 parser.add_option( 490 '-v', '--verbose', action='count', default=0, help='Print debug logging') 491 parser.add_option( 492 '--bare', 493 action='store_true', 494 default=False, 495 help='Prints the bare filename triggering the checks') 496 parser.add_option( 497 '--file', action='append', dest='files', 498 help='Specifics a list of files to check the permissions of. Only these ' 499 'files will be checked') 500 parser.add_option('--json', help='Path to JSON output file') 501 options, args = parser.parse_args() 502 503 levels = [logging.ERROR, logging.INFO, logging.DEBUG] 504 logging.basicConfig(level=levels[min(len(levels) - 1, options.verbose)]) 505 506 if len(args) > 1: 507 parser.error('Too many arguments used') 508 509 if options.root: 510 options.root = os.path.abspath(options.root) 511 512 if options.files: 513 # --file implies --bare (for PRESUBMIT.py). 514 options.bare = True 515 516 errors = check_files(options.root, options.files) 517 else: 518 api = get_scm(options.root, options.bare) 519 start_dir = args[0] if args else api.root_dir 520 errors = api.check(start_dir) 521 522 if not options.bare: 523 print('Processed %s files, %d files where tested for shebang/ELF ' 524 'header' % (api.count, api.count_read_header)) 525 526 if options.json: 527 with open(options.json, 'w') as f: 528 json.dump(errors, f) 529 530 if errors: 531 if options.bare: 532 print '\n'.join(e['full_path'] for e in errors) 533 else: 534 print '\nFAILED\n' 535 print '\n'.join('%s: %s' % (e['full_path'], e['error']) for e in errors) 536 return 1 537 if not options.bare: 538 print '\nSUCCESS\n' 539 return 0 540 541 542 if '__main__' == __name__: 543 sys.exit(main()) 544