1 #!/usr/bin/env python 2 # Copyright (c) 2012 The Chromium Authors. All rights reserved. 3 # Use of this source code is governed by a BSD-style license that can be 4 # found in the LICENSE file. 5 6 """Makes sure files have the right permissions. 7 8 Some developers have broken SCM configurations that flip the svn:executable 9 permission on for no good reason. Unix developers who run ls --color will then 10 see .cc files in green and get confused. 11 12 - For file extensions that must be executable, add it to EXECUTABLE_EXTENSIONS. 13 - For file extensions that must not be executable, add it to 14 NOT_EXECUTABLE_EXTENSIONS. 15 - To ignore all the files inside a directory, add it to IGNORED_PATHS. 16 - For file base name with ambiguous state and that should not be checked for 17 shebang, add it to IGNORED_FILENAMES. 18 19 Any file not matching the above will be opened and looked if it has a shebang. 20 It this doesn't match the executable bit on the file, the file will be flagged. 21 22 Note that all directory separators must be slashes (Unix-style) and not 23 backslashes. All directories should be relative to the source root and all 24 file paths should be only lowercase. 25 """ 26 27 import logging 28 import optparse 29 import os 30 import stat 31 import subprocess 32 import sys 33 34 #### USER EDITABLE SECTION STARTS HERE #### 35 36 # Files with these extensions must have executable bit set. 37 EXECUTABLE_EXTENSIONS = ( 38 'bat', 39 'dll', 40 'dylib', 41 'exe', 42 ) 43 44 # These files must have executable bit set. 45 EXECUTABLE_PATHS = ( 46 # TODO(maruel): Detect ELF files. 47 'chrome/installer/mac/sign_app.sh.in', 48 'chrome/installer/mac/sign_versioned_dir.sh.in', 49 ) 50 51 # These files must not have the executable bit set. This is mainly a performance 52 # optimization as these files are not checked for shebang. The list was 53 # partially generated from: 54 # git ls-files | grep "\\." | sed 's/.*\.//' | sort | uniq -c | sort -b -g 55 NON_EXECUTABLE_EXTENSIONS = ( 56 '1', 57 '3ds', 58 'S', 59 'am', 60 'applescript', 61 'asm', 62 'c', 63 'cc', 64 'cfg', 65 'chromium', 66 'cpp', 67 'crx', 68 'cs', 69 'css', 70 'cur', 71 'def', 72 'der', 73 'expected', 74 'gif', 75 'grd', 76 'gyp', 77 'gypi', 78 'h', 79 'hh', 80 'htm', 81 'html', 82 'hyph', 83 'ico', 84 'idl', 85 'java', 86 'jpg', 87 'js', 88 'json', 89 'm', 90 'm4', 91 'mm', 92 'mms', 93 'mock-http-headers', 94 'nmf', 95 'onc', 96 'pat', 97 'patch', 98 'pdf', 99 'pem', 100 'plist', 101 'png', 102 'proto', 103 'rc', 104 'rfx', 105 'rgs', 106 'rules', 107 'spec', 108 'sql', 109 'srpc', 110 'svg', 111 'tcl', 112 'test', 113 'tga', 114 'txt', 115 'vcproj', 116 'vsprops', 117 'webm', 118 'word', 119 'xib', 120 'xml', 121 'xtb', 122 'zip', 123 ) 124 125 # File names that are always whitelisted. (These are all autoconf spew.) 126 IGNORED_FILENAMES = ( 127 'config.guess', 128 'config.sub', 129 'configure', 130 'depcomp', 131 'install-sh', 132 'missing', 133 'mkinstalldirs', 134 'naclsdk', 135 'scons', 136 ) 137 138 # File paths starting with one of these will be ignored as well. 139 # Please consider fixing your file permissions, rather than adding to this list. 140 IGNORED_PATHS = ( 141 # TODO(maruel): Detect ELF files. 142 'chrome/test/data/extensions/uitest/plugins/plugin.plugin/contents/' 143 'macos/testnetscapeplugin', 144 'chrome/test/data/extensions/uitest/plugins_private/plugin.plugin/contents/' 145 'macos/testnetscapeplugin', 146 'chrome/installer/mac/sign_app.sh.in', 147 'chrome/installer/mac/sign_versioned_dir.sh.in', 148 'native_client_sdk/src/build_tools/sdk_tools/third_party/', 149 'out/', 150 # TODO(maruel): Fix these. 151 'third_party/android_testrunner/', 152 'third_party/bintrees/', 153 'third_party/closure_linter/', 154 'third_party/devscripts/licensecheck.pl.vanilla', 155 'third_party/hyphen/', 156 'third_party/jemalloc/', 157 'third_party/lcov-1.9/contrib/galaxy/conglomerate_functions.pl', 158 'third_party/lcov-1.9/contrib/galaxy/gen_makefile.sh', 159 'third_party/lcov/contrib/galaxy/conglomerate_functions.pl', 160 'third_party/lcov/contrib/galaxy/gen_makefile.sh', 161 'third_party/libevent/autogen.sh', 162 'third_party/libevent/test/test.sh', 163 'third_party/libxml/linux/xml2-config', 164 'third_party/libxml/src/ltmain.sh', 165 'third_party/mesa/', 166 'third_party/protobuf/', 167 'third_party/python_gflags/gflags.py', 168 'third_party/sqlite/', 169 'third_party/talloc/script/mksyms.sh', 170 'third_party/tcmalloc/', 171 'third_party/tlslite/setup.py', 172 ) 173 174 #### USER EDITABLE SECTION ENDS HERE #### 175 176 assert set(EXECUTABLE_EXTENSIONS) & set(NON_EXECUTABLE_EXTENSIONS) == set() 177 178 179 def capture(cmd, cwd): 180 """Returns the output of a command. 181 182 Ignores the error code or stderr. 183 """ 184 logging.debug('%s; cwd=%s' % (' '.join(cmd), cwd)) 185 env = os.environ.copy() 186 env['LANGUAGE'] = 'en_US.UTF-8' 187 p = subprocess.Popen( 188 cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, cwd=cwd, env=env) 189 return p.communicate()[0] 190 191 192 def get_svn_info(dir_path): 193 """Returns svn meta-data for a svn checkout.""" 194 if not os.path.isdir(dir_path): 195 return {} 196 out = capture(['svn', 'info', '.', '--non-interactive'], dir_path) 197 return dict(l.split(': ', 1) for l in out.splitlines() if l) 198 199 200 def get_svn_url(dir_path): 201 return get_svn_info(dir_path).get('URL') 202 203 204 def get_svn_root(dir_path): 205 """Returns the svn checkout root or None.""" 206 svn_url = get_svn_url(dir_path) 207 if not svn_url: 208 return None 209 logging.info('svn url: %s' % svn_url) 210 while True: 211 parent = os.path.dirname(dir_path) 212 if parent == dir_path: 213 return None 214 svn_url = svn_url.rsplit('/', 1)[0] 215 if svn_url != get_svn_url(parent): 216 return dir_path 217 dir_path = parent 218 219 220 def get_git_root(dir_path): 221 """Returns the git checkout root or None.""" 222 root = capture(['git', 'rev-parse', '--show-toplevel'], dir_path).strip() 223 if root: 224 return root 225 226 227 def is_ignored(rel_path): 228 """Returns True if rel_path is in our whitelist of files to ignore.""" 229 rel_path = rel_path.lower() 230 return ( 231 os.path.basename(rel_path) in IGNORED_FILENAMES or 232 rel_path.startswith(IGNORED_PATHS)) 233 234 235 def must_be_executable(rel_path): 236 """The file name represents a file type that must have the executable bit 237 set. 238 """ 239 return ( 240 os.path.splitext(rel_path)[1][1:].lower() in EXECUTABLE_EXTENSIONS or 241 rel_path in EXECUTABLE_PATHS) 242 243 244 def must_not_be_executable(rel_path): 245 """The file name represents a file type that must not have the executable 246 bit set. 247 """ 248 return os.path.splitext(rel_path)[1][1:].lower() in NON_EXECUTABLE_EXTENSIONS 249 250 251 def has_executable_bit(full_path): 252 """Returns if any executable bit is set.""" 253 permission = stat.S_IXUSR | stat.S_IXGRP | stat.S_IXOTH 254 return bool(permission & os.stat(full_path).st_mode) 255 256 257 def has_shebang(full_path): 258 """Returns if the file starts with #!/. 259 260 file_path is the absolute path to the file. 261 """ 262 with open(full_path, 'rb') as f: 263 return f.read(3) == '#!/' 264 265 def check_file(full_path, bare_output): 266 """Checks file_path's permissions and returns an error if it is 267 inconsistent. 268 269 It is assumed that the file is not ignored by is_ignored(). 270 271 If the file name is matched with must_be_executable() or 272 must_not_be_executable(), only its executable bit is checked. 273 Otherwise, the 3 first bytes of the file are read to verify if it has a 274 shebang and compares this with the executable bit on the file. 275 """ 276 try: 277 bit = has_executable_bit(full_path) 278 except OSError: 279 # It's faster to catch exception than call os.path.islink(). Chromium 280 # tree happens to have invalid symlinks under 281 # third_party/openssl/openssl/test/. 282 return None 283 284 if must_be_executable(full_path): 285 if not bit: 286 if bare_output: 287 return full_path 288 return '%s: Must have executable bit set' % full_path 289 return 290 if must_not_be_executable(full_path): 291 if bit: 292 if bare_output: 293 return full_path 294 return '%s: Must not have executable bit set' % full_path 295 return 296 297 # For the others, it depends on the shebang. 298 shebang = has_shebang(full_path) 299 if bit != shebang: 300 if bare_output: 301 return full_path 302 if bit: 303 return '%s: Has executable bit but not shebang' % full_path 304 else: 305 return '%s: Has shebang but not executable bit' % full_path 306 307 308 def check_files(root, files, bare_output): 309 errors = [] 310 for file_path in files: 311 if is_ignored(file_path): 312 continue 313 314 full_file_path = os.path.join(root, file_path) 315 316 error = check_file(full_file_path, bare_output) 317 if error: 318 errors.append(error) 319 320 return errors 321 322 class ApiBase(object): 323 def __init__(self, root_dir, bare_output): 324 self.root_dir = root_dir 325 self.bare_output = bare_output 326 self.count = 0 327 self.count_shebang = 0 328 329 def check_file(self, rel_path): 330 logging.debug('check_file(%s)' % rel_path) 331 self.count += 1 332 333 if (not must_be_executable(rel_path) and 334 not must_not_be_executable(rel_path)): 335 self.count_shebang += 1 336 337 full_path = os.path.join(self.root_dir, rel_path) 338 return check_file(full_path, self.bare_output) 339 340 def check_dir(self, rel_path): 341 return self.check(rel_path) 342 343 def check(self, start_dir): 344 """Check the files in start_dir, recursively check its subdirectories.""" 345 errors = [] 346 items = self.list_dir(start_dir) 347 logging.info('check(%s) -> %d' % (start_dir, len(items))) 348 for item in items: 349 full_path = os.path.join(self.root_dir, start_dir, item) 350 rel_path = full_path[len(self.root_dir) + 1:] 351 if is_ignored(rel_path): 352 continue 353 if os.path.isdir(full_path): 354 # Depth first. 355 errors.extend(self.check_dir(rel_path)) 356 else: 357 error = self.check_file(rel_path) 358 if error: 359 errors.append(error) 360 return errors 361 362 def list_dir(self, start_dir): 363 """Lists all the files and directory inside start_dir.""" 364 return sorted( 365 x for x in os.listdir(os.path.join(self.root_dir, start_dir)) 366 if not x.startswith('.') 367 ) 368 369 370 class ApiSvnQuick(ApiBase): 371 """Returns all files in svn-versioned directories, independent of the fact if 372 they are versionned. 373 374 Uses svn info in each directory to determine which directories should be 375 crawled. 376 """ 377 def __init__(self, *args): 378 super(ApiSvnQuick, self).__init__(*args) 379 self.url = get_svn_url(self.root_dir) 380 381 def check_dir(self, rel_path): 382 url = self.url + '/' + rel_path 383 if get_svn_url(os.path.join(self.root_dir, rel_path)) != url: 384 return [] 385 return super(ApiSvnQuick, self).check_dir(rel_path) 386 387 388 class ApiAllFilesAtOnceBase(ApiBase): 389 _files = None 390 391 def list_dir(self, start_dir): 392 """Lists all the files and directory inside start_dir.""" 393 if self._files is None: 394 self._files = sorted(self._get_all_files()) 395 if not self.bare_output: 396 print 'Found %s files' % len(self._files) 397 start_dir = start_dir[len(self.root_dir) + 1:] 398 return [ 399 x[len(start_dir):] for x in self._files if x.startswith(start_dir) 400 ] 401 402 def _get_all_files(self): 403 """Lists all the files and directory inside self._root_dir.""" 404 raise NotImplementedError() 405 406 407 class ApiSvn(ApiAllFilesAtOnceBase): 408 """Returns all the subversion controlled files. 409 410 Warning: svn ls is abnormally slow. 411 """ 412 def _get_all_files(self): 413 cmd = ['svn', 'ls', '--non-interactive', '--recursive'] 414 return ( 415 x for x in capture(cmd, self.root_dir).splitlines() 416 if not x.endswith(os.path.sep)) 417 418 419 class ApiGit(ApiAllFilesAtOnceBase): 420 def _get_all_files(self): 421 return capture(['git', 'ls-files'], cwd=self.root_dir).splitlines() 422 423 424 def get_scm(dir_path, bare): 425 """Returns a properly configured ApiBase instance.""" 426 cwd = os.getcwd() 427 root = get_svn_root(dir_path or cwd) 428 if root: 429 if not bare: 430 print('Found subversion checkout at %s' % root) 431 return ApiSvnQuick(dir_path or root, bare) 432 root = get_git_root(dir_path or cwd) 433 if root: 434 if not bare: 435 print('Found git repository at %s' % root) 436 return ApiGit(dir_path or root, bare) 437 438 # Returns a non-scm aware checker. 439 if not bare: 440 print('Failed to determine the SCM for %s' % dir_path) 441 return ApiBase(dir_path or cwd, bare) 442 443 444 def main(): 445 usage = """Usage: python %prog [--root <root>] [tocheck] 446 tocheck Specifies the directory, relative to root, to check. This defaults 447 to "." so it checks everything. 448 449 Examples: 450 python %prog 451 python %prog --root /path/to/source chrome""" 452 453 parser = optparse.OptionParser(usage=usage) 454 parser.add_option( 455 '--root', 456 help='Specifies the repository root. This defaults ' 457 'to the checkout repository root') 458 parser.add_option( 459 '-v', '--verbose', action='count', default=0, help='Print debug logging') 460 parser.add_option( 461 '--bare', 462 action='store_true', 463 default=False, 464 help='Prints the bare filename triggering the checks') 465 parser.add_option( 466 '--file', action='append', dest='files', 467 help='Specifics a list of files to check the permissions of. Only these ' 468 'files will be checked') 469 options, args = parser.parse_args() 470 471 levels = [logging.ERROR, logging.INFO, logging.DEBUG] 472 logging.basicConfig(level=levels[min(len(levels) - 1, options.verbose)]) 473 474 if len(args) > 1: 475 parser.error('Too many arguments used') 476 477 if options.root: 478 options.root = os.path.abspath(options.root) 479 480 if options.files: 481 errors = check_files(options.root, options.files, options.bare) 482 print '\n'.join(errors) 483 return bool(errors) 484 485 api = get_scm(options.root, options.bare) 486 if args: 487 start_dir = args[0] 488 else: 489 start_dir = api.root_dir 490 491 errors = api.check(start_dir) 492 493 if not options.bare: 494 print 'Processed %s files, %d files where tested for shebang' % ( 495 api.count, api.count_shebang) 496 497 if errors: 498 if not options.bare: 499 print '\nFAILED\n' 500 print '\n'.join(errors) 501 return 1 502 if not options.bare: 503 print '\nSUCCESS\n' 504 return 0 505 506 507 if '__main__' == __name__: 508 sys.exit(main()) 509
