1 #!/usr/bin/python -u 2 """ 3 Wrapper to patch pylint library functions to suit autotest. 4 5 This script is invoked as part of the presubmit checks for autotest python 6 files. It runs pylint on a list of files that it obtains either through 7 the command line or from an environment variable set in pre-upload.py. 8 9 Example: 10 run_pylint.py filename.py 11 """ 12 13 import fnmatch 14 import logging 15 import os 16 import re 17 import sys 18 19 import common 20 from autotest_lib.client.common_lib import autotemp, revision_control 21 22 # Do a basic check to see if pylint is even installed. 23 try: 24 import pylint 25 from pylint.__pkginfo__ import version as pylint_version 26 except ImportError: 27 print ("Unable to import pylint, it may need to be installed." 28 " Run 'sudo aptitude install pylint' if you haven't already.") 29 sys.exit(1) 30 31 major, minor, release = pylint_version.split('.') 32 pylint_version = float("%s.%s" % (major, minor)) 33 34 # some files make pylint blow up, so make sure we ignore them 35 BLACKLIST = ['/site-packages/*', '/contrib/*', '/frontend/afe/management.py'] 36 37 # patch up the logilab module lookup tools to understand autotest_lib.* trash 38 import logilab.common.modutils 39 _ffm = logilab.common.modutils.file_from_modpath 40 def file_from_modpath(modpath, path=None, context_file=None): 41 """ 42 Wrapper to eliminate autotest_lib from modpath. 43 44 @param modpath: name of module splitted on '.' 45 @param path: optional list of paths where module should be searched for. 46 @param context_file: path to file doing the importing. 47 @return The path to the module as returned by the parent method invocation. 48 @raises: ImportError if these is no such module. 49 """ 50 if modpath[0] == "autotest_lib": 51 return _ffm(modpath[1:], path, context_file) 52 else: 53 return _ffm(modpath, path, context_file) 54 logilab.common.modutils.file_from_modpath = file_from_modpath 55 56 57 import pylint.lint 58 from pylint.checkers import base, imports, variables 59 60 # need to put autotest root dir on sys.path so pylint will be happy 61 autotest_root = os.path.abspath(os.path.join(os.path.dirname(__file__), '..')) 62 sys.path.insert(0, autotest_root) 63 64 # patch up pylint import checker to handle our importing magic 65 ROOT_MODULE = 'autotest_lib.' 66 67 # A list of modules for pylint to ignore, specifically, these modules 68 # are imported for their side-effects and are not meant to be used. 69 _IGNORE_MODULES=['common', 'frontend_test_utils', 70 'setup_django_environment', 71 'setup_django_lite_environment', 72 'setup_django_readonly_environment', 'setup_test_environment',] 73 74 75 class pylint_error(Exception): 76 """ 77 Error raised when pylint complains about a file. 78 """ 79 80 81 class run_pylint_error(pylint_error): 82 """ 83 Error raised when an assumption made in this file is violated. 84 """ 85 86 87 def patch_modname(modname): 88 """ 89 Patches modname so we can make sense of autotest_lib modules. 90 91 @param modname: name of a module, contains '.' 92 @return modified modname string. 93 """ 94 if modname.startswith(ROOT_MODULE) or modname.startswith(ROOT_MODULE[:-1]): 95 modname = modname[len(ROOT_MODULE):] 96 return modname 97 98 99 def patch_consumed_list(to_consume=None, consumed=None): 100 """ 101 Patches the consumed modules list to ignore modules with side effects. 102 103 Autotest relies on importing certain modules solely for their side 104 effects. Pylint doesn't understand this and flags them as unused, since 105 they're not referenced anywhere in the code. To overcome this we need 106 to transplant said modules into the dictionary of modules pylint has 107 already seen, before pylint checks it. 108 109 @param to_consume: a dictionary of names pylint needs to see referenced. 110 @param consumed: a dictionary of names that pylint has seen referenced. 111 """ 112 ignore_modules = [] 113 if (to_consume is not None and consumed is not None): 114 ignore_modules = [module_name for module_name in _IGNORE_MODULES 115 if module_name in to_consume] 116 117 for module_name in ignore_modules: 118 consumed[module_name] = to_consume[module_name] 119 del to_consume[module_name] 120 121 122 class CustomImportsChecker(imports.ImportsChecker): 123 """Modifies stock imports checker to suit autotest.""" 124 def visit_from(self, node): 125 """Patches modnames so pylints understands autotest_lib.""" 126 node.modname = patch_modname(node.modname) 127 return super(CustomImportsChecker, self).visit_from(node) 128 129 130 class CustomVariablesChecker(variables.VariablesChecker): 131 """Modifies stock variables checker to suit autotest.""" 132 133 def visit_module(self, node): 134 """ 135 Unflag 'import common'. 136 137 _to_consume eg: [({to reference}, {referenced}, 'scope type')] 138 Enteries are appended to this list as we drill deeper in scope. 139 If we ever come across a module to ignore, we immediately move it 140 to the consumed list. 141 142 @param node: node of the ast we're currently checking. 143 """ 144 super(CustomVariablesChecker, self).visit_module(node) 145 scoped_names = self._to_consume.pop() 146 patch_consumed_list(scoped_names[0],scoped_names[1]) 147 self._to_consume.append(scoped_names) 148 149 def visit_from(self, node): 150 """Patches modnames so pylints understands autotest_lib.""" 151 node.modname = patch_modname(node.modname) 152 return super(CustomVariablesChecker, self).visit_from(node) 153 154 155 class CustomDocStringChecker(base.DocStringChecker): 156 """Modifies stock docstring checker to suit Autotest doxygen style.""" 157 158 def visit_module(self, node): 159 """ 160 Don't visit imported modules when checking for docstrings. 161 162 @param node: the node we're visiting. 163 """ 164 pass 165 166 167 def visit_function(self, node): 168 """ 169 Don't request docstrings for commonly overridden autotest functions. 170 171 @param node: node of the ast we're currently checking. 172 """ 173 174 # Even plain functions will have a parent, which is the 175 # module they're in, and a frame, which is the context 176 # of said module; They need not however, always have 177 # ancestors. 178 if (node.name in ('run_once', 'initialize', 'cleanup') and 179 hasattr(node.parent.frame(), 'ancestors') and 180 any(ancestor.name == 'base_test' for ancestor in 181 node.parent.frame().ancestors())): 182 return 183 184 if _is_test_case_method(node): 185 return 186 187 super(CustomDocStringChecker, self).visit_function(node) 188 189 190 @staticmethod 191 def _should_skip_arg(arg): 192 """ 193 @return: True if the argument given by arg is whitelisted, and does 194 not require a "@param" docstring. 195 """ 196 return arg in ('self', 'cls', 'args', 'kwargs', 'dargs') 197 198 base.DocStringChecker = CustomDocStringChecker 199 imports.ImportsChecker = CustomImportsChecker 200 variables.VariablesChecker = CustomVariablesChecker 201 202 203 def batch_check_files(file_paths, base_opts): 204 """ 205 Run pylint on a list of files so we get consolidated errors. 206 207 @param file_paths: a list of file paths. 208 @param base_opts: a list of pylint config options. 209 210 @raises: pylint_error if pylint finds problems with a file 211 in this commit. 212 """ 213 if not file_paths: 214 return 215 216 pylint_runner = pylint.lint.Run(list(base_opts) + list(file_paths), 217 exit=False) 218 if pylint_runner.linter.msg_status: 219 raise pylint_error(pylint_runner.linter.msg_status) 220 221 222 def should_check_file(file_path): 223 """ 224 Don't check blacklisted or non .py files. 225 226 @param file_path: abs path of file to check. 227 @return: True if this file is a non-blacklisted python file. 228 """ 229 file_path = os.path.abspath(file_path) 230 if file_path.endswith('.py'): 231 return all(not fnmatch.fnmatch(file_path, '*' + pattern) 232 for pattern in BLACKLIST) 233 return False 234 235 236 def check_file(file_path, base_opts): 237 """ 238 Invokes pylint on files after confirming that they're not black listed. 239 240 @param base_opts: pylint base options. 241 @param file_path: path to the file we need to run pylint on. 242 """ 243 if not isinstance(file_path, basestring): 244 raise TypeError('expected a string as filepath, got %s'% 245 type(file_path)) 246 247 if should_check_file(file_path): 248 pylint_runner = pylint.lint.Run(base_opts + [file_path], exit=False) 249 if pylint_runner.linter.msg_status: 250 pylint_error(pylint_runner.linter.msg_status) 251 252 253 def visit(arg, dirname, filenames): 254 """ 255 Visit function invoked in check_dir. 256 257 @param arg: arg from os.walk.path 258 @param dirname: dir from os.walk.path 259 @param filenames: files in dir from os.walk.path 260 """ 261 for filename in filenames: 262 check_file(os.path.join(dirname, filename), arg) 263 264 265 def check_dir(dir_path, base_opts): 266 """ 267 Calls visit on files in dir_path. 268 269 @param base_opts: pylint base options. 270 @param dir_path: path to directory. 271 """ 272 os.path.walk(dir_path, visit, base_opts) 273 274 275 def extend_baseopts(base_opts, new_opt): 276 """ 277 Replaces an argument in base_opts with a cmd line argument. 278 279 @param base_opts: original pylint_base_opts. 280 @param new_opt: new cmd line option. 281 """ 282 for args in base_opts: 283 if new_opt in args: 284 base_opts.remove(args) 285 base_opts.append(new_opt) 286 287 288 def get_cmdline_options(args_list, pylint_base_opts, rcfile): 289 """ 290 Parses args_list and extends pylint_base_opts. 291 292 Command line arguments might include options mixed with files. 293 Go through this list and filter out the options, if the options are 294 specified in the pylintrc file we cannot replace them and the file 295 needs to be edited. If the options are already a part of 296 pylint_base_opts we replace them, and if not we append to 297 pylint_base_opts. 298 299 @param args_list: list of files/pylint args passed in through argv. 300 @param pylint_base_opts: default pylint options. 301 @param rcfile: text from pylint_rc. 302 """ 303 for args in args_list: 304 if args.startswith('--'): 305 opt_name = args[2:].split('=')[0] 306 if opt_name in rcfile and pylint_version >= 0.21: 307 raise run_pylint_error('The rcfile already contains the %s ' 308 'option. Please edit pylintrc instead.' 309 % opt_name) 310 else: 311 extend_baseopts(pylint_base_opts, args) 312 args_list.remove(args) 313 314 315 def git_show_to_temp_file(commit, original_file, new_temp_file): 316 """ 317 'Git shows' the file in original_file to a tmp file with 318 the name new_temp_file. We need to preserve the filename 319 as it gets reflected in pylints error report. 320 321 @param commit: commit hash of the commit we're running repo upload on. 322 @param original_file: the path to the original file we'd like to run 323 'git show' on. 324 @param new_temp_file: new_temp_file is the path to a temp file we write the 325 output of 'git show' into. 326 """ 327 git_repo = revision_control.GitRepo(common.autotest_dir, None, None, 328 common.autotest_dir) 329 330 with open(new_temp_file, 'w') as f: 331 output = git_repo.gitcmd('show --no-ext-diff %s:%s' 332 % (commit, original_file), 333 ignore_status=False).stdout 334 f.write(output) 335 336 337 def check_committed_files(work_tree_files, commit, pylint_base_opts): 338 """ 339 Get a list of files corresponding to the commit hash. 340 341 The contents of a file in the git work tree can differ from the contents 342 of a file in the commit we mean to upload. To work around this we run 343 pylint on a temp file into which we've 'git show'n the committed version 344 of each file. 345 346 @param work_tree_files: list of files in this commit specified by their 347 absolute path. 348 @param commit: hash of the commit this upload applies to. 349 @param pylint_base_opts: a list of pylint config options. 350 """ 351 files_to_check = filter(should_check_file, work_tree_files) 352 353 # Map the absolute path of each file so it's relative to the autotest repo. 354 # All files that are a part of this commit should have an abs path within 355 # the autotest repo, so this regex should never fail. 356 work_tree_files = [re.search(r'%s/(.*)' % common.autotest_dir, f).group(1) 357 for f in files_to_check] 358 359 tempdir = None 360 try: 361 tempdir = autotemp.tempdir() 362 temp_files = [os.path.join(tempdir.name, file_path.split('/')[-1:][0]) 363 for file_path in work_tree_files] 364 365 for file_tuple in zip(work_tree_files, temp_files): 366 git_show_to_temp_file(commit, *file_tuple) 367 # Only check if we successfully git showed all files in the commit. 368 batch_check_files(temp_files, pylint_base_opts) 369 finally: 370 if tempdir: 371 tempdir.clean() 372 373 374 def _is_test_case_method(node): 375 """Determine if the given function node is a method of a TestCase. 376 377 We simply check for 'TestCase' being one of the parent classes in the mro of 378 the containing class. 379 380 @params node: A function node. 381 """ 382 if not hasattr(node.parent.frame(), 'ancestors'): 383 return False 384 385 parent_class_names = {x.name for x in node.parent.frame().ancestors()} 386 return 'TestCase' in parent_class_names 387 388 389 def main(): 390 """Main function checks each file in a commit for pylint violations.""" 391 392 # For now all error/warning/refactor/convention exceptions except those in 393 # the enable string are disabled. 394 # W0611: All imported modules (except common) need to be used. 395 # W1201: Logging methods should take the form 396 # logging.<loggingmethod>(format_string, format_args...); and not 397 # logging.<loggingmethod>(format_string % (format_args...)) 398 # C0111: Docstring needed. Also checks @param for each arg. 399 # C0112: Non-empty Docstring needed. 400 # Ideally we would like to enable as much as we can, but if we did so at 401 # this stage anyone who makes a tiny change to a file will be tasked with 402 # cleaning all the lint in it. See chromium-os:37364. 403 404 # Note: 405 # 1. There are three major sources of E1101/E1103/E1120 false positives: 406 # * common_lib.enum.Enum objects 407 # * DB model objects (scheduler models are the worst, but Django models 408 # also generate some errors) 409 # 2. Docstrings are optional on private methods, and any methods that begin 410 # with either 'set_' or 'get_'. 411 pylint_rc = os.path.join(os.path.dirname(os.path.abspath(__file__)), 412 'pylintrc') 413 414 no_docstring_rgx = r'((_.*)|(set_.*)|(get_.*))' 415 if pylint_version >= 0.21: 416 pylint_base_opts = ['--rcfile=%s' % pylint_rc, 417 '--reports=no', 418 '--disable=W,R,E,C,F', 419 '--enable=W0611,W1201,C0111,C0112,E0602,W0601', 420 '--no-docstring-rgx=%s' % no_docstring_rgx,] 421 else: 422 all_failures = 'error,warning,refactor,convention' 423 pylint_base_opts = ['--disable-msg-cat=%s' % all_failures, 424 '--reports=no', 425 '--include-ids=y', 426 '--ignore-docstrings=n', 427 '--no-docstring-rgx=%s' % no_docstring_rgx,] 428 429 # run_pylint can be invoked directly with command line arguments, 430 # or through a presubmit hook which uses the arguments in pylintrc. In the 431 # latter case no command line arguments are passed. If it is invoked 432 # directly without any arguments, it should check all files in the cwd. 433 args_list = sys.argv[1:] 434 if args_list: 435 get_cmdline_options(args_list, 436 pylint_base_opts, 437 open(pylint_rc).read()) 438 batch_check_files(args_list, pylint_base_opts) 439 elif os.environ.get('PRESUBMIT_FILES') is not None: 440 check_committed_files( 441 os.environ.get('PRESUBMIT_FILES').split('\n'), 442 os.environ.get('PRESUBMIT_COMMIT'), 443 pylint_base_opts) 444 else: 445 check_dir('.', pylint_base_opts) 446 447 448 if __name__ == '__main__': 449 try: 450 main() 451 except pylint_error as e: 452 logging.error(e) 453 sys.exit(1) 454