1 #!/usr/bin/python 2 """ 3 Script to verify errors on autotest code contributions (patches). 4 The workflow is as follows: 5 6 * Patch will be applied and eventual problems will be notified. 7 * If there are new files created, remember user to add them to VCS. 8 * If any added file looks like a executable file, remember user to make them 9 executable. 10 * If any of the files added or modified introduces trailing whitespaces, tabs 11 or incorrect indentation, report problems. 12 * If any of the files have problems during pylint validation, report failures. 13 * If any of the files changed have a unittest suite, run the unittest suite 14 and report any failures. 15 16 Usage: check_patch.py -p [/path/to/patch] 17 check_patch.py -i [patchwork id] 18 19 @copyright: Red Hat Inc, 2009. 20 @author: Lucas Meneghel Rodrigues <lmr (at] redhat.com> 21 """ 22 23 import os, stat, logging, sys, optparse, time 24 import common 25 from autotest_lib.client.common_lib import utils, error, logging_config 26 from autotest_lib.client.common_lib import logging_manager 27 28 29 class CheckPatchLoggingConfig(logging_config.LoggingConfig): 30 def configure_logging(self, results_dir=None, verbose=False): 31 super(CheckPatchLoggingConfig, self).configure_logging(use_console=True, 32 verbose=verbose) 33 34 35 class VCS(object): 36 """ 37 Abstraction layer to the version control system. 38 """ 39 def __init__(self): 40 """ 41 Class constructor. Guesses the version control name and instantiates it 42 as a backend. 43 """ 44 backend_name = self.guess_vcs_name() 45 if backend_name == "SVN": 46 self.backend = SubVersionBackend() 47 48 49 def guess_vcs_name(self): 50 if os.path.isdir(".svn"): 51 return "SVN" 52 else: 53 logging.error("Could not figure version control system. Are you " 54 "on a working directory? Aborting.") 55 sys.exit(1) 56 57 58 def get_unknown_files(self): 59 """ 60 Return a list of files unknown to the VCS. 61 """ 62 return self.backend.get_unknown_files() 63 64 65 def get_modified_files(self): 66 """ 67 Return a list of files that were modified, according to the VCS. 68 """ 69 return self.backend.get_modified_files() 70 71 72 def add_untracked_file(self, file): 73 """ 74 Add an untracked file to version control. 75 """ 76 return self.backend.add_untracked_file(file) 77 78 79 def revert_file(self, file): 80 """ 81 Restore file according to the latest state on the reference repo. 82 """ 83 return self.backend.revert_file(file) 84 85 86 def apply_patch(self, patch): 87 """ 88 Applies a patch using the most appropriate method to the particular VCS. 89 """ 90 return self.backend.apply_patch(patch) 91 92 93 def update(self): 94 """ 95 Updates the tree according to the latest state of the public tree 96 """ 97 return self.backend.update() 98 99 100 class SubVersionBackend(object): 101 """ 102 Implementation of a subversion backend for use with the VCS abstraction 103 layer. 104 """ 105 def __init__(self): 106 logging.debug("Subversion VCS backend initialized.") 107 self.ignored_extension_list = ['.orig', '.bak'] 108 109 110 def get_unknown_files(self): 111 status = utils.system_output("svn status --ignore-externals") 112 unknown_files = [] 113 for line in status.split("\n"): 114 status_flag = line[0] 115 if line and status_flag == "?": 116 for extension in self.ignored_extension_list: 117 if not line.endswith(extension): 118 unknown_files.append(line[1:].strip()) 119 return unknown_files 120 121 122 def get_modified_files(self): 123 status = utils.system_output("svn status --ignore-externals") 124 modified_files = [] 125 for line in status.split("\n"): 126 status_flag = line[0] 127 if line and status_flag == "M" or status_flag == "A": 128 modified_files.append(line[1:].strip()) 129 return modified_files 130 131 132 def add_untracked_file(self, file): 133 """ 134 Add an untracked file under revision control. 135 136 @param file: Path to untracked file. 137 """ 138 try: 139 utils.run('svn add %s' % file) 140 except error.CmdError, e: 141 logging.error("Problem adding file %s to svn: %s", file, e) 142 sys.exit(1) 143 144 145 def revert_file(self, file): 146 """ 147 Revert file against last revision. 148 149 @param file: Path to file to be reverted. 150 """ 151 try: 152 utils.run('svn revert %s' % file) 153 except error.CmdError, e: 154 logging.error("Problem reverting file %s: %s", file, e) 155 sys.exit(1) 156 157 158 def apply_patch(self, patch): 159 """ 160 Apply a patch to the code base. Patches are expected to be made using 161 level -p1, and taken according to the code base top level. 162 163 @param patch: Path to the patch file. 164 """ 165 try: 166 utils.system_output("patch -p1 < %s" % patch) 167 except: 168 logging.error("Patch applied incorrectly. Possible causes: ") 169 logging.error("1 - Patch might not be -p1") 170 logging.error("2 - You are not at the top of the autotest tree") 171 logging.error("3 - Patch was made using an older tree") 172 logging.error("4 - Mailer might have messed the patch") 173 sys.exit(1) 174 175 def update(self): 176 try: 177 utils.system("svn update", ignore_status=True) 178 except error.CmdError, e: 179 logging.error("SVN tree update failed: %s" % e) 180 181 182 class FileChecker(object): 183 """ 184 Picks up a given file and performs various checks, looking after problems 185 and eventually suggesting solutions. 186 """ 187 def __init__(self, path, confirm=False): 188 """ 189 Class constructor, sets the path attribute. 190 191 @param path: Path to the file that will be checked. 192 @param confirm: Whether to answer yes to all questions asked without 193 prompting the user. 194 """ 195 self.path = path 196 self.confirm = confirm 197 self.basename = os.path.basename(self.path) 198 if self.basename.endswith('.py'): 199 self.is_python = True 200 else: 201 self.is_python = False 202 203 mode = os.stat(self.path)[stat.ST_MODE] 204 if mode & stat.S_IXUSR: 205 self.is_executable = True 206 else: 207 self.is_executable = False 208 209 checked_file = open(self.path, "r") 210 self.first_line = checked_file.readline() 211 checked_file.close() 212 self.corrective_actions = [] 213 self.indentation_exceptions = ['job_unittest.py'] 214 215 216 def _check_indent(self): 217 """ 218 Verifies the file with reindent.py. This tool performs the following 219 checks on python files: 220 221 * Trailing whitespaces 222 * Tabs 223 * End of line 224 * Incorrect indentation 225 226 For the purposes of checking, the dry run mode is used and no changes 227 are made. It is up to the user to decide if he wants to run reindent 228 to correct the issues. 229 """ 230 reindent_raw = utils.system_output('reindent.py -v -d %s | head -1' % 231 self.path) 232 reindent_results = reindent_raw.split(" ")[-1].strip(".") 233 if reindent_results == "changed": 234 if self.basename not in self.indentation_exceptions: 235 self.corrective_actions.append("reindent.py -v %s" % self.path) 236 237 238 def _check_code(self): 239 """ 240 Verifies the file with run_pylint.py. This tool will call the static 241 code checker pylint using the special autotest conventions and warn 242 only on problems. If problems are found, a report will be generated. 243 Some of the problems reported might be bogus, but it's allways good 244 to look at them. 245 """ 246 c_cmd = 'run_pylint.py %s' % self.path 247 rc = utils.system(c_cmd, ignore_status=True) 248 if rc != 0: 249 logging.error("Syntax issues found during '%s'", c_cmd) 250 251 252 def _check_unittest(self): 253 """ 254 Verifies if the file in question has a unittest suite, if so, run the 255 unittest and report on any failures. This is important to keep our 256 unit tests up to date. 257 """ 258 if "unittest" not in self.basename: 259 stripped_name = self.basename.strip(".py") 260 unittest_name = stripped_name + "_unittest.py" 261 unittest_path = self.path.replace(self.basename, unittest_name) 262 if os.path.isfile(unittest_path): 263 unittest_cmd = 'python %s' % unittest_path 264 rc = utils.system(unittest_cmd, ignore_status=True) 265 if rc != 0: 266 logging.error("Unittest issues found during '%s'", 267 unittest_cmd) 268 269 270 def _check_permissions(self): 271 """ 272 Verifies the execution permissions, specifically: 273 * Files with no shebang and execution permissions are reported. 274 * Files with shebang and no execution permissions are reported. 275 """ 276 if self.first_line.startswith("#!"): 277 if not self.is_executable: 278 self.corrective_actions.append("svn propset svn:executable ON %s" % self.path) 279 else: 280 if self.is_executable: 281 self.corrective_actions.append("svn propdel svn:executable %s" % self.path) 282 283 284 def report(self): 285 """ 286 Executes all required checks, if problems are found, the possible 287 corrective actions are listed. 288 """ 289 self._check_permissions() 290 if self.is_python: 291 self._check_indent() 292 self._check_code() 293 self._check_unittest() 294 if self.corrective_actions: 295 for action in self.corrective_actions: 296 answer = utils.ask("Would you like to execute %s?" % action, 297 auto=self.confirm) 298 if answer == "y": 299 rc = utils.system(action, ignore_status=True) 300 if rc != 0: 301 logging.error("Error executing %s" % action) 302 303 304 class PatchChecker(object): 305 def __init__(self, patch=None, patchwork_id=None, confirm=False): 306 self.confirm = confirm 307 self.base_dir = os.getcwd() 308 if patch: 309 self.patch = os.path.abspath(patch) 310 if patchwork_id: 311 self.patch = self._fetch_from_patchwork(patchwork_id) 312 313 if not os.path.isfile(self.patch): 314 logging.error("Invalid patch file %s provided. Aborting.", 315 self.patch) 316 sys.exit(1) 317 318 self.vcs = VCS() 319 changed_files_before = self.vcs.get_modified_files() 320 if changed_files_before: 321 logging.error("Repository has changed files prior to patch " 322 "application. ") 323 answer = utils.ask("Would you like to revert them?", auto=self.confirm) 324 if answer == "n": 325 logging.error("Not safe to proceed without reverting files.") 326 sys.exit(1) 327 else: 328 for changed_file in changed_files_before: 329 self.vcs.revert_file(changed_file) 330 331 self.untracked_files_before = self.vcs.get_unknown_files() 332 self.vcs.update() 333 334 335 def _fetch_from_patchwork(self, id): 336 """ 337 Gets a patch file from patchwork and puts it under the cwd so it can 338 be applied. 339 340 @param id: Patchwork patch id. 341 """ 342 patch_url = "http://patchwork.test.kernel.org/patch/%s/mbox/" % id 343 patch_dest = os.path.join(self.base_dir, 'patchwork-%s.patch' % id) 344 patch = utils.get_file(patch_url, patch_dest) 345 # Patchwork sometimes puts garbage on the path, such as long 346 # sequences of underscores (_______). Get rid of those. 347 patch_ro = open(patch, 'r') 348 patch_contents = patch_ro.readlines() 349 patch_ro.close() 350 patch_rw = open(patch, 'w') 351 for line in patch_contents: 352 if not line.startswith("___"): 353 patch_rw.write(line) 354 patch_rw.close() 355 return patch 356 357 358 def _check_files_modified_patch(self): 359 untracked_files_after = self.vcs.get_unknown_files() 360 modified_files_after = self.vcs.get_modified_files() 361 add_to_vcs = [] 362 for untracked_file in untracked_files_after: 363 if untracked_file not in self.untracked_files_before: 364 add_to_vcs.append(untracked_file) 365 366 if add_to_vcs: 367 logging.info("The files: ") 368 for untracked_file in add_to_vcs: 369 logging.info(untracked_file) 370 logging.info("Might need to be added to VCS") 371 answer = utils.ask("Would you like to add them to VCS ?") 372 if answer == "y": 373 for untracked_file in add_to_vcs: 374 self.vcs.add_untracked_file(untracked_file) 375 modified_files_after.append(untracked_file) 376 elif answer == "n": 377 pass 378 379 for modified_file in modified_files_after: 380 # Additional safety check, new commits might introduce 381 # new directories 382 if os.path.isfile(modified_file): 383 file_checker = FileChecker(modified_file) 384 file_checker.report() 385 386 387 def check(self): 388 self.vcs.apply_patch(self.patch) 389 self._check_files_modified_patch() 390 391 392 if __name__ == "__main__": 393 parser = optparse.OptionParser() 394 parser.add_option('-p', '--patch', dest="local_patch", action='store', 395 help='path to a patch file that will be checked') 396 parser.add_option('-i', '--patchwork-id', dest="id", action='store', 397 help='id of a given patchwork patch') 398 parser.add_option('--verbose', dest="debug", action='store_true', 399 help='include debug messages in console output') 400 parser.add_option('-f', '--full-check', dest="full_check", 401 action='store_true', 402 help='check the full tree for corrective actions') 403 parser.add_option('-y', '--yes', dest="confirm", 404 action='store_true', 405 help='Answer yes to all questions') 406 407 options, args = parser.parse_args() 408 local_patch = options.local_patch 409 id = options.id 410 debug = options.debug 411 full_check = options.full_check 412 confirm = options.confirm 413 414 logging_manager.configure_logging(CheckPatchLoggingConfig(), verbose=debug) 415 416 ignore_file_list = ['common.py'] 417 if full_check: 418 for root, dirs, files in os.walk('.'): 419 if not '.svn' in root: 420 for file in files: 421 if file not in ignore_file_list: 422 path = os.path.join(root, file) 423 file_checker = FileChecker(path, confirm=confirm) 424 file_checker.report() 425 else: 426 if local_patch: 427 patch_checker = PatchChecker(patch=local_patch, confirm=confirm) 428 elif id: 429 patch_checker = PatchChecker(patchwork_id=id, confirm=confirm) 430 else: 431 logging.error('No patch or patchwork id specified. Aborting.') 432 sys.exit(1) 433 patch_checker.check() 434
