1 #!/usr/bin/python 2 # 3 # Copyright 2008 Google Inc. All Rights Reserved. 4 """ 5 This utility allows for easy updating, removing and importing 6 of tests into the autotest_web afe_autotests table. 7 8 Example of updating client side tests: 9 ./test_importer.py -t /usr/local/autotest/client/tests 10 11 If, for example, not all of your control files adhere to the standard outlined 12 at http://autotest.kernel.org/wiki/ControlRequirements, you can force options: 13 14 ./test_importer.py --test-type server -t /usr/local/autotest/server/tests 15 16 You would need to pass --add-noncompliant to include such control files, 17 however. An easy way to check for compliance is to run in dry mode: 18 19 ./test_importer.py --dry-run -t /usr/local/autotest/server/tests/mytest 20 21 Running with no options is equivalent to --add-all --db-clear-tests. 22 23 Most options should be fairly self explanatory, use --help to display them. 24 """ 25 26 27 import common 28 import logging, re, os, sys, optparse, compiler 29 30 from autotest_lib.frontend import setup_django_environment 31 from autotest_lib.frontend.afe import models 32 from autotest_lib.client.common_lib import control_data 33 from autotest_lib.client.common_lib import logging_config, logging_manager 34 35 36 class TestImporterLoggingConfig(logging_config.LoggingConfig): 37 #pylint: disable-msg=C0111 38 def configure_logging(self, results_dir=None, verbose=False): 39 super(TestImporterLoggingConfig, self).configure_logging( 40 use_console=True, 41 verbose=verbose) 42 43 44 # Global 45 DRY_RUN = False 46 DEPENDENCIES_NOT_FOUND = set() 47 48 49 def update_all(autotest_dir, add_noncompliant, add_experimental): 50 """ 51 Function to scan through all tests and add them to the database. 52 53 This function invoked when no parameters supplied to the command line. 54 It 'synchronizes' the test database with the current contents of the 55 client and server test directories. When test code is discovered 56 in the file system new tests may be added to the db. Likewise, 57 if test code is not found in the filesystem, tests may be removed 58 from the db. The base test directories are hard-coded to client/tests, 59 client/site_tests, server/tests and server/site_tests. 60 61 @param autotest_dir: prepended to path strings (/usr/local/autotest). 62 @param add_noncompliant: attempt adding test with invalid control files. 63 @param add_experimental: add tests with experimental attribute set. 64 """ 65 for path in [ 'server/tests', 'server/site_tests', 'client/tests', 66 'client/site_tests']: 67 test_path = os.path.join(autotest_dir, path) 68 if not os.path.exists(test_path): 69 continue 70 logging.info("Scanning %s", test_path) 71 tests = [] 72 tests = get_tests_from_fs(test_path, "^control.*", 73 add_noncompliant=add_noncompliant) 74 update_tests_in_db(tests, add_experimental=add_experimental, 75 add_noncompliant=add_noncompliant, 76 autotest_dir=autotest_dir) 77 test_suite_path = os.path.join(autotest_dir, 'test_suites') 78 if os.path.exists(test_suite_path): 79 logging.info("Scanning %s", test_suite_path) 80 tests = get_tests_from_fs(test_suite_path, '.*', 81 add_noncompliant=add_noncompliant) 82 update_tests_in_db(tests, add_experimental=add_experimental, 83 add_noncompliant=add_noncompliant, 84 autotest_dir=autotest_dir) 85 86 profilers_path = os.path.join(autotest_dir, "client/profilers") 87 if os.path.exists(profilers_path): 88 logging.info("Scanning %s", profilers_path) 89 profilers = get_tests_from_fs(profilers_path, '.*py$') 90 update_profilers_in_db(profilers, add_noncompliant=add_noncompliant, 91 description='NA') 92 # Clean bad db entries 93 db_clean_broken(autotest_dir) 94 95 96 def update_samples(autotest_dir, add_noncompliant, add_experimental): 97 """ 98 Add only sample tests to the database from the filesystem. 99 100 This function invoked when -S supplied on command line. 101 Only adds tests to the database - does not delete any. 102 Samples tests are formatted slightly differently than other tests. 103 104 @param autotest_dir: prepended to path strings (/usr/local/autotest). 105 @param add_noncompliant: attempt adding test with invalid control files. 106 @param add_experimental: add tests with experimental attribute set. 107 """ 108 sample_path = os.path.join(autotest_dir, 'server/samples') 109 if os.path.exists(sample_path): 110 logging.info("Scanning %s", sample_path) 111 tests = get_tests_from_fs(sample_path, '.*srv$', 112 add_noncompliant=add_noncompliant) 113 update_tests_in_db(tests, add_experimental=add_experimental, 114 add_noncompliant=add_noncompliant, 115 autotest_dir=autotest_dir) 116 117 118 def db_clean_broken(autotest_dir): 119 """ 120 Remove tests from autotest_web that do not have valid control files 121 122 This function invoked when -c supplied on the command line and when 123 running update_all(). Removes tests from database which are not 124 found in the filesystem. Also removes profilers which are just 125 a special case of tests. 126 127 @param autotest_dir: prepended to path strings (/usr/local/autotest). 128 """ 129 for test in models.Test.objects.all(): 130 full_path = os.path.join(autotest_dir, test.path) 131 if not os.path.isfile(full_path): 132 logging.info("Removing %s", test.path) 133 _log_or_execute(repr(test), test.delete) 134 135 # Find profilers that are no longer present 136 for profiler in models.Profiler.objects.all(): 137 full_path = os.path.join(autotest_dir, "client", "profilers", 138 profiler.name) 139 if not os.path.exists(full_path): 140 logging.info("Removing %s", profiler.name) 141 _log_or_execute(repr(profiler), profiler.delete) 142 143 144 def db_clean_all(autotest_dir): 145 """ 146 Remove all tests from autotest_web - very destructive 147 148 This function invoked when -C supplied on the command line. 149 Removes ALL tests from the database. 150 151 @param autotest_dir: prepended to path strings (/usr/local/autotest). 152 """ 153 for test in models.Test.objects.all(): 154 full_path = os.path.join(autotest_dir, test.path) 155 logging.info("Removing %s", test.path) 156 _log_or_execute(repr(test), test.delete) 157 158 # Find profilers that are no longer present 159 for profiler in models.Profiler.objects.all(): 160 full_path = os.path.join(autotest_dir, "client", "profilers", 161 profiler.name) 162 logging.info("Removing %s", profiler.name) 163 _log_or_execute(repr(profiler), profiler.delete) 164 165 166 def update_profilers_in_db(profilers, description='NA', 167 add_noncompliant=False): 168 """ 169 Add only profilers to the database from the filesystem. 170 171 This function invoked when -p supplied on command line. 172 Only adds profilers to the database - does not delete any. 173 Profilers are formatted slightly differently than tests. 174 175 @param profilers: list of profilers found in the file system. 176 @param description: simple text to satisfy docstring. 177 @param add_noncompliant: attempt adding test with invalid control files. 178 """ 179 for profiler in profilers: 180 name = os.path.basename(profiler) 181 if name.endswith('.py'): 182 name = name[:-3] 183 if not profilers[profiler]: 184 if add_noncompliant: 185 doc = description 186 else: 187 logging.warning("Skipping %s, missing docstring", profiler) 188 continue 189 else: 190 doc = profilers[profiler] 191 192 model = models.Profiler.objects.get_or_create(name=name)[0] 193 model.description = doc 194 _log_or_execute(repr(model), model.save) 195 196 197 def _set_attributes_custom(test, data): 198 # We set the test name to the dirname of the control file. 199 test_new_name = test.path.split('/') 200 if test_new_name[-1] == 'control' or test_new_name[-1] == 'control.srv': 201 test.name = test_new_name[-2] 202 else: 203 control_name = "%s:%s" 204 control_name %= (test_new_name[-2], 205 test_new_name[-1]) 206 test.name = re.sub('control.*\.', '', control_name) 207 208 # We set verify to always False (0). 209 test.run_verify = 0 210 211 if hasattr(data, 'test_parameters'): 212 for para_name in data.test_parameters: 213 test_parameter = models.TestParameter.objects.get_or_create( 214 test=test, name=para_name)[0] 215 test_parameter.save() 216 217 218 def update_tests_in_db(tests, dry_run=False, add_experimental=False, 219 add_noncompliant=False, autotest_dir=None): 220 """ 221 Scans through all tests and add them to the database. 222 223 This function invoked when -t supplied and for update_all. 224 When test code is discovered in the file system new tests may be added 225 226 @param tests: list of tests found in the filesystem. 227 @param dry_run: not used at this time. 228 @param add_experimental: add tests with experimental attribute set. 229 @param add_noncompliant: attempt adding test with invalid control files. 230 @param autotest_dir: prepended to path strings (/usr/local/autotest). 231 """ 232 for test in tests: 233 new_test = models.Test.objects.get_or_create( 234 path=test.replace(autotest_dir, '').lstrip('/'))[0] 235 logging.info("Processing %s", new_test.path) 236 237 # Set the test's attributes 238 data = tests[test] 239 _set_attributes_clean(new_test, data) 240 241 # Custom Attribute Update 242 _set_attributes_custom(new_test, data) 243 244 # This only takes place if --add-noncompliant is provided on the CLI 245 if not new_test.name: 246 test_new_test = test.split('/') 247 if test_new_test[-1] == 'control': 248 new_test.name = test_new_test[-2] 249 else: 250 control_name = "%s:%s" 251 control_name %= (test_new_test[-2], 252 test_new_test[-1]) 253 new_test.name = control_name.replace('control.', '') 254 255 # Experimental Check 256 if not add_experimental and new_test.experimental: 257 continue 258 259 _log_or_execute(repr(new_test), new_test.save) 260 add_label_dependencies(new_test) 261 262 # save TestParameter 263 for para_name in data.test_parameters: 264 test_parameter = models.TestParameter.objects.get_or_create( 265 test=new_test, name=para_name)[0] 266 test_parameter.save() 267 268 269 def _set_attributes_clean(test, data): 270 """ 271 First pass sets the attributes of the Test object from file system. 272 273 @param test: a test object to be populated for the database. 274 @param data: object with test data from the file system. 275 """ 276 test_time = { 'short' : 1, 277 'medium' : 2, 278 'long' : 3, } 279 280 281 string_attributes = ('name', 'author', 'test_class', 'test_category', 282 'test_category', 'sync_count') 283 for attribute in string_attributes: 284 setattr(test, attribute, getattr(data, attribute)) 285 286 test.description = data.doc 287 test.dependencies = ", ".join(data.dependencies) 288 289 try: 290 test.test_type = control_data.CONTROL_TYPE.get_value(data.test_type) 291 except AttributeError: 292 raise Exception('Unknown test_type %s for test %s', data.test_type, 293 data.name) 294 295 int_attributes = ('experimental', 'run_verify') 296 for attribute in int_attributes: 297 setattr(test, attribute, int(getattr(data, attribute))) 298 299 try: 300 test.test_time = int(data.time) 301 if test.test_time < 1 or test.time > 3: 302 raise Exception('Incorrect number %d for time' % test.time) 303 except ValueError: 304 pass 305 306 if not test.test_time and str == type(data.time): 307 test.test_time = test_time[data.time.lower()] 308 309 test.test_retry = data.retries 310 311 312 def add_label_dependencies(test): 313 """ 314 Add proper many-to-many relationships from DEPENDENCIES field. 315 316 @param test: test object for the database. 317 """ 318 319 # clear out old relationships 320 _log_or_execute(repr(test), test.dependency_labels.clear, 321 subject='clear dependencies from') 322 323 for label_name in test.dependencies.split(','): 324 label_name = label_name.strip().lower() 325 if not label_name: 326 continue 327 328 try: 329 label = models.Label.objects.get(name=label_name) 330 except models.Label.DoesNotExist: 331 log_dependency_not_found(label_name) 332 continue 333 334 _log_or_execute(repr(label), test.dependency_labels.add, label, 335 subject='add dependency to %s' % test.name) 336 337 338 def log_dependency_not_found(label_name): 339 """ 340 Exception processing when label not found in database. 341 342 @param label_name: from test dependencies. 343 """ 344 if label_name in DEPENDENCIES_NOT_FOUND: 345 return 346 logging.info("Dependency %s not found", label_name) 347 DEPENDENCIES_NOT_FOUND.add(label_name) 348 349 350 def get_tests_from_fs(parent_dir, control_pattern, add_noncompliant=False): 351 """ 352 Find control files in file system and load a list with their info. 353 354 @param parent_dir: directory to search recursively. 355 @param control_pattern: name format of control file. 356 @param add_noncompliant: ignore control file parse errors. 357 358 @return dictionary of the form: tests[file_path] = parsed_object 359 """ 360 tests = {} 361 profilers = False 362 if 'client/profilers' in parent_dir: 363 profilers = True 364 for dir in [ parent_dir ]: 365 files = recursive_walk(dir, control_pattern) 366 for file in files: 367 if '__init__.py' in file or '.svn' in file: 368 continue 369 if not profilers: 370 if not add_noncompliant: 371 try: 372 found_test = control_data.parse_control(file, 373 raise_warnings=True) 374 tests[file] = found_test 375 except control_data.ControlVariableException, e: 376 logging.warning("Skipping %s\n%s", file, e) 377 except Exception, e: 378 logging.error("Bad %s\n%s", file, e) 379 else: 380 found_test = control_data.parse_control(file) 381 tests[file] = found_test 382 else: 383 tests[file] = compiler.parseFile(file).doc 384 return tests 385 386 387 def recursive_walk(path, wildcard): 388 """ 389 Recursively go through a directory. 390 391 This function invoked by get_tests_from_fs(). 392 393 @param path: base directory to start search. 394 @param wildcard: name format to match. 395 396 @return A list of files that match wildcard 397 """ 398 files = [] 399 directories = [ path ] 400 while len(directories)>0: 401 directory = directories.pop() 402 for name in os.listdir(directory): 403 fullpath = os.path.join(directory, name) 404 if os.path.isfile(fullpath): 405 # if we are a control file 406 if re.search(wildcard, name): 407 files.append(fullpath) 408 elif os.path.isdir(fullpath): 409 directories.append(fullpath) 410 return files 411 412 413 def _log_or_execute(content, func, *args, **kwargs): 414 """ 415 Log a message if dry_run is enabled, or execute the given function. 416 417 Relies on the DRY_RUN global variable. 418 419 @param content: the actual log message. 420 @param func: function to execute if dry_run is not enabled. 421 @param subject: (Optional) The type of log being written. Defaults to 422 the name of the provided function. 423 """ 424 subject = kwargs.get('subject', func.__name__) 425 426 if DRY_RUN: 427 logging.info("Would %s: %s", subject, content) 428 else: 429 func(*args) 430 431 432 def _create_whitelist_set(whitelist_path): 433 """ 434 Create a set with contents from a whitelist file for membership testing. 435 436 @param whitelist_path: full path to the whitelist file. 437 438 @return set with files listed one/line - newlines included. 439 """ 440 f = open(whitelist_path, 'r') 441 whitelist_set = set([line.strip() for line in f]) 442 f.close() 443 return whitelist_set 444 445 446 def update_from_whitelist(whitelist_set, add_experimental, add_noncompliant, 447 autotest_dir): 448 """ 449 Scans through all tests in the whitelist and add them to the database. 450 451 This function invoked when -w supplied. 452 453 @param whitelist_set: set of tests in full-path form from a whitelist. 454 @param add_experimental: add tests with experimental attribute set. 455 @param add_noncompliant: attempt adding test with invalid control files. 456 @param autotest_dir: prepended to path strings (/usr/local/autotest). 457 """ 458 tests = {} 459 profilers = {} 460 for file_path in whitelist_set: 461 if file_path.find('client/profilers') == -1: 462 try: 463 found_test = control_data.parse_control(file_path, 464 raise_warnings=True) 465 tests[file_path] = found_test 466 except control_data.ControlVariableException, e: 467 logging.warning("Skipping %s\n%s", file, e) 468 else: 469 profilers[file_path] = compiler.parseFile(file_path).doc 470 471 if len(tests) > 0: 472 update_tests_in_db(tests, add_experimental=add_experimental, 473 add_noncompliant=add_noncompliant, 474 autotest_dir=autotest_dir) 475 if len(profilers) > 0: 476 update_profilers_in_db(profilers, add_noncompliant=add_noncompliant, 477 description='NA') 478 479 480 def main(argv): 481 """Main function 482 @param argv: List of command line parameters. 483 """ 484 485 global DRY_RUN 486 parser = optparse.OptionParser() 487 parser.add_option('-c', '--db-clean-tests', 488 dest='clean_tests', action='store_true', 489 default=False, 490 help='Clean client and server tests with invalid control files') 491 parser.add_option('-C', '--db-clear-all-tests', 492 dest='clear_all_tests', action='store_true', 493 default=False, 494 help='Clear ALL client and server tests') 495 parser.add_option('-d', '--dry-run', 496 dest='dry_run', action='store_true', default=False, 497 help='Dry run for operation') 498 parser.add_option('-A', '--add-all', 499 dest='add_all', action='store_true', 500 default=False, 501 help='Add site_tests, tests, and test_suites') 502 parser.add_option('-S', '--add-samples', 503 dest='add_samples', action='store_true', 504 default=False, 505 help='Add samples.') 506 parser.add_option('-E', '--add-experimental', 507 dest='add_experimental', action='store_true', 508 default=True, 509 help='Add experimental tests to frontend, works only ' 510 'with -A (--add-all) option') 511 parser.add_option('-N', '--add-noncompliant', 512 dest='add_noncompliant', action='store_true', 513 default=False, 514 help='Add non-compliant tests (i.e. tests that do not ' 515 'define all required control variables), works ' 516 'only with -A (--add-all) option') 517 parser.add_option('-p', '--profile-dir', dest='profile_dir', 518 help='Directory to recursively check for profiles') 519 parser.add_option('-t', '--tests-dir', dest='tests_dir', 520 help='Directory to recursively check for control.*') 521 parser.add_option('-r', '--control-pattern', dest='control_pattern', 522 default='^control.*', 523 help='The pattern to look for in directories for control files') 524 parser.add_option('-v', '--verbose', 525 dest='verbose', action='store_true', default=False, 526 help='Run in verbose mode') 527 parser.add_option('-w', '--whitelist-file', dest='whitelist_file', 528 help='Filename for list of test names that must match') 529 parser.add_option('-z', '--autotest-dir', dest='autotest_dir', 530 default=os.path.join(os.path.dirname(__file__), '..'), 531 help='Autotest directory root') 532 options, args = parser.parse_args() 533 534 logging_manager.configure_logging(TestImporterLoggingConfig(), 535 verbose=options.verbose) 536 537 DRY_RUN = options.dry_run 538 if DRY_RUN: 539 logging.getLogger().setLevel(logging.WARN) 540 541 if len(argv) > 1 and options.add_noncompliant and not options.add_all: 542 logging.error('-N (--add-noncompliant) must be ran with option -A ' 543 '(--add-All).') 544 return 1 545 546 if len(argv) > 1 and options.add_experimental and not options.add_all: 547 logging.error('-E (--add-experimental) must be ran with option -A ' 548 '(--add-All).') 549 return 1 550 551 # Make sure autotest_dir is the absolute path 552 options.autotest_dir = os.path.abspath(options.autotest_dir) 553 554 if len(args) > 0: 555 logging.error("Invalid option(s) provided: %s", args) 556 parser.print_help() 557 return 1 558 559 if options.verbose: 560 logging.getLogger().setLevel(logging.DEBUG) 561 562 if len(argv) == 1 or (len(argv) == 2 and options.verbose): 563 update_all(options.autotest_dir, options.add_noncompliant, 564 options.add_experimental) 565 db_clean_broken(options.autotest_dir) 566 return 0 567 568 if options.clear_all_tests: 569 if (options.clean_tests or options.add_all or options.add_samples or 570 options.add_noncompliant): 571 logging.error( 572 "Can only pass --autotest-dir, --dry-run and --verbose with " 573 "--db-clear-all-tests") 574 return 1 575 db_clean_all(options.autotest_dir) 576 577 whitelist_set = None 578 if options.whitelist_file: 579 if options.add_all: 580 logging.error("Cannot pass both --add-all and --whitelist-file") 581 return 1 582 whitelist_path = os.path.abspath(options.whitelist_file) 583 if not os.path.isfile(whitelist_path): 584 logging.error("--whitelist-file (%s) not found", whitelist_path) 585 return 1 586 logging.info("Using whitelist file %s", whitelist_path) 587 whitelist_set = _create_whitelist_set(whitelist_path) 588 update_from_whitelist(whitelist_set, 589 add_experimental=options.add_experimental, 590 add_noncompliant=options.add_noncompliant, 591 autotest_dir=options.autotest_dir) 592 if options.add_all: 593 update_all(options.autotest_dir, options.add_noncompliant, 594 options.add_experimental) 595 if options.add_samples: 596 update_samples(options.autotest_dir, options.add_noncompliant, 597 options.add_experimental) 598 if options.tests_dir: 599 options.tests_dir = os.path.abspath(options.tests_dir) 600 tests = get_tests_from_fs(options.tests_dir, options.control_pattern, 601 add_noncompliant=options.add_noncompliant) 602 update_tests_in_db(tests, add_experimental=options.add_experimental, 603 add_noncompliant=options.add_noncompliant, 604 autotest_dir=options.autotest_dir) 605 if options.profile_dir: 606 profilers = get_tests_from_fs(options.profile_dir, '.*py$') 607 update_profilers_in_db(profilers, 608 add_noncompliant=options.add_noncompliant, 609 description='NA') 610 if options.clean_tests: 611 db_clean_broken(options.autotest_dir) 612 613 614 if __name__ == "__main__": 615 sys.exit(main(sys.argv)) 616
