1 # pylint: disable-msg=C0111 2 # Copyright 2008 Google Inc. Released under the GPL v2 3 4 import warnings 5 with warnings.catch_warnings(): 6 # The 'compiler' module is gone in Python 3.0. Let's not say 7 # so in every log file. 8 warnings.simplefilter("ignore", DeprecationWarning) 9 import compiler 10 import logging 11 import textwrap 12 import re 13 14 from autotest_lib.client.common_lib import enum 15 from autotest_lib.client.common_lib import global_config 16 from autotest_lib.client.common_lib import priorities 17 18 REQUIRED_VARS = set(['author', 'doc', 'name', 'time', 'test_type']) 19 OBSOLETE_VARS = set(['experimental']) 20 21 CONTROL_TYPE = enum.Enum('Server', 'Client', start_value=1) 22 CONTROL_TYPE_NAMES = enum.Enum(*CONTROL_TYPE.names, string_values=True) 23 24 _SUITE_ATTRIBUTE_PREFIX = 'suite:' 25 26 CONFIG = global_config.global_config 27 28 # Default maximum test result size in kB. 29 DEFAULT_MAX_RESULT_SIZE_KB = CONFIG.get_config_value( 30 'AUTOSERV', 'default_max_result_size_KB', type=int, default=20000) 31 32 33 class ControlVariableException(Exception): 34 pass 35 36 def _validate_control_file_fields(control_file_path, control_file_vars, 37 raise_warnings): 38 """Validate the given set of variables from a control file. 39 40 @param control_file_path: string path of the control file these were 41 loaded from. 42 @param control_file_vars: dict of variables set in a control file. 43 @param raise_warnings: True iff we should raise on invalid variables. 44 45 """ 46 diff = REQUIRED_VARS - set(control_file_vars) 47 if diff: 48 warning = ('WARNING: Not all required control ' 49 'variables were specified in %s. Please define ' 50 '%s.') % (control_file_path, ', '.join(diff)) 51 if raise_warnings: 52 raise ControlVariableException(warning) 53 print textwrap.wrap(warning, 80) 54 55 obsolete = OBSOLETE_VARS & set(control_file_vars) 56 if obsolete: 57 warning = ('WARNING: Obsolete variables were ' 58 'specified in %s. Please remove ' 59 '%s.') % (control_file_path, ', '.join(obsolete)) 60 if raise_warnings: 61 raise ControlVariableException(warning) 62 print textwrap.wrap(warning, 80) 63 64 65 class ControlData(object): 66 # Available TIME settings in control file, the list must be in lower case 67 # and in ascending order, test running faster comes first. 68 TEST_TIME_LIST = ['fast', 'short', 'medium', 'long', 'lengthy'] 69 TEST_TIME = enum.Enum(*TEST_TIME_LIST, string_values=False) 70 71 @staticmethod 72 def get_test_time_index(time): 73 """ 74 Get the order of estimated test time, based on the TIME setting in 75 Control file. Faster test gets a lower index number. 76 """ 77 try: 78 return ControlData.TEST_TIME.get_value(time.lower()) 79 except AttributeError: 80 # Raise exception if time value is not a valid TIME setting. 81 error_msg = '%s is not a valid TIME.' % time 82 logging.error(error_msg) 83 raise ControlVariableException(error_msg) 84 85 86 def __init__(self, vars, path, raise_warnings=False): 87 # Defaults 88 self.path = path 89 self.dependencies = set() 90 # TODO(jrbarnette): This should be removed once outside 91 # code that uses can be changed. 92 self.experimental = False 93 self.run_verify = True 94 self.sync_count = 1 95 self.test_parameters = set() 96 self.test_category = '' 97 self.test_class = '' 98 self.retries = 0 99 self.job_retries = 0 100 # Default to require server-side package. Unless require_ssp is 101 # explicitly set to False, server-side package will be used for the 102 # job. This can be overridden by global config 103 # AUTOSERV/enable_ssp_container 104 self.require_ssp = None 105 self.attributes = set() 106 self.max_result_size_KB = DEFAULT_MAX_RESULT_SIZE_KB 107 self.priority = priorities.Priority.DEFAULT 108 self.fast = False 109 110 _validate_control_file_fields(self.path, vars, raise_warnings) 111 112 for key, val in vars.iteritems(): 113 try: 114 self.set_attr(key, val, raise_warnings) 115 except Exception, e: 116 if raise_warnings: 117 raise 118 print 'WARNING: %s; skipping' % e 119 120 self._patch_up_suites_from_attributes() 121 122 123 @property 124 def suite_tag_parts(self): 125 """Return the part strings of the test's suite tag.""" 126 if hasattr(self, 'suite'): 127 return [part.strip() for part in self.suite.split(',')] 128 else: 129 return [] 130 131 132 def set_attr(self, attr, val, raise_warnings=False): 133 attr = attr.lower() 134 try: 135 set_fn = getattr(self, 'set_%s' % attr) 136 set_fn(val) 137 except AttributeError: 138 # This must not be a variable we care about 139 pass 140 141 142 def _patch_up_suites_from_attributes(self): 143 """Patch up the set of suites this test is part of. 144 145 Legacy builds will not have an appropriate ATTRIBUTES field set. 146 Take the union of suites specified via ATTRIBUTES and suites specified 147 via SUITE. 148 149 SUITE used to be its own variable, but now suites are taken only from 150 the attributes. 151 152 """ 153 154 suite_names = set() 155 # Extract any suites we know ourselves to be in based on the SUITE 156 # line. This line is deprecated, but control files in old builds will 157 # still have it. 158 if hasattr(self, 'suite'): 159 existing_suites = self.suite.split(',') 160 existing_suites = [name.strip() for name in existing_suites] 161 existing_suites = [name for name in existing_suites if name] 162 suite_names.update(existing_suites) 163 164 # Figure out if our attributes mention any suites. 165 for attribute in self.attributes: 166 if not attribute.startswith(_SUITE_ATTRIBUTE_PREFIX): 167 continue 168 suite_name = attribute[len(_SUITE_ATTRIBUTE_PREFIX):] 169 suite_names.add(suite_name) 170 171 # Rebuild the suite field if necessary. 172 if suite_names: 173 self.set_suite(','.join(sorted(list(suite_names)))) 174 175 176 def _set_string(self, attr, val): 177 val = str(val) 178 setattr(self, attr, val) 179 180 181 def _set_option(self, attr, val, options): 182 val = str(val) 183 if val.lower() not in [x.lower() for x in options]: 184 raise ValueError("%s must be one of the following " 185 "options: %s" % (attr, 186 ', '.join(options))) 187 setattr(self, attr, val) 188 189 190 def _set_bool(self, attr, val): 191 val = str(val).lower() 192 if val == "false": 193 val = False 194 elif val == "true": 195 val = True 196 else: 197 msg = "%s must be either true or false" % attr 198 raise ValueError(msg) 199 setattr(self, attr, val) 200 201 202 def _set_int(self, attr, val, min=None, max=None): 203 val = int(val) 204 if min is not None and min > val: 205 raise ValueError("%s is %d, which is below the " 206 "minimum of %d" % (attr, val, min)) 207 if max is not None and max < val: 208 raise ValueError("%s is %d, which is above the " 209 "maximum of %d" % (attr, val, max)) 210 setattr(self, attr, val) 211 212 213 def _set_set(self, attr, val): 214 val = str(val) 215 items = [x.strip() for x in val.split(',') if x.strip()] 216 setattr(self, attr, set(items)) 217 218 219 def set_author(self, val): 220 self._set_string('author', val) 221 222 223 def set_dependencies(self, val): 224 self._set_set('dependencies', val) 225 226 227 def set_doc(self, val): 228 self._set_string('doc', val) 229 230 231 def set_name(self, val): 232 self._set_string('name', val) 233 234 235 def set_run_verify(self, val): 236 self._set_bool('run_verify', val) 237 238 239 def set_sync_count(self, val): 240 self._set_int('sync_count', val, min=1) 241 242 243 def set_suite(self, val): 244 self._set_string('suite', val) 245 246 247 def set_time(self, val): 248 self._set_option('time', val, ControlData.TEST_TIME_LIST) 249 250 251 def set_test_class(self, val): 252 self._set_string('test_class', val.lower()) 253 254 255 def set_test_category(self, val): 256 self._set_string('test_category', val.lower()) 257 258 259 def set_test_type(self, val): 260 self._set_option('test_type', val, list(CONTROL_TYPE.names)) 261 262 263 def set_test_parameters(self, val): 264 self._set_set('test_parameters', val) 265 266 267 def set_retries(self, val): 268 self._set_int('retries', val) 269 270 271 def set_job_retries(self, val): 272 self._set_int('job_retries', val) 273 274 275 def set_bug_template(self, val): 276 if type(val) == dict: 277 setattr(self, 'bug_template', val) 278 279 280 def set_require_ssp(self, val): 281 self._set_bool('require_ssp', val) 282 283 284 def set_build(self, val): 285 self._set_string('build', val) 286 287 288 def set_builds(self, val): 289 if type(val) == dict: 290 setattr(self, 'builds', val) 291 292 def set_max_result_size_kb(self, val): 293 self._set_int('max_result_size_KB', val) 294 295 def set_priority(self, val): 296 self._set_int('priority', val) 297 298 def set_fast(self, val): 299 self._set_bool('fast', val) 300 301 def set_attributes(self, val): 302 # Add subsystem:default if subsystem is not specified. 303 self._set_set('attributes', val) 304 if not any(a.startswith('subsystem') for a in self.attributes): 305 self.attributes.add('subsystem:default') 306 307 308 def _extract_const(expr): 309 assert(expr.__class__ == compiler.ast.Const) 310 assert(expr.value.__class__ in (str, int, float, unicode)) 311 return str(expr.value).strip() 312 313 314 def _extract_dict(expr): 315 assert(expr.__class__ == compiler.ast.Dict) 316 assert(expr.items.__class__ == list) 317 cf_dict = {} 318 for key, value in expr.items: 319 try: 320 key = _extract_const(key) 321 val = _extract_expression(value) 322 except (AssertionError, ValueError): 323 pass 324 else: 325 cf_dict[key] = val 326 return cf_dict 327 328 329 def _extract_list(expr): 330 assert(expr.__class__ == compiler.ast.List) 331 list_values = [] 332 for value in expr.nodes: 333 try: 334 list_values.append(_extract_expression(value)) 335 except (AssertionError, ValueError): 336 pass 337 return list_values 338 339 340 def _extract_name(expr): 341 assert(expr.__class__ == compiler.ast.Name) 342 assert(expr.name in ('False', 'True', 'None')) 343 return str(expr.name) 344 345 346 def _extract_expression(expr): 347 if expr.__class__ == compiler.ast.Const: 348 return _extract_const(expr) 349 if expr.__class__ == compiler.ast.Name: 350 return _extract_name(expr) 351 if expr.__class__ == compiler.ast.Dict: 352 return _extract_dict(expr) 353 if expr.__class__ == compiler.ast.List: 354 return _extract_list(expr) 355 raise ValueError('Unknown rval %s' % expr) 356 357 358 def _extract_assignment(n): 359 assert(n.__class__ == compiler.ast.Assign) 360 assert(n.nodes.__class__ == list) 361 assert(len(n.nodes) == 1) 362 assert(n.nodes[0].__class__ == compiler.ast.AssName) 363 assert(n.nodes[0].flags.__class__ == str) 364 assert(n.nodes[0].name.__class__ == str) 365 366 val = _extract_expression(n.expr) 367 key = n.nodes[0].name.lower() 368 369 return (key, val) 370 371 372 def parse_control_string(control, raise_warnings=False, path=''): 373 """Parse a control file from a string. 374 375 @param control: string containing the text of a control file. 376 @param raise_warnings: True iff ControlData should raise an error on 377 warnings about control file contents. 378 @param path: string path to the control file. 379 380 """ 381 try: 382 mod = compiler.parse(control) 383 except SyntaxError as e: 384 logging.error('Syntax error (%s) while parsing control string:', e) 385 lines = control.split('\n') 386 for n, l in enumerate(lines): 387 logging.error('Line %d: %s', n + 1, l) 388 raise ControlVariableException("Error parsing data because %s" % e) 389 return finish_parse(mod, path, raise_warnings) 390 391 392 def parse_control(path, raise_warnings=False): 393 try: 394 mod = compiler.parseFile(path) 395 except SyntaxError, e: 396 raise ControlVariableException("Error parsing %s because %s" % 397 (path, e)) 398 return finish_parse(mod, path, raise_warnings) 399 400 401 def _try_extract_assignment(node, variables): 402 """Try to extract assignment from the given node. 403 404 @param node: An Assign object. 405 @param variables: Dictionary to store the parsed assignments. 406 """ 407 try: 408 key, val = _extract_assignment(node) 409 variables[key] = val 410 except (AssertionError, ValueError): 411 pass 412 413 414 def finish_parse(mod, path, raise_warnings): 415 assert(mod.__class__ == compiler.ast.Module) 416 assert(mod.node.__class__ == compiler.ast.Stmt) 417 assert(mod.node.nodes.__class__ == list) 418 419 variables = {} 420 injection_variables = {} 421 for n in mod.node.nodes: 422 if (n.__class__ == compiler.ast.Function and 423 re.match('step\d+', n.name)): 424 vars_in_step = {} 425 for sub_node in n.code.nodes: 426 _try_extract_assignment(sub_node, vars_in_step) 427 if vars_in_step: 428 # Empty the vars collection so assignments from multiple steps 429 # won't be mixed. 430 variables.clear() 431 variables.update(vars_in_step) 432 else: 433 _try_extract_assignment(n, injection_variables) 434 435 variables.update(injection_variables) 436 return ControlData(variables, path, raise_warnings) 437
