1 # pylint: disable-msg=C0111 2 # Copyright 2008 Google Inc. Released under the GPL v2 3 4 import warnings 5 with warnings.catch_warnings(): 6 # The 'compiler' module is gone in Python 3.0. Let's not say 7 # so in every log file. 8 warnings.simplefilter("ignore", DeprecationWarning) 9 import compiler 10 import logging 11 import textwrap 12 import re 13 14 from autotest_lib.client.common_lib import enum 15 from autotest_lib.client.common_lib import global_config 16 from autotest_lib.client.common_lib import priorities 17 18 REQUIRED_VARS = set(['author', 'doc', 'name', 'time', 'test_type']) 19 OBSOLETE_VARS = set(['experimental']) 20 21 CONTROL_TYPE = enum.Enum('Server', 'Client', start_value=1) 22 CONTROL_TYPE_NAMES = enum.Enum(*CONTROL_TYPE.names, string_values=True) 23 24 _SUITE_ATTRIBUTE_PREFIX = 'suite:' 25 26 CONFIG = global_config.global_config 27 28 # Default maximum test result size in kB. 29 DEFAULT_MAX_RESULT_SIZE_KB = CONFIG.get_config_value( 30 'AUTOSERV', 'default_max_result_size_KB', type=int, default=20000) 31 32 33 class ControlVariableException(Exception): 34 pass 35 36 def _validate_control_file_fields(control_file_path, control_file_vars, 37 raise_warnings): 38 """Validate the given set of variables from a control file. 39 40 @param control_file_path: string path of the control file these were 41 loaded from. 42 @param control_file_vars: dict of variables set in a control file. 43 @param raise_warnings: True iff we should raise on invalid variables. 44 45 """ 46 diff = REQUIRED_VARS - set(control_file_vars) 47 if diff: 48 warning = ('WARNING: Not all required control ' 49 'variables were specified in %s. Please define ' 50 '%s.') % (control_file_path, ', '.join(diff)) 51 if raise_warnings: 52 raise ControlVariableException(warning) 53 print textwrap.wrap(warning, 80) 54 55 obsolete = OBSOLETE_VARS & set(control_file_vars) 56 if obsolete: 57 warning = ('WARNING: Obsolete variables were ' 58 'specified in %s. Please remove ' 59 '%s.') % (control_file_path, ', '.join(obsolete)) 60 if raise_warnings: 61 raise ControlVariableException(warning) 62 print textwrap.wrap(warning, 80) 63 64 65 class ControlData(object): 66 # Available TIME settings in control file, the list must be in lower case 67 # and in ascending order, test running faster comes first. 68 TEST_TIME_LIST = ['fast', 'short', 'medium', 'long', 'lengthy'] 69 TEST_TIME = enum.Enum(*TEST_TIME_LIST, string_values=False) 70 71 @staticmethod 72 def get_test_time_index(time): 73 """ 74 Get the order of estimated test time, based on the TIME setting in 75 Control file. Faster test gets a lower index number. 76 """ 77 try: 78 return ControlData.TEST_TIME.get_value(time.lower()) 79 except AttributeError: 80 # Raise exception if time value is not a valid TIME setting. 81 error_msg = '%s is not a valid TIME.' % time 82 logging.error(error_msg) 83 raise ControlVariableException(error_msg) 84 85 86 def __init__(self, vars, path, raise_warnings=False): 87 # Defaults 88 self.path = path 89 self.dependencies = set() 90 # TODO(jrbarnette): This should be removed once outside 91 # code that uses can be changed. 92 self.experimental = False 93 self.run_verify = True 94 self.sync_count = 1 95 self.test_parameters = set() 96 self.test_category = '' 97 self.test_class = '' 98 self.job_retries = 0 99 # Default to require server-side package. Unless require_ssp is 100 # explicitly set to False, server-side package will be used for the 101 # job. This can be overridden by global config 102 # AUTOSERV/enable_ssp_container 103 self.require_ssp = None 104 self.attributes = set() 105 self.max_result_size_KB = DEFAULT_MAX_RESULT_SIZE_KB 106 self.priority = priorities.Priority.DEFAULT 107 self.fast = False 108 109 _validate_control_file_fields(self.path, vars, raise_warnings) 110 111 for key, val in vars.iteritems(): 112 try: 113 self.set_attr(key, val, raise_warnings) 114 except Exception, e: 115 if raise_warnings: 116 raise 117 print 'WARNING: %s; skipping' % e 118 119 self._patch_up_suites_from_attributes() 120 121 122 @property 123 def suite_tag_parts(self): 124 """Return the part strings of the test's suite tag.""" 125 if hasattr(self, 'suite'): 126 return [part.strip() for part in self.suite.split(',')] 127 else: 128 return [] 129 130 131 def set_attr(self, attr, val, raise_warnings=False): 132 attr = attr.lower() 133 try: 134 set_fn = getattr(self, 'set_%s' % attr) 135 set_fn(val) 136 except AttributeError: 137 # This must not be a variable we care about 138 pass 139 140 141 def _patch_up_suites_from_attributes(self): 142 """Patch up the set of suites this test is part of. 143 144 Legacy builds will not have an appropriate ATTRIBUTES field set. 145 Take the union of suites specified via ATTRIBUTES and suites specified 146 via SUITE. 147 148 SUITE used to be its own variable, but now suites are taken only from 149 the attributes. 150 151 """ 152 153 suite_names = set() 154 # Extract any suites we know ourselves to be in based on the SUITE 155 # line. This line is deprecated, but control files in old builds will 156 # still have it. 157 if hasattr(self, 'suite'): 158 existing_suites = self.suite.split(',') 159 existing_suites = [name.strip() for name in existing_suites] 160 existing_suites = [name for name in existing_suites if name] 161 suite_names.update(existing_suites) 162 163 # Figure out if our attributes mention any suites. 164 for attribute in self.attributes: 165 if not attribute.startswith(_SUITE_ATTRIBUTE_PREFIX): 166 continue 167 suite_name = attribute[len(_SUITE_ATTRIBUTE_PREFIX):] 168 suite_names.add(suite_name) 169 170 # Rebuild the suite field if necessary. 171 if suite_names: 172 self.set_suite(','.join(sorted(list(suite_names)))) 173 174 175 def _set_string(self, attr, val): 176 val = str(val) 177 setattr(self, attr, val) 178 179 180 def _set_option(self, attr, val, options): 181 val = str(val) 182 if val.lower() not in [x.lower() for x in options]: 183 raise ValueError("%s must be one of the following " 184 "options: %s" % (attr, 185 ', '.join(options))) 186 setattr(self, attr, val) 187 188 189 def _set_bool(self, attr, val): 190 val = str(val).lower() 191 if val == "false": 192 val = False 193 elif val == "true": 194 val = True 195 else: 196 msg = "%s must be either true or false" % attr 197 raise ValueError(msg) 198 setattr(self, attr, val) 199 200 201 def _set_int(self, attr, val, min=None, max=None): 202 val = int(val) 203 if min is not None and min > val: 204 raise ValueError("%s is %d, which is below the " 205 "minimum of %d" % (attr, val, min)) 206 if max is not None and max < val: 207 raise ValueError("%s is %d, which is above the " 208 "maximum of %d" % (attr, val, max)) 209 setattr(self, attr, val) 210 211 212 def _set_set(self, attr, val): 213 val = str(val) 214 items = [x.strip() for x in val.split(',') if x.strip()] 215 setattr(self, attr, set(items)) 216 217 218 def set_author(self, val): 219 self._set_string('author', val) 220 221 222 def set_dependencies(self, val): 223 self._set_set('dependencies', val) 224 225 226 def set_doc(self, val): 227 self._set_string('doc', val) 228 229 230 def set_name(self, val): 231 self._set_string('name', val) 232 233 234 def set_run_verify(self, val): 235 self._set_bool('run_verify', val) 236 237 238 def set_sync_count(self, val): 239 self._set_int('sync_count', val, min=1) 240 241 242 def set_suite(self, val): 243 self._set_string('suite', val) 244 245 246 def set_time(self, val): 247 self._set_option('time', val, ControlData.TEST_TIME_LIST) 248 249 250 def set_test_class(self, val): 251 self._set_string('test_class', val.lower()) 252 253 254 def set_test_category(self, val): 255 self._set_string('test_category', val.lower()) 256 257 258 def set_test_type(self, val): 259 self._set_option('test_type', val, list(CONTROL_TYPE.names)) 260 261 262 def set_test_parameters(self, val): 263 self._set_set('test_parameters', val) 264 265 266 def set_job_retries(self, val): 267 self._set_int('job_retries', val) 268 269 270 def set_bug_template(self, val): 271 if type(val) == dict: 272 setattr(self, 'bug_template', val) 273 274 275 def set_require_ssp(self, val): 276 self._set_bool('require_ssp', val) 277 278 279 def set_build(self, val): 280 self._set_string('build', val) 281 282 283 def set_builds(self, val): 284 if type(val) == dict: 285 setattr(self, 'builds', val) 286 287 def set_max_result_size_kb(self, val): 288 self._set_int('max_result_size_KB', val) 289 290 def set_priority(self, val): 291 self._set_int('priority', val) 292 293 def set_fast(self, val): 294 self._set_bool('fast', val) 295 296 def set_attributes(self, val): 297 # Add subsystem:default if subsystem is not specified. 298 self._set_set('attributes', val) 299 if not any(a.startswith('subsystem') for a in self.attributes): 300 self.attributes.add('subsystem:default') 301 302 303 def _extract_const(expr): 304 assert(expr.__class__ == compiler.ast.Const) 305 assert(expr.value.__class__ in (str, int, float, unicode)) 306 return str(expr.value).strip() 307 308 309 def _extract_dict(expr): 310 assert(expr.__class__ == compiler.ast.Dict) 311 assert(expr.items.__class__ == list) 312 cf_dict = {} 313 for key, value in expr.items: 314 try: 315 key = _extract_const(key) 316 val = _extract_expression(value) 317 except (AssertionError, ValueError): 318 pass 319 else: 320 cf_dict[key] = val 321 return cf_dict 322 323 324 def _extract_list(expr): 325 assert(expr.__class__ == compiler.ast.List) 326 list_values = [] 327 for value in expr.nodes: 328 try: 329 list_values.append(_extract_expression(value)) 330 except (AssertionError, ValueError): 331 pass 332 return list_values 333 334 335 def _extract_name(expr): 336 assert(expr.__class__ == compiler.ast.Name) 337 assert(expr.name in ('False', 'True', 'None')) 338 return str(expr.name) 339 340 341 def _extract_expression(expr): 342 if expr.__class__ == compiler.ast.Const: 343 return _extract_const(expr) 344 if expr.__class__ == compiler.ast.Name: 345 return _extract_name(expr) 346 if expr.__class__ == compiler.ast.Dict: 347 return _extract_dict(expr) 348 if expr.__class__ == compiler.ast.List: 349 return _extract_list(expr) 350 raise ValueError('Unknown rval %s' % expr) 351 352 353 def _extract_assignment(n): 354 assert(n.__class__ == compiler.ast.Assign) 355 assert(n.nodes.__class__ == list) 356 assert(len(n.nodes) == 1) 357 assert(n.nodes[0].__class__ == compiler.ast.AssName) 358 assert(n.nodes[0].flags.__class__ == str) 359 assert(n.nodes[0].name.__class__ == str) 360 361 val = _extract_expression(n.expr) 362 key = n.nodes[0].name.lower() 363 364 return (key, val) 365 366 367 def parse_control_string(control, raise_warnings=False, path=''): 368 """Parse a control file from a string. 369 370 @param control: string containing the text of a control file. 371 @param raise_warnings: True iff ControlData should raise an error on 372 warnings about control file contents. 373 @param path: string path to the control file. 374 375 """ 376 try: 377 mod = compiler.parse(control) 378 except SyntaxError as e: 379 logging.error('Syntax error (%s) while parsing control string:', e) 380 lines = control.split('\n') 381 for n, l in enumerate(lines): 382 logging.error('Line %d: %s', n + 1, l) 383 raise ControlVariableException("Error parsing data because %s" % e) 384 return finish_parse(mod, path, raise_warnings) 385 386 387 def parse_control(path, raise_warnings=False): 388 try: 389 mod = compiler.parseFile(path) 390 except SyntaxError, e: 391 raise ControlVariableException("Error parsing %s because %s" % 392 (path, e)) 393 return finish_parse(mod, path, raise_warnings) 394 395 396 def _try_extract_assignment(node, variables): 397 """Try to extract assignment from the given node. 398 399 @param node: An Assign object. 400 @param variables: Dictionary to store the parsed assignments. 401 """ 402 try: 403 key, val = _extract_assignment(node) 404 variables[key] = val 405 except (AssertionError, ValueError): 406 pass 407 408 409 def finish_parse(mod, path, raise_warnings): 410 assert(mod.__class__ == compiler.ast.Module) 411 assert(mod.node.__class__ == compiler.ast.Stmt) 412 assert(mod.node.nodes.__class__ == list) 413 414 variables = {} 415 injection_variables = {} 416 for n in mod.node.nodes: 417 if (n.__class__ == compiler.ast.Function and 418 re.match('step\d+', n.name)): 419 vars_in_step = {} 420 for sub_node in n.code.nodes: 421 _try_extract_assignment(sub_node, vars_in_step) 422 if vars_in_step: 423 # Empty the vars collection so assignments from multiple steps 424 # won't be mixed. 425 variables.clear() 426 variables.update(vars_in_step) 427 else: 428 _try_extract_assignment(n, injection_variables) 429 430 variables.update(injection_variables) 431 return ControlData(variables, path, raise_warnings) 432
