1 # pylint: disable-msg=C0111 2 # Copyright 2008 Google Inc. Released under the GPL v2 3 4 import warnings 5 with warnings.catch_warnings(): 6 # The 'compiler' module is gone in Python 3.0. Let's not say 7 # so in every log file. 8 warnings.simplefilter("ignore", DeprecationWarning) 9 import compiler 10 import logging 11 import textwrap 12 import re 13 14 from autotest_lib.client.common_lib import enum 15 from autotest_lib.client.common_lib import global_config 16 17 REQUIRED_VARS = set(['author', 'doc', 'name', 'time', 'test_type']) 18 OBSOLETE_VARS = set(['experimental']) 19 20 CONTROL_TYPE = enum.Enum('Server', 'Client', start_value=1) 21 CONTROL_TYPE_NAMES = enum.Enum(*CONTROL_TYPE.names, string_values=True) 22 23 _SUITE_ATTRIBUTE_PREFIX = 'suite:' 24 25 CONFIG = global_config.global_config 26 27 # Default maximum test result size in kB. 28 DEFAULT_MAX_RESULT_SIZE_KB = CONFIG.get_config_value( 29 'AUTOSERV', 'default_max_result_size_KB', type=int) 30 31 class ControlVariableException(Exception): 32 pass 33 34 def _validate_control_file_fields(control_file_path, control_file_vars, 35 raise_warnings): 36 """Validate the given set of variables from a control file. 37 38 @param control_file_path: string path of the control file these were 39 loaded from. 40 @param control_file_vars: dict of variables set in a control file. 41 @param raise_warnings: True iff we should raise on invalid variables. 42 43 """ 44 diff = REQUIRED_VARS - set(control_file_vars) 45 if diff: 46 warning = ('WARNING: Not all required control ' 47 'variables were specified in %s. Please define ' 48 '%s.') % (control_file_path, ', '.join(diff)) 49 if raise_warnings: 50 raise ControlVariableException(warning) 51 print textwrap.wrap(warning, 80) 52 53 obsolete = OBSOLETE_VARS & set(control_file_vars) 54 if obsolete: 55 warning = ('WARNING: Obsolete variables were ' 56 'specified in %s. Please remove ' 57 '%s.') % (control_file_path, ', '.join(obsolete)) 58 if raise_warnings: 59 raise ControlVariableException(warning) 60 print textwrap.wrap(warning, 80) 61 62 63 class ControlData(object): 64 # Available TIME settings in control file, the list must be in lower case 65 # and in ascending order, test running faster comes first. 66 TEST_TIME_LIST = ['fast', 'short', 'medium', 'long', 'lengthy'] 67 TEST_TIME = enum.Enum(*TEST_TIME_LIST, string_values=False) 68 69 @staticmethod 70 def get_test_time_index(time): 71 """ 72 Get the order of estimated test time, based on the TIME setting in 73 Control file. Faster test gets a lower index number. 74 """ 75 try: 76 return ControlData.TEST_TIME.get_value(time.lower()) 77 except AttributeError: 78 # Raise exception if time value is not a valid TIME setting. 79 error_msg = '%s is not a valid TIME.' % time 80 logging.error(error_msg) 81 raise ControlVariableException(error_msg) 82 83 84 def __init__(self, vars, path, raise_warnings=False): 85 # Defaults 86 self.path = path 87 self.dependencies = set() 88 # TODO(jrbarnette): This should be removed once outside 89 # code that uses can be changed. 90 self.experimental = False 91 self.run_verify = True 92 self.sync_count = 1 93 self.test_parameters = set() 94 self.test_category = '' 95 self.test_class = '' 96 self.retries = 0 97 self.job_retries = 0 98 # Default to require server-side package. Unless require_ssp is 99 # explicitly set to False, server-side package will be used for the 100 # job. This can be overridden by global config 101 # AUTOSERV/enable_ssp_container 102 self.require_ssp = None 103 self.attributes = set() 104 self.max_result_size_KB = DEFAULT_MAX_RESULT_SIZE_KB 105 106 _validate_control_file_fields(self.path, vars, raise_warnings) 107 108 for key, val in vars.iteritems(): 109 try: 110 self.set_attr(key, val, raise_warnings) 111 except Exception, e: 112 if raise_warnings: 113 raise 114 print 'WARNING: %s; skipping' % e 115 116 self._patch_up_suites_from_attributes() 117 118 119 @property 120 def suite_tag_parts(self): 121 """Return the part strings of the test's suite tag.""" 122 if hasattr(self, 'suite'): 123 return [part.strip() for part in self.suite.split(',')] 124 else: 125 return [] 126 127 128 def set_attr(self, attr, val, raise_warnings=False): 129 attr = attr.lower() 130 try: 131 set_fn = getattr(self, 'set_%s' % attr) 132 set_fn(val) 133 except AttributeError: 134 # This must not be a variable we care about 135 pass 136 137 138 def _patch_up_suites_from_attributes(self): 139 """Patch up the set of suites this test is part of. 140 141 Legacy builds will not have an appropriate ATTRIBUTES field set. 142 Take the union of suites specified via ATTRIBUTES and suites specified 143 via SUITE. 144 145 SUITE used to be its own variable, but now suites are taken only from 146 the attributes. 147 148 """ 149 150 suite_names = set() 151 # Extract any suites we know ourselves to be in based on the SUITE 152 # line. This line is deprecated, but control files in old builds will 153 # still have it. 154 if hasattr(self, 'suite'): 155 existing_suites = self.suite.split(',') 156 existing_suites = [name.strip() for name in existing_suites] 157 existing_suites = [name for name in existing_suites if name] 158 suite_names.update(existing_suites) 159 160 # Figure out if our attributes mention any suites. 161 for attribute in self.attributes: 162 if not attribute.startswith(_SUITE_ATTRIBUTE_PREFIX): 163 continue 164 suite_name = attribute[len(_SUITE_ATTRIBUTE_PREFIX):] 165 suite_names.add(suite_name) 166 167 # Rebuild the suite field if necessary. 168 if suite_names: 169 self.set_suite(','.join(sorted(list(suite_names)))) 170 171 172 def _set_string(self, attr, val): 173 val = str(val) 174 setattr(self, attr, val) 175 176 177 def _set_option(self, attr, val, options): 178 val = str(val) 179 if val.lower() not in [x.lower() for x in options]: 180 raise ValueError("%s must be one of the following " 181 "options: %s" % (attr, 182 ', '.join(options))) 183 setattr(self, attr, val) 184 185 186 def _set_bool(self, attr, val): 187 val = str(val).lower() 188 if val == "false": 189 val = False 190 elif val == "true": 191 val = True 192 else: 193 msg = "%s must be either true or false" % attr 194 raise ValueError(msg) 195 setattr(self, attr, val) 196 197 198 def _set_int(self, attr, val, min=None, max=None): 199 val = int(val) 200 if min is not None and min > val: 201 raise ValueError("%s is %d, which is below the " 202 "minimum of %d" % (attr, val, min)) 203 if max is not None and max < val: 204 raise ValueError("%s is %d, which is above the " 205 "maximum of %d" % (attr, val, max)) 206 setattr(self, attr, val) 207 208 209 def _set_set(self, attr, val): 210 val = str(val) 211 items = [x.strip() for x in val.split(',') if x.strip()] 212 setattr(self, attr, set(items)) 213 214 215 def set_author(self, val): 216 self._set_string('author', val) 217 218 219 def set_dependencies(self, val): 220 self._set_set('dependencies', val) 221 222 223 def set_doc(self, val): 224 self._set_string('doc', val) 225 226 227 def set_name(self, val): 228 self._set_string('name', val) 229 230 231 def set_run_verify(self, val): 232 self._set_bool('run_verify', val) 233 234 235 def set_sync_count(self, val): 236 self._set_int('sync_count', val, min=1) 237 238 239 def set_suite(self, val): 240 self._set_string('suite', val) 241 242 243 def set_time(self, val): 244 self._set_option('time', val, ControlData.TEST_TIME_LIST) 245 246 247 def set_test_class(self, val): 248 self._set_string('test_class', val.lower()) 249 250 251 def set_test_category(self, val): 252 self._set_string('test_category', val.lower()) 253 254 255 def set_test_type(self, val): 256 self._set_option('test_type', val, list(CONTROL_TYPE.names)) 257 258 259 def set_test_parameters(self, val): 260 self._set_set('test_parameters', val) 261 262 263 def set_retries(self, val): 264 self._set_int('retries', val) 265 266 267 def set_job_retries(self, val): 268 self._set_int('job_retries', val) 269 270 271 def set_bug_template(self, val): 272 if type(val) == dict: 273 setattr(self, 'bug_template', val) 274 275 276 def set_require_ssp(self, val): 277 self._set_bool('require_ssp', val) 278 279 280 def set_build(self, val): 281 self._set_string('build', val) 282 283 284 def set_builds(self, val): 285 if type(val) == dict: 286 setattr(self, 'builds', val) 287 288 def set_max_result_size_kb(self, val): 289 self._set_int('max_result_size_KB', val) 290 291 def set_attributes(self, val): 292 # Add subsystem:default if subsystem is not specified. 293 self._set_set('attributes', val) 294 if not any(a.startswith('subsystem') for a in self.attributes): 295 self.attributes.add('subsystem:default') 296 297 298 def _extract_const(expr): 299 assert(expr.__class__ == compiler.ast.Const) 300 assert(expr.value.__class__ in (str, int, float, unicode)) 301 return str(expr.value).strip() 302 303 304 def _extract_dict(expr): 305 assert(expr.__class__ == compiler.ast.Dict) 306 assert(expr.items.__class__ == list) 307 cf_dict = {} 308 for key, value in expr.items: 309 try: 310 key = _extract_const(key) 311 val = _extract_expression(value) 312 except (AssertionError, ValueError): 313 pass 314 else: 315 cf_dict[key] = val 316 return cf_dict 317 318 319 def _extract_list(expr): 320 assert(expr.__class__ == compiler.ast.List) 321 list_values = [] 322 for value in expr.nodes: 323 try: 324 list_values.append(_extract_expression(value)) 325 except (AssertionError, ValueError): 326 pass 327 return list_values 328 329 330 def _extract_name(expr): 331 assert(expr.__class__ == compiler.ast.Name) 332 assert(expr.name in ('False', 'True', 'None')) 333 return str(expr.name) 334 335 336 def _extract_expression(expr): 337 if expr.__class__ == compiler.ast.Const: 338 return _extract_const(expr) 339 if expr.__class__ == compiler.ast.Name: 340 return _extract_name(expr) 341 if expr.__class__ == compiler.ast.Dict: 342 return _extract_dict(expr) 343 if expr.__class__ == compiler.ast.List: 344 return _extract_list(expr) 345 raise ValueError('Unknown rval %s' % expr) 346 347 348 def _extract_assignment(n): 349 assert(n.__class__ == compiler.ast.Assign) 350 assert(n.nodes.__class__ == list) 351 assert(len(n.nodes) == 1) 352 assert(n.nodes[0].__class__ == compiler.ast.AssName) 353 assert(n.nodes[0].flags.__class__ == str) 354 assert(n.nodes[0].name.__class__ == str) 355 356 val = _extract_expression(n.expr) 357 key = n.nodes[0].name.lower() 358 359 return (key, val) 360 361 362 def parse_control_string(control, raise_warnings=False, path=''): 363 """Parse a control file from a string. 364 365 @param control: string containing the text of a control file. 366 @param raise_warnings: True iff ControlData should raise an error on 367 warnings about control file contents. 368 @param path: string path to the control file. 369 370 """ 371 try: 372 mod = compiler.parse(control) 373 except SyntaxError, e: 374 raise ControlVariableException("Error parsing data because %s" % e) 375 return finish_parse(mod, path, raise_warnings) 376 377 378 def parse_control(path, raise_warnings=False): 379 try: 380 mod = compiler.parseFile(path) 381 except SyntaxError, e: 382 raise ControlVariableException("Error parsing %s because %s" % 383 (path, e)) 384 return finish_parse(mod, path, raise_warnings) 385 386 387 def _try_extract_assignment(node, variables): 388 """Try to extract assignment from the given node. 389 390 @param node: An Assign object. 391 @param variables: Dictionary to store the parsed assignments. 392 """ 393 try: 394 key, val = _extract_assignment(node) 395 variables[key] = val 396 except (AssertionError, ValueError): 397 pass 398 399 400 def finish_parse(mod, path, raise_warnings): 401 assert(mod.__class__ == compiler.ast.Module) 402 assert(mod.node.__class__ == compiler.ast.Stmt) 403 assert(mod.node.nodes.__class__ == list) 404 405 variables = {} 406 for n in mod.node.nodes: 407 if (n.__class__ == compiler.ast.Function and 408 re.match('step\d+', n.name)): 409 vars_in_step = {} 410 for sub_node in n.code.nodes: 411 _try_extract_assignment(sub_node, vars_in_step) 412 if vars_in_step: 413 # Empty the vars collection so assignments from multiple steps 414 # won't be mixed. 415 variables.clear() 416 variables.update(vars_in_step) 417 else: 418 _try_extract_assignment(n, variables) 419 420 return ControlData(variables, path, raise_warnings) 421
