1 """\ 2 Logic for control file generation. 3 """ 4 5 __author__ = 'showard (at] google.com (Steve Howard)' 6 7 import re, os 8 9 import common 10 from autotest_lib.frontend.afe import model_logic 11 import frontend.settings 12 13 AUTOTEST_DIR = os.path.abspath(os.path.join( 14 os.path.dirname(frontend.settings.__file__), '..')) 15 16 EMPTY_TEMPLATE = 'def step_init():\n' 17 18 CLIENT_KERNEL_TEMPLATE = """\ 19 kernel_list = %(client_kernel_list)s 20 21 def step_init(): 22 for kernel_info in kernel_list: 23 job.next_step(boot_kernel, kernel_info) 24 job.next_step(step_test, kernel_info['version']) 25 if len(kernel_list) > 1: 26 job.use_sequence_number = True # include run numbers in directory names 27 28 29 def boot_kernel(kernel_info): 30 # remove kernels (and associated data) not referenced by the bootloader 31 for host in job.hosts: 32 host.cleanup_kernels() 33 34 testkernel = job.kernel(kernel_info['version']) 35 if kernel_info['config_file']: 36 testkernel.config(kernel_info['config_file']) 37 testkernel.build() 38 testkernel.install() 39 40 cmdline = ' '.join((kernel_info.get('cmdline', ''), '%(kernel_args)s')) 41 testkernel.boot(args=cmdline) 42 43 44 def step_test(kernel_version): 45 global kernel 46 kernel = kernel_version # Set the global in case anyone is using it. 47 if len(kernel_list) > 1: 48 # this is local to a machine, safe to assume there's only one host 49 host, = job.hosts 50 job.automatic_test_tag = host.get_kernel_ver() 51 """ 52 53 SERVER_KERNEL_TEMPLATE = """\ 54 kernel_list = %%(server_kernel_list)s 55 kernel_install_control = \""" 56 %s pass 57 \""" 58 59 from autotest_lib.client.common_lib import error 60 61 at = autotest.Autotest() 62 63 %%(upload_config_func)s 64 def install_kernel(machine, kernel_info): 65 host = hosts.create_host(machine) 66 at.install(host=host) 67 %%(call_upload_config)s 68 at.run(kernel_install_control %%%% 69 {'client_kernel_list': repr([kernel_info])}, host=host) 70 71 72 num_machines_required = len(machines) 73 if len(machines) > 4: 74 # Allow a large multi-host tests to proceed despite a couple of hosts 75 # failing to properly install the desired kernel (exclude those hosts). 76 # TODO(gps): Figure out how to get and use SYNC_COUNT here. It is defined 77 # within some control files and will end up inside of stepN functions below. 78 num_machines_required = len(machines) - 2 79 80 81 def step_init(): 82 # a host object we use solely for the purpose of finding out the booted 83 # kernel version, we use machines[0] since we already check that the same 84 # kernel has been booted on all machines 85 if len(kernel_list) > 1: 86 kernel_host = hosts.create_host(machines[0]) 87 88 for kernel_info in kernel_list: 89 func = lambda machine: install_kernel(machine, kernel_info) 90 good_machines = job.parallel_on_machines(func, machines) 91 if len(good_machines) < num_machines_required: 92 raise error.TestError( 93 "kernel installed on only %%%%d of %%%%d machines." 94 %%%% (len(good_machines), num_machines_required)) 95 96 # Replace the machines list that step_test() will use with the 97 # ones that successfully installed the kernel. 98 machines[:] = good_machines 99 100 # have server_job.run_test() automatically add the kernel version as 101 # a suffix to the test name otherwise we cannot run the same test on 102 # different kernel versions 103 if len(kernel_list) > 1: 104 job.automatic_test_tag = kernel_host.get_kernel_ver() 105 step_test() 106 107 108 def step_test(): 109 """ % CLIENT_KERNEL_TEMPLATE 110 111 CLIENT_STEP_TEMPLATE = " job.next_step('step%d')\n" 112 SERVER_STEP_TEMPLATE = ' step%d()\n' 113 114 UPLOAD_CONFIG_FUNC = """ 115 def upload_kernel_config(host, kernel_info): 116 \""" 117 If the kernel_info['config_file'] is a URL it will be downloaded 118 locally and then uploaded to the client and a copy of the original 119 dictionary with the new path to the config file will be returned. 120 If the config file is not a URL the function returns the original 121 dictionary. 122 \""" 123 import os 124 from autotest_lib.client.common_lib import autotemp, utils 125 126 config_orig = kernel_info.get('config_file') 127 128 # if the file is not an URL then we assume it's a local client path 129 if not config_orig or not utils.is_url(config_orig): 130 return kernel_info 131 132 # download it locally (on the server) and send it to the client 133 config_tmp = autotemp.tempfile('kernel_config_upload', dir=job.tmpdir) 134 try: 135 utils.urlretrieve(config_orig, config_tmp.name) 136 config_new = os.path.join(host.get_autodir(), 'tmp', 137 os.path.basename(config_orig)) 138 host.send_file(config_tmp.name, config_new) 139 finally: 140 config_tmp.clean() 141 142 return dict(kernel_info, config_file=config_new) 143 144 """ 145 146 CALL_UPLOAD_CONFIG = 'kernel_info = upload_kernel_config(host, kernel_info)' 147 148 149 def kernel_config_file(kernel, platform): 150 if (not kernel.endswith('.rpm') and platform and 151 platform.kernel_config): 152 return platform.kernel_config 153 return None 154 155 156 def read_control_file(test): 157 control_file = open(os.path.join(AUTOTEST_DIR, test.path)) 158 control_contents = control_file.read() 159 control_file.close() 160 return control_contents 161 162 163 def get_kernel_stanza(kernel_list, platform=None, kernel_args='', 164 is_server=False, upload_kernel_config=False): 165 166 template_args = {'kernel_args' : kernel_args} 167 168 # add 'config_file' keys to the kernel_info dictionaries 169 new_kernel_list = [] 170 for kernel_info in kernel_list: 171 if kernel_info.get('config_file'): 172 # already got a config file from the user 173 new_kernel_info = kernel_info 174 else: 175 config_file = kernel_config_file(kernel_info['version'], platform) 176 new_kernel_info = dict(kernel_info, config_file=config_file) 177 178 new_kernel_list.append(new_kernel_info) 179 180 if is_server: 181 template = SERVER_KERNEL_TEMPLATE 182 # leave client_kernel_list as a placeholder 183 template_args['client_kernel_list'] = '%(client_kernel_list)s' 184 template_args['server_kernel_list'] = repr(new_kernel_list) 185 186 if upload_kernel_config: 187 template_args['call_upload_config'] = CALL_UPLOAD_CONFIG 188 template_args['upload_config_func'] = UPLOAD_CONFIG_FUNC 189 else: 190 template_args['call_upload_config'] = '' 191 template_args['upload_config_func'] = '' 192 else: 193 template = CLIENT_KERNEL_TEMPLATE 194 template_args['client_kernel_list'] = repr(new_kernel_list) 195 196 return template % template_args 197 198 199 def add_boilerplate_to_nested_steps(lines): 200 # Look for a line that begins with 'def step_init():' while 201 # being flexible on spacing. If it's found, this will be 202 # a nested set of steps, so add magic to make it work. 203 # See client/bin/job.py's step_engine for more info. 204 if re.search(r'^(.*\n)*def\s+step_init\s*\(\s*\)\s*:', lines): 205 lines += '\nreturn locals() ' 206 lines += '# Boilerplate magic for nested sets of steps' 207 return lines 208 209 210 def format_step(item, lines): 211 lines = indent_text(lines, ' ') 212 lines = 'def step%d():\n%s' % (item, lines) 213 return lines 214 215 216 def get_tests_stanza(tests, is_server, prepend=None, append=None, 217 client_control_file=''): 218 """ Constructs the control file test step code from a list of tests. 219 220 @param tests A sequence of test control files to run. 221 @param is_server bool, Is this a server side test? 222 @param prepend A list of steps to prepend to each client test. 223 Defaults to []. 224 @param append A list of steps to append to each client test. 225 Defaults to []. 226 @param client_control_file If specified, use this text as the body of a 227 final client control file to run after tests. is_server must be False. 228 229 @returns The control file test code to be run. 230 """ 231 assert not (client_control_file and is_server) 232 if not prepend: 233 prepend = [] 234 if not append: 235 append = [] 236 raw_control_files = [read_control_file(test) for test in tests] 237 return _get_tests_stanza(raw_control_files, is_server, prepend, append, 238 client_control_file=client_control_file) 239 240 241 def _get_tests_stanza(raw_control_files, is_server, prepend, append, 242 client_control_file=''): 243 """ 244 Implements the common parts of get_test_stanza. 245 246 A site_control_file that wants to implement its own get_tests_stanza 247 likely wants to call this in the end. 248 249 @param raw_control_files A list of raw control file data to be combined 250 into a single control file. 251 @param is_server bool, Is this a server side test? 252 @param prepend A list of steps to prepend to each client test. 253 @param append A list of steps to append to each client test. 254 @param client_control_file If specified, use this text as the body of a 255 final client control file to append to raw_control_files after fixups. 256 257 @returns The combined mega control file. 258 """ 259 if client_control_file: 260 # 'return locals()' is always appended incase the user forgot, it 261 # is necessary to allow for nested step engine execution to work. 262 raw_control_files.append(client_control_file + '\nreturn locals()') 263 raw_steps = prepend + [add_boilerplate_to_nested_steps(step) 264 for step in raw_control_files] + append 265 steps = [format_step(index, step) 266 for index, step in enumerate(raw_steps)] 267 if is_server: 268 step_template = SERVER_STEP_TEMPLATE 269 footer = '\n\nstep_init()\n' 270 else: 271 step_template = CLIENT_STEP_TEMPLATE 272 footer = '' 273 274 header = ''.join(step_template % i for i in xrange(len(steps))) 275 return header + '\n' + '\n\n'.join(steps) + footer 276 277 278 def indent_text(text, indent): 279 """Indent given lines of python code avoiding indenting multiline 280 quoted content (only for triple " and ' quoting for now).""" 281 regex = re.compile('(\\\\*)("""|\'\'\')') 282 283 res = [] 284 in_quote = None 285 for line in text.splitlines(): 286 # if not within a multinline quote indent the line contents 287 if in_quote: 288 res.append(line) 289 else: 290 res.append(indent + line) 291 292 while line: 293 match = regex.search(line) 294 if match: 295 # for an even number of backslashes before the triple quote 296 if len(match.group(1)) % 2 == 0: 297 if not in_quote: 298 in_quote = match.group(2)[0] 299 elif in_quote == match.group(2)[0]: 300 # if we found a matching end triple quote 301 in_quote = None 302 line = line[match.end():] 303 else: 304 break 305 306 return '\n'.join(res) 307 308 309 def _get_profiler_commands(profilers, is_server, profile_only): 310 prepend, append = [], [] 311 if profile_only is not None: 312 prepend.append("job.default_profile_only = %r" % profile_only) 313 for profiler in profilers: 314 prepend.append("job.profilers.add('%s')" % profiler.name) 315 append.append("job.profilers.delete('%s')" % profiler.name) 316 return prepend, append 317 318 319 def _sanity_check_generate_control(is_server, client_control_file, kernels, 320 upload_kernel_config): 321 """ 322 Sanity check some of the parameters to generate_control(). 323 324 This exists as its own function so that site_control_file may call it as 325 well from its own generate_control(). 326 327 @raises ValidationError if any of the parameters do not make sense. 328 """ 329 if is_server and client_control_file: 330 raise model_logic.ValidationError( 331 {'tests' : 'You cannot run server tests at the same time ' 332 'as directly supplying a client-side control file.'}) 333 334 if kernels: 335 # make sure that kernel is a list of dictionarions with at least 336 # the 'version' key in them 337 kernel_error = model_logic.ValidationError( 338 {'kernel': 'The kernel parameter must be a sequence of ' 339 'dictionaries containing at least the "version" key ' 340 '(got: %r)' % kernels}) 341 try: 342 iter(kernels) 343 except TypeError: 344 raise kernel_error 345 for kernel_info in kernels: 346 if (not isinstance(kernel_info, dict) or 347 'version' not in kernel_info): 348 raise kernel_error 349 350 if upload_kernel_config and not is_server: 351 raise model_logic.ValidationError( 352 {'upload_kernel_config': 'Cannot use upload_kernel_config ' 353 'with client side tests'}) 354 355 356 def generate_control(tests, kernels=None, platform=None, is_server=False, 357 profilers=(), client_control_file='', profile_only=None, 358 upload_kernel_config=False): 359 """ 360 Generate a control file for a sequence of tests. 361 362 @param tests A sequence of test control files to run. 363 @param kernels A sequence of kernel info dictionaries configuring which 364 kernels to boot for this job and other options for them 365 @param platform A platform object with a kernel_config attribute. 366 @param is_server bool, Is this a server control file rather than a client? 367 @param profilers A list of profiler objects to enable during the tests. 368 @param client_control_file Contents of a client control file to run as the 369 last test after everything in tests. Requires is_server=False. 370 @param profile_only bool, should this control file run all tests in 371 profile_only mode by default 372 @param upload_kernel_config: if enabled it will generate server control 373 file code that uploads the kernel config file to the client and 374 tells the client of the new (local) path when compiling the kernel; 375 the tests must be server side tests 376 377 @returns The control file text as a string. 378 """ 379 _sanity_check_generate_control(is_server=is_server, kernels=kernels, 380 client_control_file=client_control_file, 381 upload_kernel_config=upload_kernel_config) 382 383 control_file_text = '' 384 if kernels: 385 control_file_text = get_kernel_stanza( 386 kernels, platform, is_server=is_server, 387 upload_kernel_config=upload_kernel_config) 388 else: 389 control_file_text = EMPTY_TEMPLATE 390 391 prepend, append = _get_profiler_commands(profilers, is_server, profile_only) 392 393 control_file_text += get_tests_stanza(tests, is_server, prepend, append, 394 client_control_file) 395 return control_file_text 396
