1 """\ 2 Logic for control file generation. 3 """ 4 5 __author__ = 'showard (at] google.com (Steve Howard)' 6 7 import re, os 8 9 import common 10 from autotest_lib.client.common_lib import error 11 from autotest_lib.client.common_lib.cros import dev_server 12 from autotest_lib.frontend.afe import model_logic 13 from autotest_lib.server.cros.dynamic_suite import control_file_getter 14 from autotest_lib.server.cros.dynamic_suite import suite_common 15 import frontend.settings 16 17 AUTOTEST_DIR = os.path.abspath(os.path.join( 18 os.path.dirname(frontend.settings.__file__), '..')) 19 20 EMPTY_TEMPLATE = 'def step_init():\n' 21 22 CLIENT_STEP_TEMPLATE = " job.next_step('step%d')\n" 23 SERVER_STEP_TEMPLATE = ' step%d()\n' 24 25 26 def _read_control_file(test): 27 """Reads the test control file from local disk. 28 29 @param test The test name. 30 31 @return The test control file string. 32 """ 33 control_file = open(os.path.join(AUTOTEST_DIR, test.path)) 34 control_contents = control_file.read() 35 control_file.close() 36 return control_contents 37 38 39 def _add_boilerplate_to_nested_steps(lines): 40 """Adds boilerplate magic. 41 42 @param lines The string of lines. 43 44 @returns The string lines. 45 """ 46 # Look for a line that begins with 'def step_init():' while 47 # being flexible on spacing. If it's found, this will be 48 # a nested set of steps, so add magic to make it work. 49 # See client/bin/job.py's step_engine for more info. 50 if re.search(r'^(.*\n)*def\s+step_init\s*\(\s*\)\s*:', lines): 51 lines += '\nreturn locals() ' 52 lines += '# Boilerplate magic for nested sets of steps' 53 return lines 54 55 56 def _format_step(item, lines): 57 """Format a line item. 58 @param item The item number. 59 @param lines The string of lines. 60 61 @returns The string lines. 62 """ 63 lines = _indent_text(lines, ' ') 64 lines = 'def step%d():\n%s' % (item, lines) 65 return lines 66 67 68 def _get_tests_stanza(tests, is_server, prepend=None, append=None, 69 client_control_file='', test_source_build=None): 70 """ Constructs the control file test step code from a list of tests. 71 72 @param tests A sequence of test control files to run. 73 @param is_server bool, Is this a server side test? 74 @param prepend A list of steps to prepend to each client test. 75 Defaults to []. 76 @param append A list of steps to append to each client test. 77 Defaults to []. 78 @param client_control_file If specified, use this text as the body of a 79 final client control file to run after tests. is_server must be False. 80 @param test_source_build: Build to be used to retrieve test code. Default 81 to None. 82 83 @returns The control file test code to be run. 84 """ 85 assert not (client_control_file and is_server) 86 if not prepend: 87 prepend = [] 88 if not append: 89 append = [] 90 if test_source_build: 91 raw_control_files = _get_test_control_files_by_build( 92 tests, test_source_build) 93 else: 94 raw_control_files = [_read_control_file(test) for test in tests] 95 if client_control_file: 96 # 'return locals()' is always appended in case the user forgot, it 97 # is necessary to allow for nested step engine execution to work. 98 raw_control_files.append(client_control_file + '\nreturn locals()') 99 raw_steps = prepend + [_add_boilerplate_to_nested_steps(step) 100 for step in raw_control_files] + append 101 steps = [_format_step(index, step) 102 for index, step in enumerate(raw_steps)] 103 if is_server: 104 step_template = SERVER_STEP_TEMPLATE 105 footer = '\n\nstep_init()\n' 106 else: 107 step_template = CLIENT_STEP_TEMPLATE 108 footer = '' 109 110 header = ''.join(step_template % i for i in xrange(len(steps))) 111 return header + '\n' + '\n\n'.join(steps) + footer 112 113 114 def _indent_text(text, indent): 115 """Indent given lines of python code avoiding indenting multiline 116 quoted content (only for triple " and ' quoting for now). 117 118 @param text The string of lines. 119 @param indent The indent string. 120 121 @return The indented string. 122 """ 123 regex = re.compile('(\\\\*)("""|\'\'\')') 124 125 res = [] 126 in_quote = None 127 for line in text.splitlines(): 128 # if not within a multinline quote indent the line contents 129 if in_quote: 130 res.append(line) 131 else: 132 res.append(indent + line) 133 134 while line: 135 match = regex.search(line) 136 if match: 137 # for an even number of backslashes before the triple quote 138 if len(match.group(1)) % 2 == 0: 139 if not in_quote: 140 in_quote = match.group(2)[0] 141 elif in_quote == match.group(2)[0]: 142 # if we found a matching end triple quote 143 in_quote = None 144 line = line[match.end():] 145 else: 146 break 147 148 return '\n'.join(res) 149 150 151 def _get_profiler_commands(profilers, is_server, profile_only): 152 prepend, append = [], [] 153 if profile_only is not None: 154 prepend.append("job.default_profile_only = %r" % profile_only) 155 for profiler in profilers: 156 prepend.append("job.profilers.add('%s')" % profiler.name) 157 append.append("job.profilers.delete('%s')" % profiler.name) 158 return prepend, append 159 160 161 def _sanity_check_generate_control(is_server, client_control_file): 162 """ 163 Sanity check some of the parameters to generate_control(). 164 165 This exists as its own function so that site_control_file may call it as 166 well from its own generate_control(). 167 168 @raises ValidationError if any of the parameters do not make sense. 169 """ 170 if is_server and client_control_file: 171 raise model_logic.ValidationError( 172 {'tests' : 'You cannot run server tests at the same time ' 173 'as directly supplying a client-side control file.'}) 174 175 176 def generate_control(tests, is_server=False, profilers=(), 177 client_control_file='', profile_only=None, 178 test_source_build=None): 179 """ 180 Generate a control file for a sequence of tests. 181 182 @param tests A sequence of test control files to run. 183 @param is_server bool, Is this a server control file rather than a client? 184 @param profilers A list of profiler objects to enable during the tests. 185 @param client_control_file Contents of a client control file to run as the 186 last test after everything in tests. Requires is_server=False. 187 @param profile_only bool, should this control file run all tests in 188 profile_only mode by default 189 @param test_source_build: Build to be used to retrieve test code. Default 190 to None. 191 192 @returns The control file text as a string. 193 """ 194 _sanity_check_generate_control(is_server=is_server, 195 client_control_file=client_control_file) 196 control_file_text = EMPTY_TEMPLATE 197 prepend, append = _get_profiler_commands(profilers, is_server, profile_only) 198 control_file_text += _get_tests_stanza(tests, is_server, prepend, append, 199 client_control_file, 200 test_source_build) 201 return control_file_text 202 203 204 def _get_test_control_files_by_build(tests, build, ignore_invalid_tests=False): 205 """Get the test control files that are available for the specified build. 206 207 @param tests A sequence of test objects to run. 208 @param build: unique name by which to refer to the image. 209 @param ignore_invalid_tests: flag on if unparsable tests are ignored. 210 211 @return: A sorted list of all tests that are in the build specified. 212 """ 213 raw_control_files = [] 214 # shortcut to avoid staging the image. 215 if not tests: 216 return raw_control_files 217 218 cfile_getter = _initialize_control_file_getter(build) 219 if suite_common.ENABLE_CONTROLS_IN_BATCH: 220 control_file_info_list = cfile_getter.get_suite_info() 221 222 for test in tests: 223 # Read and parse the control file 224 if suite_common.ENABLE_CONTROLS_IN_BATCH: 225 control_file = control_file_info_list[test.path] 226 else: 227 control_file = cfile_getter.get_control_file_contents( 228 test.path) 229 raw_control_files.append(control_file) 230 return raw_control_files 231 232 233 def _initialize_control_file_getter(build): 234 """Get the remote control file getter. 235 236 @param build: unique name by which to refer to a remote build image. 237 238 @return: A control file getter object. 239 """ 240 # Stage the test artifacts. 241 try: 242 ds = dev_server.ImageServer.resolve(build) 243 ds_name = ds.hostname 244 build = ds.translate(build) 245 except dev_server.DevServerException as e: 246 raise ValueError('Could not resolve build %s: %s' % 247 (build, e)) 248 249 try: 250 ds.stage_artifacts(image=build, artifacts=['test_suites']) 251 except dev_server.DevServerException as e: 252 raise error.StageControlFileFailure( 253 'Failed to stage %s on %s: %s' % (build, ds_name, e)) 254 255 # Collect the control files specified in this build 256 return control_file_getter.DevServerGetter.create(build, ds) 257