1 #!/usr/bin/python 2 # 3 # Copyright 2010 Google Inc. All Rights Reserved. 4 5 import datetime 6 import optparse 7 import os 8 import smtplib 9 import sys 10 import time 11 from email.mime.text import MIMEText 12 13 from autotest_gatherer import AutotestGatherer as AutotestGatherer 14 from autotest_run import AutotestRun as AutotestRun 15 from machine_manager_singleton import MachineManagerSingleton as MachineManagerSingleton 16 from cros_utils import logger 17 from cros_utils.file_utils import FileUtils 18 19 20 def CanonicalizeChromeOSRoot(chromeos_root): 21 chromeos_root = os.path.expanduser(chromeos_root) 22 if os.path.isfile(os.path.join(chromeos_root, 'src/scripts/enter_chroot.sh')): 23 return chromeos_root 24 else: 25 return None 26 27 28 class Autotest(object): 29 30 def __init__(self, autotest_string): 31 self.name = None 32 self.iterations = None 33 self.args = None 34 fields = autotest_string.split(',', 1) 35 self.name = fields[0] 36 if len(fields) > 1: 37 autotest_string = fields[1] 38 fields = autotest_string.split(',', 1) 39 else: 40 return 41 self.iterations = int(fields[0]) 42 if len(fields) > 1: 43 self.args = fields[1] 44 else: 45 return 46 47 def __str__(self): 48 return '\n'.join([self.name, self.iterations, self.args]) 49 50 51 def CreateAutotestListFromString(autotest_strings, default_iterations=None): 52 autotest_list = [] 53 for autotest_string in autotest_strings.split(':'): 54 autotest = Autotest(autotest_string) 55 if default_iterations and not autotest.iterations: 56 autotest.iterations = default_iterations 57 58 autotest_list.append(autotest) 59 return autotest_list 60 61 62 def CreateAutotestRuns(images, 63 autotests, 64 remote, 65 board, 66 exact_remote, 67 rerun, 68 rerun_if_failed, 69 main_chromeos_root=None): 70 autotest_runs = [] 71 for image in images: 72 logger.GetLogger().LogOutput('Computing md5sum of: %s' % image) 73 image_checksum = FileUtils().Md5File(image) 74 logger.GetLogger().LogOutput('md5sum %s: %s' % (image, image_checksum)) 75 ### image_checksum = "abcdefghi" 76 77 chromeos_root = main_chromeos_root 78 if not main_chromeos_root: 79 image_chromeos_root = os.path.join( 80 os.path.dirname(image), '../../../../..') 81 chromeos_root = CanonicalizeChromeOSRoot(image_chromeos_root) 82 assert chromeos_root, 'chromeos_root: %s invalid' % image_chromeos_root 83 else: 84 chromeos_root = CanonicalizeChromeOSRoot(main_chromeos_root) 85 assert chromeos_root, 'chromeos_root: %s invalid' % main_chromeos_root 86 87 # We just need a single ChromeOS root in the MachineManagerSingleton. It is 88 # needed because we can save re-image time by checking the image checksum at 89 # the beginning and assigning autotests to machines appropriately. 90 if not MachineManagerSingleton().chromeos_root: 91 MachineManagerSingleton().chromeos_root = chromeos_root 92 93 for autotest in autotests: 94 for iteration in range(autotest.iterations): 95 autotest_run = AutotestRun(autotest, 96 chromeos_root=chromeos_root, 97 chromeos_image=image, 98 board=board, 99 remote=remote, 100 iteration=iteration, 101 image_checksum=image_checksum, 102 exact_remote=exact_remote, 103 rerun=rerun, 104 rerun_if_failed=rerun_if_failed) 105 autotest_runs.append(autotest_run) 106 return autotest_runs 107 108 109 def GetNamesAndIterations(autotest_runs): 110 strings = [] 111 for autotest_run in autotest_runs: 112 strings.append('%s:%s' % (autotest_run.autotest.name, 113 autotest_run.iteration)) 114 return ' %s (%s)' % (len(strings), ' '.join(strings)) 115 116 117 def GetStatusString(autotest_runs): 118 status_bins = {} 119 for autotest_run in autotest_runs: 120 if autotest_run.status not in status_bins: 121 status_bins[autotest_run.status] = [] 122 status_bins[autotest_run.status].append(autotest_run) 123 124 status_strings = [] 125 for key, val in status_bins.items(): 126 status_strings.append('%s: %s' % (key, GetNamesAndIterations(val))) 127 return 'Thread Status:\n%s' % '\n'.join(status_strings) 128 129 130 def GetProgressBar(num_done, num_total): 131 ret = 'Done: %s%%' % int(100.0 * num_done / num_total) 132 bar_length = 50 133 done_char = '>' 134 undone_char = ' ' 135 num_done_chars = bar_length * num_done / num_total 136 num_undone_chars = bar_length - num_done_chars 137 ret += ' [%s%s]' % (num_done_chars * done_char, 138 num_undone_chars * undone_char) 139 return ret 140 141 142 def GetProgressString(start_time, num_remain, num_total): 143 current_time = time.time() 144 elapsed_time = current_time - start_time 145 try: 146 eta_seconds = float(num_remain) * elapsed_time / (num_total - num_remain) 147 eta_seconds = int(eta_seconds) 148 eta = datetime.timedelta(seconds=eta_seconds) 149 except ZeroDivisionError: 150 eta = 'Unknown' 151 strings = [] 152 strings.append('Current time: %s Elapsed: %s ETA: %s' % 153 (datetime.datetime.now(), 154 datetime.timedelta(seconds=int(elapsed_time)), eta)) 155 strings.append(GetProgressBar(num_total - num_remain, num_total)) 156 return '\n'.join(strings) 157 158 159 def RunAutotestRunsInParallel(autotest_runs): 160 start_time = time.time() 161 active_threads = [] 162 for autotest_run in autotest_runs: 163 # Set threads to daemon so program exits when ctrl-c is pressed. 164 autotest_run.daemon = True 165 autotest_run.start() 166 active_threads.append(autotest_run) 167 168 print_interval = 30 169 last_printed_time = time.time() 170 while active_threads: 171 try: 172 active_threads = [t for t in active_threads 173 if t is not None and t.isAlive()] 174 for t in active_threads: 175 t.join(1) 176 if time.time() - last_printed_time > print_interval: 177 border = '==============================' 178 logger.GetLogger().LogOutput(border) 179 logger.GetLogger().LogOutput(GetProgressString(start_time, len( 180 [t for t in autotest_runs if t.status not in ['SUCCEEDED', 'FAILED'] 181 ]), len(autotest_runs))) 182 logger.GetLogger().LogOutput(GetStatusString(autotest_runs)) 183 logger.GetLogger().LogOutput('%s\n' % 184 MachineManagerSingleton().AsString()) 185 logger.GetLogger().LogOutput(border) 186 last_printed_time = time.time() 187 except KeyboardInterrupt: 188 print 'C-c received... cleaning up threads.' 189 for t in active_threads: 190 t.terminate = True 191 return 1 192 return 0 193 194 195 def RunAutotestRunsSerially(autotest_runs): 196 for autotest_run in autotest_runs: 197 retval = autotest_run.Run() 198 if retval: 199 return retval 200 201 202 def ProduceTables(autotest_runs, full_table, fit_string): 203 l = logger.GetLogger() 204 ags_dict = {} 205 for autotest_run in autotest_runs: 206 name = autotest_run.full_name 207 if name not in ags_dict: 208 ags_dict[name] = AutotestGatherer() 209 ags_dict[name].runs.append(autotest_run) 210 output = '' 211 for b, ag in ags_dict.items(): 212 output += 'Benchmark: %s\n' % b 213 output += ag.GetFormattedMainTable(percents_only=not full_table, 214 fit_string=fit_string) 215 output += '\n' 216 217 summary = '' 218 for b, ag in ags_dict.items(): 219 summary += 'Benchmark Summary Table: %s\n' % b 220 summary += ag.GetFormattedSummaryTable(percents_only=not full_table, 221 fit_string=fit_string) 222 summary += '\n' 223 224 output += summary 225 output += ('Number of re-images performed: %s' % 226 MachineManagerSingleton().num_reimages) 227 l.LogOutput(output) 228 229 if autotest_runs: 230 board = autotest_runs[0].board 231 else: 232 board = '' 233 234 subject = '%s: %s' % (board, ', '.join(ags_dict.keys())) 235 236 if any(autotest_run.run_completed for autotest_run in autotest_runs): 237 SendEmailToUser(subject, summary) 238 239 240 def SendEmailToUser(subject, text_to_send): 241 # Email summary to the current user. 242 msg = MIMEText(text_to_send) 243 244 # me == the sender's email address 245 # you == the recipient's email address 246 me = os.path.basename(__file__) 247 you = os.getlogin() 248 msg['Subject'] = '[%s] %s' % (os.path.basename(__file__), subject) 249 msg['From'] = me 250 msg['To'] = you 251 252 # Send the message via our own SMTP server, but don't include the 253 # envelope header. 254 s = smtplib.SMTP('localhost') 255 s.sendmail(me, [you], msg.as_string()) 256 s.quit() 257 258 259 def Main(argv): 260 """The main function.""" 261 # Common initializations 262 ### command_executer.InitCommandExecuter(True) 263 l = logger.GetLogger() 264 265 parser = optparse.OptionParser() 266 parser.add_option('-t', 267 '--tests', 268 dest='tests', 269 help=('Tests to compare.' 270 'Optionally specify per-test iterations by:' 271 '<test>,<iter>:<args>')) 272 parser.add_option('-c', 273 '--chromeos_root', 274 dest='chromeos_root', 275 help='A *single* chromeos_root where scripts can be found.') 276 parser.add_option('-n', 277 '--iterations', 278 dest='iterations', 279 help='Iterations to run per benchmark.', 280 default=1) 281 parser.add_option('-r', 282 '--remote', 283 dest='remote', 284 help='The remote chromeos machine.') 285 parser.add_option('-b', '--board', dest='board', help='The remote board.') 286 parser.add_option('--full_table', 287 dest='full_table', 288 help='Print full tables.', 289 action='store_true', 290 default=True) 291 parser.add_option('--exact_remote', 292 dest='exact_remote', 293 help='Run tests on the exact remote.', 294 action='store_true', 295 default=False) 296 parser.add_option('--fit_string', 297 dest='fit_string', 298 help='Fit strings to fixed sizes.', 299 action='store_true', 300 default=False) 301 parser.add_option('--rerun', 302 dest='rerun', 303 help='Re-run regardless of cache hit.', 304 action='store_true', 305 default=False) 306 parser.add_option('--rerun_if_failed', 307 dest='rerun_if_failed', 308 help='Re-run if previous run was a failure.', 309 action='store_true', 310 default=False) 311 parser.add_option('--no_lock', 312 dest='no_lock', 313 help='Do not lock the machine before running the tests.', 314 action='store_true', 315 default=False) 316 l.LogOutput(' '.join(argv)) 317 [options, args] = parser.parse_args(argv) 318 319 if options.remote is None: 320 l.LogError('No remote machine specified.') 321 parser.print_help() 322 return 1 323 324 if not options.board: 325 l.LogError('No board specified.') 326 parser.print_help() 327 return 1 328 329 remote = options.remote 330 tests = options.tests 331 board = options.board 332 exact_remote = options.exact_remote 333 iterations = int(options.iterations) 334 335 autotests = CreateAutotestListFromString(tests, iterations) 336 337 main_chromeos_root = options.chromeos_root 338 images = args[1:] 339 fit_string = options.fit_string 340 full_table = options.full_table 341 rerun = options.rerun 342 rerun_if_failed = options.rerun_if_failed 343 344 MachineManagerSingleton().no_lock = options.no_lock 345 346 # Now try creating all the Autotests 347 autotest_runs = CreateAutotestRuns(images, autotests, remote, board, 348 exact_remote, rerun, rerun_if_failed, 349 main_chromeos_root) 350 351 try: 352 # At this point we have all the autotest runs. 353 for machine in remote.split(','): 354 MachineManagerSingleton().AddMachine(machine) 355 356 retval = RunAutotestRunsInParallel(autotest_runs) 357 if retval: 358 return retval 359 360 # Now print tables 361 ProduceTables(autotest_runs, full_table, fit_string) 362 finally: 363 # not sure why this isn't called at the end normally... 364 MachineManagerSingleton().__del__() 365 366 return 0 367 368 369 if __name__ == '__main__': 370 sys.exit(Main(sys.argv)) 371
