1 #!/usr/bin/env python 2 3 # Copyright (c) 2014 The Chromium OS Authors. All rights reserved. 4 # Use of this source code is governed by a BSD-style license that can be 5 # found in the LICENSE file. 6 7 # This script is used to compare the performance of duts when running the same 8 # test/special task. For example: 9 # 10 # python compare_dut_perf.py -l 240 --board stumpy 11 # 12 # compares the test runtime of all stumpy for the last 10 days. Sample output: 13 # ============================================================================== 14 # Test hardware_MemoryTotalSize 15 # ============================================================================== 16 # chromeos2-row2-rack8-host8 : min= 479, max= 479, mean= 479, med= 479, cnt= 1 17 # chromeos2-row2-rack8-host12 : min= 440, max= 440, mean= 440, med= 440, cnt= 1 18 # chromeos2-row2-rack8-host11 : min= 504, max= 504, mean= 504, med= 504, cnt= 1 19 # 20 # At the end of each row, it also lists the last 5 jobs running in the dut. 21 22 23 import argparse 24 import datetime 25 import multiprocessing.pool 26 import pprint 27 import time 28 from itertools import groupby 29 30 import common 31 import numpy 32 from autotest_lib.frontend import setup_django_environment 33 from autotest_lib.frontend.afe import models 34 from autotest_lib.frontend.afe import rpc_utils 35 from autotest_lib.frontend.tko import models as tko_models 36 from autotest_lib.server.cros.dynamic_suite import frontend_wrappers 37 38 39 def get_matched_duts(hostnames=None, board=None, pool=None, other_labels=None): 40 """Get duts with matching board and pool labels from given autotest instance 41 42 @param hostnames: A list of hostnames. 43 @param board: board of DUT, set to None if board doesn't need to match. 44 Default is None. 45 @param pool: pool of DUT, set to None if pool doesn't need to match. Default 46 is None. 47 @param other_labels: Other labels to filter duts. 48 @return: A list of duts that match the specified board and pool. 49 """ 50 if hostnames: 51 hosts = models.Host.objects.filter(hostname__in=hostnames) 52 else: 53 multiple_labels = () 54 if pool: 55 multiple_labels += ('pool:%s' % pool,) 56 if board: 57 multiple_labels += ('board:%s' % board,) 58 if other_labels: 59 for label in other_labels: 60 multiple_labels += (label,) 61 hosts = rpc_utils.get_host_query(multiple_labels, 62 exclude_only_if_needed_labels=False, 63 exclude_atomic_group_hosts=False, 64 valid_only=True, filter_data={}) 65 return [host_obj.get_object_dict() for host_obj in hosts] 66 67 68 def get_job_runtime(input): 69 """Get all test jobs and special tasks' runtime for a given host during 70 a give time period. 71 72 @param input: input arguments, including: 73 start_time: Start time of the search interval. 74 end_time: End time of the search interval. 75 host_id: id of the dut. 76 hostname: Name of the dut. 77 @return: A list of records, e.g., 78 [{'job_name':'dummy_Pass', 'time_used': 3, 'id': 12313, 79 'hostname': '1.2.3.4'}, 80 {'task_name':'Cleanup', 'time_used': 30, 'id': 5687, 81 'hostname': '1.2.3.4'}] 82 """ 83 start_time = input['start_time'] 84 end_time = input['end_time'] 85 host_id = input['host_id'] 86 hostname = input['hostname'] 87 records = [] 88 special_tasks = models.SpecialTask.objects.filter( 89 host_id=host_id, 90 time_started__gte=start_time, 91 time_started__lte=end_time, 92 time_started__isnull=False, 93 time_finished__isnull=False).values('task', 'id', 'time_started', 94 'time_finished') 95 for task in special_tasks: 96 time_used = task['time_finished'] - task['time_started'] 97 records.append({'name': task['task'], 98 'id': task['id'], 99 'time_used': time_used.total_seconds(), 100 'hostname': hostname}) 101 hqes = models.HostQueueEntry.objects.filter( 102 host_id=host_id, 103 started_on__gte=start_time, 104 started_on__lte=end_time, 105 started_on__isnull=False, 106 finished_on__isnull=False) 107 for hqe in hqes: 108 time_used = (hqe.finished_on - hqe.started_on).total_seconds() 109 records.append({'name': hqe.job.name.split('/')[-1], 110 'id': hqe.job.id, 111 'time_used': time_used, 112 'hostname': hostname}) 113 return records 114 115 def get_job_stats(jobs): 116 """Get the stats of a list of jobs. 117 118 @param jobs: A list of jobs. 119 @return: Stats of the jobs' runtime, including: 120 t_min: minimum runtime. 121 t_max: maximum runtime. 122 t_average: average runtime. 123 t_median: median runtime. 124 """ 125 runtimes = [job['time_used'] for job in jobs] 126 t_min = min(runtimes) 127 t_max = max(runtimes) 128 t_mean = numpy.mean(runtimes) 129 t_median = numpy.median(runtimes) 130 return t_min, t_max, t_mean, t_median, len(runtimes) 131 132 133 def process_results(results): 134 """Compare the results. 135 136 @param results: A list of a list of job/task information. 137 """ 138 # Merge list of all results. 139 all_results = [] 140 for result in results: 141 all_results.extend(result) 142 all_results = sorted(all_results, key=lambda r: r['name']) 143 for name,jobs_for_test in groupby(all_results, lambda r: r['name']): 144 print '='*80 145 print 'Test %s' % name 146 print '='*80 147 for hostname,jobs_for_dut in groupby(jobs_for_test, 148 lambda j: j['hostname']): 149 jobs = list(jobs_for_dut) 150 t_min, t_max, t_mean, t_median, count = get_job_stats(jobs) 151 ids = [str(job['id']) for job in jobs] 152 print ('%-28s: min= %-3.0f max= %-3.0f mean= %-3.0f med= %-3.0f ' 153 'cnt= %-3s IDs: %s' % 154 (hostname, t_min, t_max, t_mean, t_median, count, 155 ','.join(sorted(ids)[-5:]))) 156 157 158 def main(): 159 """main script. """ 160 t_now = time.time() 161 t_now_minus_one_day = t_now - 3600 * 24 162 parser = argparse.ArgumentParser() 163 parser.add_argument('-l', type=float, dest='last', 164 help='last hours to search results across', 165 default=24) 166 parser.add_argument('--board', type=str, dest='board', 167 help='restrict query by board', 168 default=None) 169 parser.add_argument('--pool', type=str, dest='pool', 170 help='restrict query by pool', 171 default=None) 172 parser.add_argument('--hosts', nargs='+', dest='hosts', 173 help='Enter space deliminated hostnames', 174 default=[]) 175 parser.add_argument('--start', type=str, dest='start', 176 help=('Enter start time as: yyyy-mm-dd hh-mm-ss,' 177 'defualts to 24h ago.')) 178 parser.add_argument('--end', type=str, dest='end', 179 help=('Enter end time in as: yyyy-mm-dd hh-mm-ss,' 180 'defualts to current time.')) 181 options = parser.parse_args() 182 183 if not options.start or not options.end: 184 end_time = datetime.datetime.now() 185 start_time = end_time - datetime.timedelta(seconds=3600 * options.last) 186 else: 187 start_time = time_utils.time_string_to_datetime(options.start) 188 end_time = time_utils.time_string_to_datetime(options.end) 189 190 hosts = get_matched_duts(hostnames=options.hosts, board=options.board, 191 pool=options.pool) 192 if not hosts: 193 raise Exception('No host found to search for history.') 194 print 'Found %d duts.' % len(hosts) 195 print 'Start time: %s' % start_time 196 print 'End time: %s' % end_time 197 args = [] 198 for host in hosts: 199 args.append({'start_time': start_time, 200 'end_time': end_time, 201 'host_id': host['id'], 202 'hostname': host['hostname']}) 203 get_job_runtime(args[0]) 204 # Parallizing this process. 205 pool = multiprocessing.pool.ThreadPool() 206 results = pool.imap_unordered(get_job_runtime, args) 207 process_results(results) 208 209 210 if __name__ == '__main__': 211 main() 212
