Home | History | Annotate | Download | only in contrib
      1 #!/usr/bin/env python
      2 
      3 # Copyright (c) 2014 The Chromium OS Authors. All rights reserved.
      4 # Use of this source code is governed by a BSD-style license that can be
      5 # found in the LICENSE file.
      6 
      7 # This script is used to compare the performance of duts when running the same
      8 # test/special task. For example:
      9 #
     10 # python compare_dut_perf.py -l 240 --board stumpy
     11 #
     12 # compares the test runtime of all stumpy for the last 10 days. Sample output:
     13 # ==============================================================================
     14 # Test hardware_MemoryTotalSize
     15 # ==============================================================================
     16 # chromeos2-row2-rack8-host8  : min= 479, max= 479, mean= 479, med= 479, cnt= 1
     17 # chromeos2-row2-rack8-host12 : min= 440, max= 440, mean= 440, med= 440, cnt= 1
     18 # chromeos2-row2-rack8-host11 : min= 504, max= 504, mean= 504, med= 504, cnt= 1
     19 #
     20 # At the end of each row, it also lists the last 5 jobs running in the dut.
     21 
     22 
     23 import argparse
     24 import datetime
     25 import multiprocessing.pool
     26 import pprint
     27 import time
     28 from itertools import groupby
     29 
     30 import common
     31 import numpy
     32 from autotest_lib.frontend import setup_django_environment
     33 from autotest_lib.frontend.afe import models
     34 from autotest_lib.frontend.afe import rpc_utils
     35 from autotest_lib.frontend.tko import models as tko_models
     36 from autotest_lib.server.cros.dynamic_suite import frontend_wrappers
     37 
     38 
     39 def get_matched_duts(hostnames=None, board=None, pool=None, other_labels=None):
     40     """Get duts with matching board and pool labels from given autotest instance
     41 
     42     @param hostnames: A list of hostnames.
     43     @param board: board of DUT, set to None if board doesn't need to match.
     44                   Default is None.
     45     @param pool: pool of DUT, set to None if pool doesn't need to match. Default
     46                  is None.
     47     @param other_labels: Other labels to filter duts.
     48     @return: A list of duts that match the specified board and pool.
     49     """
     50     if hostnames:
     51         hosts = models.Host.objects.filter(hostname__in=hostnames)
     52     else:
     53         multiple_labels = ()
     54         if pool:
     55             multiple_labels += ('pool:%s' % pool,)
     56         if board:
     57             multiple_labels += ('board:%s' % board,)
     58         if other_labels:
     59             for label in other_labels:
     60                 multiple_labels += (label,)
     61         hosts = rpc_utils.get_host_query(multiple_labels,
     62                                          exclude_only_if_needed_labels=False,
     63                                          exclude_atomic_group_hosts=False,
     64                                          valid_only=True, filter_data={})
     65     return [host_obj.get_object_dict() for host_obj in hosts]
     66 
     67 
     68 def get_job_runtime(input):
     69     """Get all test jobs and special tasks' runtime for a given host during
     70     a give time period.
     71 
     72     @param input: input arguments, including:
     73                   start_time: Start time of the search interval.
     74                   end_time: End time of the search interval.
     75                   host_id: id of the dut.
     76                   hostname: Name of the dut.
     77     @return: A list of records, e.g.,
     78                      [{'job_name':'dummy_Pass', 'time_used': 3, 'id': 12313,
     79                        'hostname': '1.2.3.4'},
     80                       {'task_name':'Cleanup', 'time_used': 30, 'id': 5687,
     81                        'hostname': '1.2.3.4'}]
     82     """
     83     start_time = input['start_time']
     84     end_time = input['end_time']
     85     host_id = input['host_id']
     86     hostname = input['hostname']
     87     records = []
     88     special_tasks = models.SpecialTask.objects.filter(
     89             host_id=host_id,
     90             time_started__gte=start_time,
     91             time_started__lte=end_time,
     92             time_started__isnull=False,
     93             time_finished__isnull=False).values('task', 'id', 'time_started',
     94                                                 'time_finished')
     95     for task in special_tasks:
     96         time_used = task['time_finished'] - task['time_started']
     97         records.append({'name': task['task'],
     98                         'id': task['id'],
     99                         'time_used': time_used.total_seconds(),
    100                         'hostname': hostname})
    101     hqes = models.HostQueueEntry.objects.filter(
    102             host_id=host_id,
    103             started_on__gte=start_time,
    104             started_on__lte=end_time,
    105             started_on__isnull=False,
    106             finished_on__isnull=False)
    107     for hqe in hqes:
    108         time_used = (hqe.finished_on - hqe.started_on).total_seconds()
    109         records.append({'name': hqe.job.name.split('/')[-1],
    110                         'id': hqe.job.id,
    111                         'time_used': time_used,
    112                         'hostname': hostname})
    113     return records
    114 
    115 def get_job_stats(jobs):
    116     """Get the stats of a list of jobs.
    117 
    118     @param jobs: A list of jobs.
    119     @return: Stats of the jobs' runtime, including:
    120              t_min: minimum runtime.
    121              t_max: maximum runtime.
    122              t_average: average runtime.
    123              t_median: median runtime.
    124     """
    125     runtimes = [job['time_used'] for job in jobs]
    126     t_min = min(runtimes)
    127     t_max = max(runtimes)
    128     t_mean = numpy.mean(runtimes)
    129     t_median = numpy.median(runtimes)
    130     return t_min, t_max, t_mean, t_median, len(runtimes)
    131 
    132 
    133 def process_results(results):
    134     """Compare the results.
    135 
    136     @param results: A list of a list of job/task information.
    137     """
    138     # Merge list of all results.
    139     all_results = []
    140     for result in results:
    141         all_results.extend(result)
    142     all_results = sorted(all_results, key=lambda r: r['name'])
    143     for name,jobs_for_test in groupby(all_results, lambda r: r['name']):
    144         print '='*80
    145         print 'Test %s' % name
    146         print '='*80
    147         for hostname,jobs_for_dut in groupby(jobs_for_test,
    148                                              lambda j: j['hostname']):
    149             jobs = list(jobs_for_dut)
    150             t_min, t_max, t_mean, t_median, count = get_job_stats(jobs)
    151             ids = [str(job['id']) for job in jobs]
    152             print ('%-28s: min= %-3.0f max= %-3.0f mean= %-3.0f med= %-3.0f '
    153                    'cnt= %-3s IDs: %s' %
    154                    (hostname, t_min, t_max, t_mean, t_median, count,
    155                     ','.join(sorted(ids)[-5:])))
    156 
    157 
    158 def main():
    159     """main script. """
    160     t_now = time.time()
    161     t_now_minus_one_day = t_now - 3600 * 24
    162     parser = argparse.ArgumentParser()
    163     parser.add_argument('-l', type=float, dest='last',
    164                         help='last hours to search results across',
    165                         default=24)
    166     parser.add_argument('--board', type=str, dest='board',
    167                         help='restrict query by board',
    168                         default=None)
    169     parser.add_argument('--pool', type=str, dest='pool',
    170                         help='restrict query by pool',
    171                         default=None)
    172     parser.add_argument('--hosts', nargs='+', dest='hosts',
    173                         help='Enter space deliminated hostnames',
    174                         default=[])
    175     parser.add_argument('--start', type=str, dest='start',
    176                         help=('Enter start time as: yyyy-mm-dd hh-mm-ss,'
    177                               'defualts to 24h ago.'))
    178     parser.add_argument('--end', type=str, dest='end',
    179                         help=('Enter end time in as: yyyy-mm-dd hh-mm-ss,'
    180                               'defualts to current time.'))
    181     options = parser.parse_args()
    182 
    183     if not options.start or not options.end:
    184         end_time = datetime.datetime.now()
    185         start_time = end_time - datetime.timedelta(seconds=3600 * options.last)
    186     else:
    187         start_time = time_utils.time_string_to_datetime(options.start)
    188         end_time = time_utils.time_string_to_datetime(options.end)
    189 
    190     hosts = get_matched_duts(hostnames=options.hosts, board=options.board,
    191                              pool=options.pool)
    192     if not hosts:
    193         raise Exception('No host found to search for history.')
    194     print 'Found %d duts.' % len(hosts)
    195     print 'Start time: %s' % start_time
    196     print 'End time:   %s' % end_time
    197     args = []
    198     for host in hosts:
    199         args.append({'start_time': start_time,
    200                      'end_time': end_time,
    201                      'host_id': host['id'],
    202                      'hostname': host['hostname']})
    203     get_job_runtime(args[0])
    204     # Parallizing this process.
    205     pool = multiprocessing.pool.ThreadPool()
    206     results = pool.imap_unordered(get_job_runtime, args)
    207     process_results(results)
    208 
    209 
    210 if __name__ == '__main__':
    211     main()
    212