Home | History | Annotate | Download | only in site_utils 
      1 #!/usr/bin/env python
      2 
      3 # Copyright (c) 2014 The Chromium OS Authors. All rights reserved.
      4 # Use of this source code is governed by a BSD-style license that can be
      5 # found in the LICENSE file.
      6 
      7 """This script is to be run daily to report machine utilization stats across
      8 each board and pool.
      9 """
     10 
     11 
     12 import argparse
     13 from datetime import date
     14 from datetime import datetime
     15 from datetime import timedelta
     16 
     17 import common
     18 from autotest_lib.client.common_lib import time_utils
     19 from autotest_lib.client.common_lib.cros.graphite import autotest_stats
     20 from autotest_lib.site_utils import gmail_lib
     21 from autotest_lib.site_utils import host_history
     22 from autotest_lib.site_utils import host_history_utils
     23 from autotest_lib.site_utils import host_label_utils
     24 
     25 
     26 def report_stats(board, pool, start_time, end_time, span):
     27     """Report machine stats for given board, pool and time period.
     28 
     29     @param board: Name of board.
     30     @param pool: Name of pool.
     31     @param start_time: start time to collect stats.
     32     @param end_time: end time to collect stats.
     33     @param span: Number of hours that the stats should be collected for.
     34     @return: Error message collected when calculating the stats.
     35     """
     36     print '================ %-12s %-12s ================' % (board, pool)
     37     try:
     38         history = host_history.get_history_details(start_time=start_time,
     39                                                    end_time=end_time,
     40                                                    board=board,
     41                                                    pool=pool)
     42     except host_history_utils.NoHostFoundException as e:
     43         print 'No history found. Error:\n%s' % e
     44         history = None
     45         mur = -1
     46         mar = -1
     47         mir = -1
     48 
     49     if history:
     50         status_intervals = host_history_utils.get_status_intervals(history)
     51         stats_all, num_hosts = host_history_utils.aggregate_hosts(
     52                 status_intervals)
     53         total = 0
     54         total_time = span*3600*num_hosts
     55         for status, interval in stats_all.iteritems():
     56             total += interval
     57         if abs(total - total_time) > 10:
     58             error = ('Status intervals do not add up. No stats will be '
     59                      'collected for board: %s, pool: %s, diff: %s' %
     60                      (board, pool, total - total_time))
     61             hosts = []
     62             for history_for_host in status_intervals:
     63                 total = 0
     64                 for interval in history_for_host.keys():
     65                     total += interval[1] - interval[0]
     66                 if total > span*3600:
     67                     hosts.append(history_for_host.values()[0]['metadata']['hostname'])
     68             error += ' hosts: %s' % ','.join(hosts)
     69             print error
     70             return error
     71 
     72         mur = host_history_utils.get_machine_utilization_rate(stats_all)
     73         mar = host_history_utils.get_machine_availability_rate(stats_all)
     74         mir = mar - mur
     75 
     76         for status, interval in stats_all.iteritems():
     77             print '%-18s %-16s %-10.2f%%' % (status, interval,
     78                                              100*interval/total_time)
     79         print 'Machine utilization rate  = %-4.2f%%' % (100*mur)
     80         print 'Machine availability rate = %-4.2f%%' % (100*mar)
     81 
     82     autotest_stats.Gauge('machine_utilization_rate').send('%s_hours.%s.%s' %
     83                                                           (span, board, pool),
     84                                                           mur)
     85     autotest_stats.Gauge('machine_availability_rate').send('%s_hours.%s.%s' %
     86                                                            (span, board, pool),
     87                                                            mar)
     88     autotest_stats.Gauge('machine_idle_rate').send('%s_hours.%s.%s' %
     89                                                    (span, board, pool), mir)
     90 
     91 
     92 def main():
     93     """main script. """
     94     parser = argparse.ArgumentParser()
     95     parser.add_argument('--span', type=int, dest='span', default=1,
     96                         help=('Number of hours that stats should be collected. '
     97                               'If it is set to 24, the end time of stats being '
     98                               'collected will set to the mid of the night. '
     99                               'Default is set to 1 hour.'))
    100     parser.add_argument('-e', '--email', dest='email', default=None,
    101                         help='Email any errors to the given email address.')
    102     options = parser.parse_args()
    103 
    104     boards = host_label_utils.get_all_boards()
    105     pools = ['bvt', 'suites', 'cq']
    106 
    107     if options.span == 24:
    108         today = datetime.combine(date.today(), datetime.min.time())
    109         end_time = time_utils.to_epoch_time(today)
    110     else:
    111         now = datetime.now()
    112         end_time = datetime(year=now.year, month=now.month, day=now.day,
    113                             hour=now.hour)
    114         end_time = time_utils.to_epoch_time(end_time)
    115 
    116     start_time = end_time - timedelta(hours=options.span).total_seconds()
    117     print ('Collecting host stats from %s to %s...' %
    118            (time_utils.epoch_time_to_date_string(start_time),
    119             time_utils.epoch_time_to_date_string(end_time)))
    120 
    121     errors = []
    122     if not boards:
    123         errors.append('Error! No board found in metadb.')
    124     for board in boards:
    125         for pool in pools:
    126             error = report_stats(board, pool, start_time, end_time,
    127                                  options.span)
    128             if error:
    129                 errors.append(error)
    130     if options.email and errors:
    131         gmail_lib.send_email(options.email,
    132                              'Error occured when collecting host stats.',
    133                              '\n'.join(errors))
    134 
    135 
    136 if __name__ == '__main__':
    137     main()
    138