1 # Copyright 2016 The Chromium OS Authors. All rights reserved. 2 # Use of this source code is governed by a BSD-style license that can be 3 # found in the LICENSE file. 4 5 """Services relating to generating a suite timeline and report.""" 6 7 from __future__ import print_function 8 9 import common 10 import json 11 12 from autotest_lib.client.common_lib import time_utils 13 from autotest_lib.server import frontend 14 from autotest_lib.server.lib import status_history 15 from chromite.lib import cros_logging as logging 16 17 18 HostJobHistory = status_history.HostJobHistory 19 20 # TODO: Handle other statuses like infra failures. 21 TKO_STATUS_MAP = { 22 'ERROR': 'fail', 23 'FAIL': 'fail', 24 'GOOD': 'pass', 25 'PASS': 'pass', 26 'Failed': 'fail', 27 'Completed': 'pass', 28 } 29 30 31 def parse_tko_status_string(status_string): 32 """Parse a status string from TKO or the HQE databases. 33 34 @param status_string: A status string from TKO or HQE databases. 35 36 @return A status string suitable for inclusion within Cloud Datastore. 37 """ 38 return TKO_STATUS_MAP.get(status_string, 'unknown:' + status_string) 39 40 41 def make_entry(entry_id, name, status, start_time, 42 finish_time=None, parent=None): 43 """Generate an event log entry to be stored in Cloud Datastore. 44 45 @param entry_id: A (Kind, id) tuple representing the key. 46 @param name: A string identifying the event 47 @param status: A string identifying the status of the event. 48 @param start_time: A unix timestamp of the start of the event. 49 @param finish_time: A unix timestamp of the finish of the event. 50 @param parent: A (Kind, id) tuple representing the parent key. 51 52 @return A dictionary representing the entry suitable for dumping via JSON. 53 """ 54 entry = { 55 'id': entry_id, 56 'name': name, 57 'status': status, 58 'start_time': start_time, 59 } 60 if finish_time is not None: 61 entry['finish_time'] = finish_time 62 if parent is not None: 63 entry['parent'] = parent 64 return entry 65 66 67 def find_start_finish_times(statuses): 68 """Determines the start and finish times for a list of statuses. 69 70 @param statuses: A list of job test statuses. 71 72 @return (start_tme, finish_time) tuple of seconds past epoch. If either 73 cannot be determined, None for that time. 74 """ 75 starts = {int(time_utils.to_epoch_time(s.test_started_time)) 76 for s in statuses if s.test_started_time != 'None'} 77 finishes = {int(time_utils.to_epoch_time(s.test_finished_time)) 78 for s in statuses if s.test_finished_time != 'None'} 79 start_time = min(starts) if starts else None 80 finish_time = max(finishes) if finishes else None 81 return start_time, finish_time 82 83 84 def make_job_entry(tko, job, parent=None, suite_job=False, job_entries=None): 85 """Generate a Suite or HWTest event log entry. 86 87 @param tko: TKO database handle. 88 @param job: A frontend.Job to generate an entry for. 89 @param parent: A (Kind, id) tuple representing the parent key. 90 @param suite_job: A boolean indicating wheret this represents a suite job. 91 @param job_entries: A dictionary mapping job id to earlier job entries. 92 93 @return A dictionary representing the entry suitable for dumping via JSON. 94 """ 95 statuses = tko.get_job_test_statuses_from_db(job.id) 96 status = 'pass' 97 dut = None 98 for s in statuses: 99 parsed_status = parse_tko_status_string(s.status) 100 # TODO: Improve this generation of status. 101 if parsed_status != 'pass': 102 status = parsed_status 103 if s.hostname: 104 dut = s.hostname 105 if s.test_started_time == 'None' or s.test_finished_time == 'None': 106 logging.warn('TKO entry for %d missing time: %s' % (job.id, str(s))) 107 start_time, finish_time = find_start_finish_times(statuses) 108 entry = make_entry(('Suite' if suite_job else 'HWTest', int(job.id)), 109 job.name.split('/')[-1], status, start_time, 110 finish_time=finish_time, parent=parent) 111 112 if dut: 113 entry['dut'] = dut 114 if job.shard: 115 entry['shard'] = job.shard 116 # Determine the try of this job by looking back through what the 117 # original job id is. 118 if 'retry_original_job_id' in job.keyvals: 119 original_job_id = int(job.keyvals['retry_original_job_id']) 120 original_job = job_entries.get(original_job_id, None) 121 if original_job: 122 entry['try'] = original_job['try'] + 1 123 else: 124 entry['try'] = 0 125 else: 126 entry['try'] = 1 127 entry['gs_url'] = status_history.get_job_gs_url(job) 128 return entry 129 130 131 def make_hqe_entry(hostname, hqe, hqe_statuses, parent=None): 132 """Generate a HQE event log entry. 133 134 @param hostname: A string of the hostname. 135 @param hqe: A host history to generate an event for. 136 @param hqe_statuses: A dictionary mapping HQE ids to job status. 137 @param parent: A (Kind, id) tuple representing the parent key. 138 139 @return A dictionary representing the entry suitable for dumping via JSON. 140 """ 141 entry = make_entry( 142 ('HQE', int(hqe.id)), hostname, 143 hqe_statuses.get(hqe.id, parse_tko_status_string(hqe.job_status)), 144 hqe.start_time, finish_time=hqe.end_time, parent=parent) 145 146 entry['task_name'] = hqe.name.split('/')[-1] 147 entry['in_suite'] = hqe.id in hqe_statuses 148 entry['job_url'] = hqe.job_url 149 entry['gs_url'] = hqe.gs_url 150 if hqe.job_id is not None: 151 entry['job_id'] = hqe.job_id 152 entry['is_special'] = hqe.is_special 153 return entry 154 155 156 def generate_suite_report(suite_job_id, afe=None, tko=None): 157 """Generate a list of events corresonding to a single suite job. 158 159 @param suite_job_id: The AFE id of the suite job. 160 @param afe: AFE database handle. 161 @param tko: TKO database handle. 162 163 @return A list of entries suitable for dumping via JSON. 164 """ 165 if afe is None: 166 afe = frontend.AFE() 167 if tko is None: 168 tko = frontend.TKO() 169 170 # Retrieve the main suite job. 171 suite_job = afe.get_jobs(id=suite_job_id)[0] 172 173 suite_entry = make_job_entry(tko, suite_job, suite_job=True) 174 entries = [suite_entry] 175 176 # Retrieve the child jobs and cache all their statuses 177 logging.debug('Fetching child jobs...') 178 child_jobs = afe.get_jobs(parent_job_id=suite_job_id) 179 logging.debug('... fetched %s child jobs.' % len(child_jobs)) 180 job_statuses = {} 181 job_entries = {} 182 for j in child_jobs: 183 job_entry = make_job_entry(tko, j, suite_entry['id'], 184 job_entries=job_entries) 185 entries.append(job_entry) 186 job_statuses[j.id] = job_entry['status'] 187 job_entries[j.id] = job_entry 188 189 # Retrieve the HQEs from all the child jobs, record statuses from 190 # job statuses. 191 child_job_ids = {j.id for j in child_jobs} 192 logging.debug('Fetching HQEs...') 193 hqes = afe.get_host_queue_entries(job_id__in=list(child_job_ids)) 194 logging.debug('... fetched %s HQEs.' % len(hqes)) 195 hqe_statuses = {h.id: job_statuses.get(h.job.id, None) for h in hqes} 196 197 # Generate list of hosts. 198 hostnames = {h.host.hostname for h in hqes if h.host} 199 logging.debug('%s distinct hosts participated in the suite.' % 200 len(hostnames)) 201 202 # Retrieve histories for the time of the suite for all associated hosts. 203 # TODO: Include all hosts in the pool. 204 if suite_entry['start_time'] and suite_entry['finish_time']: 205 histories = [HostJobHistory.get_host_history(afe, hostname, 206 suite_entry['start_time'], 207 suite_entry['finish_time']) 208 for hostname in sorted(hostnames)] 209 210 for history in histories: 211 entries.extend(make_hqe_entry(history.hostname, h, hqe_statuses, 212 suite_entry['id']) for h in history) 213 214 return entries 215 216 def dump_entries_as_json(entries, output_file): 217 """Dump event log entries as json to a file. 218 219 @param entries: A list of event log entries to dump. 220 @param output_file: The file to write to. 221 """ 222 # Write the entries out as JSON. 223 logging.debug('Dumping %d entries' % len(entries)) 224 for e in entries: 225 json.dump(e, output_file, sort_keys=True) 226 output_file.write('\n') 227
