1 import os, re, db, sys, datetime 2 import common 3 from autotest_lib.client.common_lib import kernel_versions 4 5 MAX_RECORDS = 50000L 6 MAX_CELLS = 500000L 7 8 tko = os.path.dirname(os.path.realpath(os.path.abspath(__file__))) 9 root_url_file = os.path.join(tko, '.root_url') 10 if os.path.exists(root_url_file): 11 html_root = open(root_url_file, 'r').readline().rstrip() 12 else: 13 html_root = '/results/' 14 15 16 class status_cell: 17 # One cell in the matrix of status data. 18 def __init__(self): 19 # Count is a dictionary: status -> count of tests with status 20 self.status_count = {} 21 self.reasons_list = [] 22 self.job_tag = None 23 self.job_tag_count = 0 24 25 26 def add(self, status, count, job_tags, reasons = None): 27 assert count > 0 28 29 self.job_tag = job_tags 30 self.job_tag_count += count 31 if self.job_tag_count > 1: 32 self.job_tag = None 33 34 self.status_count[status] = count 35 ### status == 6 means 'GOOD' 36 if status != 6: 37 ## None implies sorting problems and extra CRs in a cell 38 if reasons: 39 self.reasons_list.append(reasons) 40 41 42 class status_data: 43 def __init__(self, sql_rows, x_field, y_field, query_reasons = False): 44 data = {} 45 y_values = set() 46 47 # Walk through the query, filing all results by x, y info 48 for row in sql_rows: 49 if query_reasons: 50 (x,y, status, count, job_tags, reasons) = row 51 else: 52 (x,y, status, count, job_tags) = row 53 reasons = None 54 if not data.has_key(x): 55 data[x] = {} 56 if not data[x].has_key(y): 57 y_values.add(y) 58 data[x][y] = status_cell() 59 data[x][y].add(status, count, job_tags, reasons) 60 61 # 2-d hash of data - [x-value][y-value] 62 self.data = data 63 # List of possible columns (x-values) 64 self.x_values = smart_sort(data.keys(), x_field) 65 # List of rows columns (y-values) 66 self.y_values = smart_sort(list(y_values), y_field) 67 nCells = len(self.y_values)*len(self.x_values) 68 if nCells > MAX_CELLS: 69 msg = 'Exceeded allowed number of cells in a table' 70 raise db.MySQLTooManyRows(msg) 71 72 73 def get_matrix_data(db_obj, x_axis, y_axis, where = None, 74 query_reasons = False): 75 # Searches on the test_view table - x_axis and y_axis must both be 76 # column names in that table. 77 x_field = test_view_field_dict[x_axis] 78 y_field = test_view_field_dict[y_axis] 79 query_fields_list = [x_field, y_field, 'status','COUNT(status)'] 80 query_fields_list.append("LEFT(GROUP_CONCAT(job_tag),100)") 81 if query_reasons: 82 query_fields_list.append( 83 "LEFT(GROUP_CONCAT(DISTINCT reason SEPARATOR '|'),500)" 84 ) 85 fields = ','.join(query_fields_list) 86 87 group_by = '%s, %s, status' % (x_field, y_field) 88 rows = db_obj.select(fields, 'tko_test_view', 89 where=where, group_by=group_by, max_rows = MAX_RECORDS) 90 return status_data(rows, x_field, y_field, query_reasons) 91 92 93 # Dictionary used simply for fast lookups from short reference names for users 94 # to fieldnames in test_view 95 test_view_field_dict = { 96 'kernel' : 'kernel_printable', 97 'hostname' : 'machine_hostname', 98 'test' : 'test', 99 'label' : 'job_label', 100 'machine_group' : 'machine_group', 101 'reason' : 'reason', 102 'tag' : 'job_tag', 103 'user' : 'job_username', 104 'status' : 'status_word', 105 'time' : 'test_finished_time', 106 'start_time' : 'test_started_time', 107 'time_daily' : 'DATE(test_finished_time)' 108 } 109 110 111 def smart_sort(list, field): 112 if field == 'kernel_printable': 113 def kernel_encode(kernel): 114 return kernel_versions.version_encode(kernel) 115 list.sort(key = kernel_encode, reverse = True) 116 return list 117 ## old records may contain time=None 118 ## make None comparable with timestamp datetime or date 119 elif field == 'test_finished_time': 120 def convert_None_to_datetime(date_time): 121 if not date_time: 122 return datetime.datetime(1970, 1, 1, 0, 0, 0) 123 else: 124 return date_time 125 list = map(convert_None_to_datetime, list) 126 elif field == 'DATE(test_finished_time)': 127 def convert_None_to_date(date): 128 if not date: 129 return datetime.date(1970, 1, 1) 130 else: 131 return date 132 list = map(convert_None_to_date, list) 133 list.sort() 134 return list 135 136 137 class group: 138 @classmethod 139 def select(klass, db): 140 """Return all possible machine groups""" 141 rows = db.select('distinct machine_group', 'tko_machines', 142 'machine_group is not null') 143 groupnames = sorted([row[0] for row in rows]) 144 return [klass(db, groupname) for groupname in groupnames] 145 146 147 def __init__(self, db, name): 148 self.name = name 149 self.db = db 150 151 152 def machines(self): 153 return machine.select(self.db, { 'machine_group' : self.name }) 154 155 156 def tests(self, where = {}): 157 values = [self.name] 158 sql = 't inner join tko_machines m on m.machine_idx=t.machine_idx' 159 sql += ' where m.machine_group=%s' 160 for key in where.keys(): 161 sql += ' and %s=%%s' % key 162 values.append(where[key]) 163 return test.select_sql(self.db, sql, values) 164 165 166 class machine: 167 @classmethod 168 def select(klass, db, where = {}): 169 fields = ['machine_idx', 'hostname', 'machine_group', 'owner'] 170 machines = [] 171 for row in db.select(','.join(fields), 'tko_machines', where): 172 machines.append(klass(db, *row)) 173 return machines 174 175 176 def __init__(self, db, idx, hostname, group, owner): 177 self.db = db 178 self.idx = idx 179 self.hostname = hostname 180 self.group = group 181 self.owner = owner 182 183 184 class kernel: 185 @classmethod 186 def select(klass, db, where = {}): 187 fields = ['kernel_idx', 'kernel_hash', 'base', 'printable'] 188 rows = db.select(','.join(fields), 'tko_kernels', where) 189 return [klass(db, *row) for row in rows] 190 191 192 def __init__(self, db, idx, hash, base, printable): 193 self.db = db 194 self.idx = idx 195 self.hash = hash 196 self.base = base 197 self.printable = printable 198 self.patches = [] # THIS SHOULD PULL IN PATCHES! 199 200 201 class test: 202 @classmethod 203 def select(klass, db, where={}, distinct=False): 204 fields = ['test_idx', 'job_idx', 'test', 'subdir', 205 'kernel_idx', 'status', 'reason', 'machine_idx'] 206 tests = [] 207 for row in db.select(','.join(fields), 'tko_tests', where, 208 distinct): 209 tests.append(klass(db, *row)) 210 return tests 211 212 213 @classmethod 214 def select_sql(klass, db, sql, values): 215 fields = ['test_idx', 'job_idx', 'test', 'subdir', 216 'kernel_idx', 'status', 'reason', 'machine_idx'] 217 fields = ['t.'+field for field in fields] 218 rows = db.select_sql(','.join(fields), 'tko_tests', sql, values) 219 return [klass(db, *row) for row in rows] 220 221 222 def __init__(self, db, test_idx, job_idx, testname, subdir, kernel_idx, 223 status_num, reason, machine_idx): 224 self.idx = test_idx 225 self.job = job(db, job_idx) 226 self.testname = testname 227 self.subdir = subdir 228 self.kernel_idx = kernel_idx 229 self.__kernel = None 230 self.__iterations = None 231 self.machine_idx = machine_idx 232 self.__machine = None 233 self.status_num = status_num 234 self.status_word = db.status_word[status_num] 235 self.reason = reason 236 self.db = db 237 if self.subdir: 238 self.url = html_root + self.job.tag + '/' + self.subdir 239 else: 240 self.url = None 241 242 243 def iterations(self): 244 """ 245 Caching function for iterations 246 """ 247 if not self.__iterations: 248 self.__iterations = {} 249 # A dictionary - dict{key} = [value1, value2, ....] 250 where = {'test_idx' : self.idx} 251 for i in iteration.select(self.db, where): 252 if self.__iterations.has_key(i.key): 253 self.__iterations[i.key].append(i.value) 254 else: 255 self.__iterations[i.key] = [i.value] 256 return self.__iterations 257 258 259 def kernel(self): 260 """ 261 Caching function for kernels 262 """ 263 if not self.__kernel: 264 where = {'kernel_idx' : self.kernel_idx} 265 self.__kernel = kernel.select(self.db, where)[0] 266 return self.__kernel 267 268 269 def machine(self): 270 """ 271 Caching function for kernels 272 """ 273 if not self.__machine: 274 where = {'machine_idx' : self.machine_idx} 275 self.__machine = machine.select(self.db, where)[0] 276 return self.__machine 277 278 279 class job: 280 def __init__(self, db, job_idx): 281 where = {'job_idx' : job_idx} 282 rows = db.select('tag, machine_idx', 'tko_jobs', where) 283 if rows: 284 self.tag, self.machine_idx = rows[0] 285 self.job_idx = job_idx 286 287 288 class iteration: 289 @classmethod 290 def select(klass, db, where): 291 fields = ['iteration', 'attribute', 'value'] 292 iterations = [] 293 rows = db.select(','.join(fields), 'tko_iteration_result', where) 294 for row in rows: 295 iterations.append(klass(*row)) 296 return iterations 297 298 299 def __init__(self, iteration, key, value): 300 self.iteration = iteration 301 self.key = key 302 self.value = value 303 304 # class patch: 305 # def __init__(self): 306 # self.spec = None 307
