1 # Copyright 2015 The Chromium Authors. All rights reserved. 2 # Use of this source code is governed by a BSD-style license that can be 3 # found in the LICENSE file. 4 5 import collections 6 import logging 7 import re 8 import time 9 10 from google.appengine.api import urlfetch 11 import webapp2 12 13 from base import bigquery 14 from base import constants 15 from common import buildbot 16 17 18 class Builds(webapp2.RequestHandler): 19 20 def get(self): 21 urlfetch.set_default_fetch_deadline(300) 22 23 bq = bigquery.BigQuery() 24 25 current_events = [] 26 events = [] 27 for master_name in constants.MASTER_NAMES: 28 builders = buildbot.Builders(master_name) 29 available_builds = _AvailableBuilds(builders) 30 recorded_builds = _RecordedBuilds(bq, builders, available_builds) 31 for builder in builders: 32 # Filter out recorded builds from available builds. 33 build_numbers = (available_builds[builder.name] - 34 recorded_builds[builder.name]) 35 builder_current_events, builder_events = _TraceEventsForBuilder( 36 builder, build_numbers) 37 current_events += builder_current_events 38 events += builder_events 39 40 jobs = [] 41 if current_events: 42 jobs += bq.InsertRowsAsync( 43 constants.DATASET, constants.CURRENT_BUILDS_TABLE, 44 current_events, truncate=True) 45 if events: 46 jobs += bq.InsertRowsAsync(constants.DATASET, constants.BUILDS_TABLE, 47 events) 48 49 for job in jobs: 50 bq.PollJob(job, 60 * 20) # 20 minutes. 51 52 53 def _AvailableBuilds(builders): 54 available_builds = {} 55 for builder in builders: 56 if not builder.cached_builds: 57 available_builds[builder.name] = frozenset() 58 continue 59 60 max_build = max(builder.cached_builds) 61 # Buildbot on tryserver.chromium.perf is occasionally including build 0 in 62 # its list of cached builds. That results in more builds than we want. 63 # Limit the list to the last 100 builds, because the urlfetch URL limit is 64 # 2048 bytes, and "&select=100000" * 100 is 1400 bytes. 65 builds = frozenset(build for build in builder.cached_builds 66 if build >= max_build - 100) 67 available_builds[builder.name] = builds 68 return available_builds 69 70 71 def _RecordedBuilds(bq, builders, available_builds): 72 # 105 days / 15 weeks. Must be some number greater than 100 days, because 73 # we request up to 100 builds (see above comment), and the slowest cron bots 74 # run one job every day. 75 start_time_ms = -1000 * 60 * 60 * 24 * 105 76 table = '%s.%s@%d-' % (constants.DATASET, constants.BUILDS_TABLE, 77 start_time_ms) 78 79 conditions = [] 80 for builder in builders: 81 if not available_builds[builder.name]: 82 continue 83 max_build = max(available_builds[builder.name]) 84 min_build = min(available_builds[builder.name]) 85 conditions.append('WHEN builder = "%s" THEN build >= %d AND build <= %d' % 86 (builder.name, min_build, max_build)) 87 88 query = ( 89 'SELECT builder, build ' 90 'FROM [%s] ' % table + 91 'WHERE CASE %s END ' % ' '.join(conditions) + 92 'GROUP BY builder, build' 93 ) 94 query_result = bq.QuerySync(query, 600) 95 96 builds = collections.defaultdict(set) 97 for row in query_result: 98 builds[row['f'][0]['v']].add(int(row['f'][1]['v'])) 99 return builds 100 101 102 def _TraceEventsForBuilder(builder, build_numbers): 103 if not build_numbers: 104 return (), () 105 106 build_numbers_string = ', '.join(map(str, sorted(build_numbers))) 107 logging.info('Getting %s: %s', builder.name, build_numbers_string) 108 109 # Fetch build information and generate trace events. 110 current_events = [] 111 events = [] 112 113 builder_builds = builder.builds.Fetch(build_numbers) 114 query_time = time.time() 115 for build in builder_builds: 116 if build.complete: 117 events += _TraceEventsFromBuild(builder, build, query_time) 118 else: 119 current_events += _TraceEventsFromBuild(builder, build, query_time) 120 121 return current_events, events 122 123 124 def _TraceEventsFromBuild(builder, build, query_time): 125 match = re.match(r'(.+) \(([0-9]+)\)', builder.name) 126 if match: 127 configuration, host_shard = match.groups() 128 host_shard = int(host_shard) 129 else: 130 configuration = builder.name 131 host_shard = 0 132 133 # Build trace event. 134 if build.end_time: 135 build_end_time = build.end_time 136 else: 137 build_end_time = query_time 138 os, os_version, role = _ParseBuilderName(builder.master_name, builder.name) 139 yield { 140 'name': 'Build %d' % build.number, 141 'start_time': build.start_time, 142 'end_time': build_end_time, 143 144 'build': build.number, 145 'builder': builder.name, 146 'configuration': configuration, 147 'host_shard': host_shard, 148 'hostname': build.slave_name, 149 'master': builder.master_name, 150 'os': os, 151 'os_version': os_version, 152 'role': role, 153 'status': build.status, 154 'url': build.url, 155 } 156 157 # Step trace events. 158 for step in build.steps: 159 if not step.start_time: 160 continue 161 162 if step.name == 'steps': 163 continue 164 165 if step.end_time: 166 step_end_time = step.end_time 167 else: 168 step_end_time = query_time 169 yield { 170 'name': step.name, 171 'start_time': step.start_time, 172 'end_time': step_end_time, 173 174 'benchmark': step.name, # TODO(dtu): This isn't always right. 175 'build': build.number, 176 'builder': builder.name, 177 'configuration': configuration, 178 'host_shard': host_shard, 179 'hostname': build.slave_name, 180 'master': builder.master_name, 181 'os': os, 182 'os_version': os_version, 183 'role': role, 184 'status': step.status, 185 'url': step.url, 186 } 187 188 189 def _ParseBuilderName(master_name, builder_name): 190 if master_name == 'chromium.perf': 191 match = re.match(r'^([A-Za-z]+)(?: ([0-9\.]+|XP))?([A-Za-z0-9-\. ]+)? ' 192 r'(Builder|Perf)(?: \([0-9]+\))?$', builder_name).groups() 193 os = match[0] 194 if match[1]: 195 os_version = match[1] 196 else: 197 os_version = None 198 if match[3] == 'Builder': 199 role = 'builder' 200 elif match[3] == 'Perf': 201 role = 'tester' 202 else: 203 raise NotImplementedError() 204 elif master_name == 'client.catapult': 205 match = re.match(r'^Catapult(?: ([A-Za-z])+)? ([A-Za-z]+)$', 206 builder_name).groups() 207 os = match[1] 208 os_version = None 209 role = match[0] 210 if not role: 211 role = 'tester' 212 elif master_name == 'tryserver.chromium.perf': 213 match = re.match(r'^(android|linux|mac|win).*_([a-z]+)$', 214 builder_name).groups() 215 os = match[0] 216 os_version = None 217 role = match[1] 218 elif master_name == 'tryserver.client.catapult': 219 match = re.match(r'^Catapult(?: (Android|Linux|Mac|Windows))? ([A-Za-z]+)$', 220 builder_name).groups() 221 os = match[0] 222 os_version = None 223 role = match[1] 224 else: 225 raise NotImplementedError() 226 227 if os: 228 os = os.lower() 229 if os == 'windows': 230 os = 'win' 231 if os_version: 232 os_version = os_version.lower() 233 role = role.lower() 234 235 return (os, os_version, role) 236
