1 # Copyright (C) 2013 Google Inc. All rights reserved. 2 # 3 # Redistribution and use in source and binary forms, with or without 4 # modification, are permitted provided that the following conditions are 5 # met: 6 # 7 # * Redistributions of source code must retain the above copyright 8 # notice, this list of conditions and the following disclaimer. 9 # * Redistributions in binary form must reproduce the above 10 # copyright notice, this list of conditions and the following disclaimer 11 # in the documentation and/or other materials provided with the 12 # distribution. 13 # * Neither the name of Google Inc. nor the names of its 14 # contributors may be used to endorse or promote products derived from 15 # this software without specific prior written permission. 16 # 17 # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 18 # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 19 # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 20 # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 21 # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 22 # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 23 # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 24 # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 25 # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 26 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 27 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 28 29 import datetime 30 import json 31 import logging 32 import sys 33 import traceback 34 import urllib2 35 import webapp2 36 37 from google.appengine.api import memcache 38 39 MASTERS = [ 40 {'name': 'ChromiumWin', 'url': 'http://build.chromium.org/p/chromium.win', 'groups': ['@ToT Chromium']}, 41 {'name': 'ChromiumMac', 'url': 'http://build.chromium.org/p/chromium.mac', 'groups': ['@ToT Chromium']}, 42 {'name': 'ChromiumLinux', 'url': 'http://build.chromium.org/p/chromium.linux', 'groups': ['@ToT Chromium']}, 43 {'name': 'ChromiumChromiumOS', 'url': 'http://build.chromium.org/p/chromium.chromiumos', 'groups': ['@ToT ChromeOS']}, 44 {'name': 'ChromiumGPU', 'url': 'http://build.chromium.org/p/chromium.gpu', 'groups': ['@ToT Chromium']}, 45 {'name': 'ChromiumGPUFYI', 'url': 'http://build.chromium.org/p/chromium.gpu.fyi', 'groups': ['@ToT Chromium FYI']}, 46 {'name': 'ChromiumPerfAv', 'url': 'http://build.chromium.org/p/chromium.perf_av', 'groups': ['@ToT Chromium']}, 47 {'name': 'ChromiumWebkit', 'url': 'http://build.chromium.org/p/chromium.webkit', 'groups': ['@ToT Chromium', '@ToT Blink']}, 48 {'name': 'ChromiumFYI', 'url': 'http://build.chromium.org/p/chromium.fyi', 'groups': ['@ToT Chromium FYI']}, 49 {'name': 'V8', 'url': 'http://build.chromium.org/p/client.v8', 'groups': ['@ToT V8']}, 50 ] 51 52 53 class FetchBuildersException(Exception): pass 54 55 56 def master_json_url(master_url): 57 return master_url + '/json/builders' 58 59 60 def builder_json_url(master_url, builder): 61 return master_json_url(master_url) + '/' + urllib2.quote(builder) 62 63 64 def cached_build_json_url(master_url, builder, build_number): 65 return builder_json_url(master_url, builder) + '/builds/' + str(build_number) 66 67 68 def fetch_json(url): 69 logging.debug('Fetching %s' % url) 70 fetched_json = {} 71 try: 72 resp = urllib2.urlopen(url) 73 except: 74 exc_info = sys.exc_info() 75 logging.warning('Error while fetching %s: %s', url, exc_info[1]) 76 return fetched_json 77 78 try: 79 fetched_json = json.load(resp) 80 except: 81 exc_info = sys.exc_info() 82 logging.warning('Unable to parse JSON response from %s: %s', url, exc_info[1]) 83 84 return fetched_json 85 86 87 def get_latest_build(build_data): 88 cached_builds = [] 89 if 'cachedBuilds' in build_data: 90 cached_builds = build_data['cachedBuilds'] 91 92 current_builds = build_data['currentBuilds'] 93 94 latest_cached_builds = set(cached_builds) - set(current_builds) 95 if len(latest_cached_builds) != 0: 96 latest_cached_builds = sorted(list(latest_cached_builds)) 97 latest_build = latest_cached_builds[-1] 98 elif len(current_builds) != 0: 99 latest_build = current_builds[0] 100 else: 101 basedir = build_data['basedir'] if 'basedir' in build_data else 'current builder' 102 logging.info('No cached or current builds for %s', basedir) 103 return None 104 105 return latest_build 106 107 108 def dump_json(data): 109 return json.dumps(data, separators=(',', ':'), sort_keys=True) 110 111 112 def fetch_buildbot_data(masters, force_update=False): 113 if force_update: 114 logging.info('Starting a forced buildbot update. Failure to fetch a master\'s data will not abort the fetch.') 115 116 start_time = datetime.datetime.now() 117 master_data = masters[:] 118 for master in master_data: 119 master_url = master['url'] 120 tests_object = master.setdefault('tests', {}) 121 master['tests'] = tests_object 122 123 builders = fetch_json(master_json_url(master_url)) 124 if not builders: 125 msg = 'Could not fetch builders from master "%s": %s.' % (master['name'], master_url) 126 logging.warning(msg) 127 if force_update: 128 continue 129 else: 130 logging.warning('Aborting fetch.') 131 raise FetchBuildersException(msg) 132 133 for builder in builders: 134 build_data = fetch_json(builder_json_url(master_url, builder)) 135 136 latest_build = get_latest_build(build_data) 137 if not latest_build: 138 logging.info('Skipping builder %s because it lacked cached or current builds.', builder) 139 continue 140 141 build = fetch_json(cached_build_json_url(master_url, builder, latest_build)) 142 if not build: 143 logging.info('Skipping build %s on builder %s due to empty data', latest_build, builder) 144 for step in build['steps']: 145 step_name = step['name'] 146 is_test_step = 'test' in step_name and 'archive' not in step_name and 'Run tests' not in step_name 147 if not is_test_step: 148 continue 149 150 if step_name == 'webkit_tests': 151 step_name = 'layout-tests' 152 153 tests_object.setdefault(step_name, {'builders': []}) 154 tests_object[step_name]['builders'].append(builder) 155 156 for builders in tests_object.values(): 157 builders['builders'].sort() 158 159 output_data = {'masters': master_data} 160 161 delta = datetime.datetime.now() - start_time 162 163 logging.info('Fetched buildbot data in %s seconds.', delta.seconds) 164 165 return dump_json(output_data) 166 167 168 class UpdateBuilders(webapp2.RequestHandler): 169 """Fetch and update the cached buildbot data.""" 170 def get(self): 171 force_update = True if self.request.get('force') else False 172 try: 173 buildbot_data = fetch_buildbot_data(MASTERS, force_update) 174 memcache.set('buildbot_data', buildbot_data) 175 self.response.set_status(200) 176 self.response.out.write("ok") 177 except FetchBuildersException, ex: 178 logging.error('Not updating builders because fetch failed: %s', str(ex)) 179 self.response.set_status(500) 180 self.response.out.write(ex.message) 181 182 183 184 class GetBuilders(webapp2.RequestHandler): 185 """Return a list of masters mapped to their respective builders, possibly using cached data.""" 186 def get(self): 187 callback = self.request.get('callback') 188 189 buildbot_data = memcache.get('buildbot_data') 190 191 if not buildbot_data: 192 logging.warning('No buildbot data in memcache. If this message repeats, something is probably wrong with memcache.') 193 194 # Since we have no cached buildbot data, we would rather have missing masters than no data at all. 195 buildbot_data = fetch_buildbot_data(MASTERS, True) 196 try: 197 memcache.set('buildbot_data', buildbot_data) 198 except ValueError, err: 199 logging.error(str(err)) 200 201 if callback: 202 buildbot_data = callback + '(' + buildbot_data + ');' 203 204 self.response.out.write(buildbot_data) 205
