1 #!/usr/bin/python 2 3 """ 4 Copyright 2014 Google Inc. 5 6 Use of this source code is governed by a BSD-style license that can be 7 found in the LICENSE file. 8 9 Download actual GM results for a particular builder. 10 """ 11 12 # System-level imports 13 import optparse 14 import os 15 import posixpath 16 import re 17 import urllib2 18 19 # Imports from within Skia 20 import fix_pythonpath # must do this first 21 from pyutils import gs_utils 22 from pyutils import url_utils 23 import buildbot_globals 24 import gm_json 25 26 27 GM_SUMMARIES_BUCKET = buildbot_globals.Get('gm_summaries_bucket') 28 DEFAULT_ACTUALS_BASE_URL = ( 29 'http://storage.googleapis.com/%s' % GM_SUMMARIES_BUCKET) 30 DEFAULT_JSON_FILENAME = 'actual-results.json' 31 32 33 class Download(object): 34 35 def __init__(self, actuals_base_url=DEFAULT_ACTUALS_BASE_URL, 36 json_filename=DEFAULT_JSON_FILENAME, 37 gm_actuals_root_url=gm_json.GM_ACTUALS_ROOT_HTTP_URL): 38 """ 39 Args: 40 actuals_base_url: URL pointing at the root directory 41 containing all actual-results.json files, e.g., 42 http://domain.name/path/to/dir OR 43 file:///absolute/path/to/localdir 44 json_filename: The JSON filename to read from within each directory. 45 gm_actuals_root_url: Base URL under which the actually-generated-by-bots 46 GM images are stored. 47 """ 48 self._actuals_base_url = actuals_base_url 49 self._json_filename = json_filename 50 self._gm_actuals_root_url = gm_actuals_root_url 51 self._image_filename_re = re.compile(gm_json.IMAGE_FILENAME_PATTERN) 52 53 def fetch(self, builder_name, dest_dir): 54 """ Downloads actual GM results for a particular builder. 55 56 Args: 57 builder_name: which builder to download results of 58 dest_dir: path to directory where the image files will be written; 59 if the directory does not exist yet, it will be created 60 61 TODO(epoger): Display progress info. Right now, it can take a long time 62 to download all of the results, and there is no indication of progress. 63 64 TODO(epoger): Download multiple images in parallel to speed things up. 65 """ 66 json_url = posixpath.join(self._actuals_base_url, builder_name, 67 self._json_filename) 68 json_contents = urllib2.urlopen(json_url).read() 69 results_dict = gm_json.LoadFromString(json_contents) 70 71 actual_results_dict = results_dict[gm_json.JSONKEY_ACTUALRESULTS] 72 for result_type in sorted(actual_results_dict.keys()): 73 results_of_this_type = actual_results_dict[result_type] 74 if not results_of_this_type: 75 continue 76 for image_name in sorted(results_of_this_type.keys()): 77 (test, config) = self._image_filename_re.match(image_name).groups() 78 (hash_type, hash_digest) = results_of_this_type[image_name] 79 source_url = gm_json.CreateGmActualUrl( 80 test_name=test, hash_type=hash_type, hash_digest=hash_digest, 81 gm_actuals_root_url=self._gm_actuals_root_url) 82 dest_path = os.path.join(dest_dir, config, test + '.png') 83 url_utils.copy_contents(source_url=source_url, dest_path=dest_path, 84 create_subdirs_if_needed=True) 85 86 87 def get_builders_list(summaries_bucket=GM_SUMMARIES_BUCKET): 88 """ Returns the list of builders we have actual results for. 89 90 Args: 91 summaries_bucket: Google Cloud Storage bucket containing the summary 92 JSON files 93 """ 94 dirs, _ = gs_utils.list_bucket_contents(bucket=GM_SUMMARIES_BUCKET) 95 return dirs 96 97 98 def main(): 99 parser = optparse.OptionParser() 100 required_params = [] 101 parser.add_option('--actuals-base-url', 102 action='store', type='string', 103 default=DEFAULT_ACTUALS_BASE_URL, 104 help=('Base URL from which to read files containing JSON ' 105 'summaries of actual GM results; defaults to ' 106 '"%default".')) 107 required_params.append('builder') 108 # TODO(epoger): Before https://codereview.chromium.org/309653005 , when this 109 # tool downloaded the JSON summaries from skia-autogen, it had the ability 110 # to get results as of a specific revision number. We should add similar 111 # functionality when retrieving the summaries from Google Storage. 112 parser.add_option('--builder', 113 action='store', type='string', 114 help=('REQUIRED: Which builder to download results for. ' 115 'To see a list of builders, run with the ' 116 '--list-builders option set.')) 117 required_params.append('dest_dir') 118 parser.add_option('--dest-dir', 119 action='store', type='string', 120 help=('REQUIRED: Directory where all images should be ' 121 'written. If this directory does not exist yet, it ' 122 'will be created.')) 123 parser.add_option('--json-filename', 124 action='store', type='string', 125 default=DEFAULT_JSON_FILENAME, 126 help=('JSON summary filename to read for each builder; ' 127 'defaults to "%default".')) 128 parser.add_option('--list-builders', action='store_true', 129 help=('List all available builders.')) 130 (params, remaining_args) = parser.parse_args() 131 132 if params.list_builders: 133 print '\n'.join(get_builders_list()) 134 return 135 136 # Make sure all required options were set, 137 # and that there were no items left over in the command line. 138 for required_param in required_params: 139 if not getattr(params, required_param): 140 raise Exception('required option \'%s\' was not set' % required_param) 141 if len(remaining_args) is not 0: 142 raise Exception('extra items specified in the command line: %s' % 143 remaining_args) 144 145 downloader = Download(actuals_base_url=params.actuals_base_url) 146 downloader.fetch(builder_name=params.builder, 147 dest_dir=params.dest_dir) 148 149 150 151 if __name__ == '__main__': 152 main() 153
