1 #!/usr/bin/python 2 3 """ 4 Copyright 2014 Google Inc. 5 6 Use of this source code is governed by a BSD-style license that can be 7 found in the LICENSE file. 8 9 Download actual GM results for a particular builder. 10 """ 11 12 # System-level imports 13 import optparse 14 import os 15 import posixpath 16 import re 17 import urllib2 18 19 # Must fix up PYTHONPATH before importing from within Skia 20 import rs_fixpypath # pylint: disable=W0611 21 22 # Imports from within Skia 23 from py.utils import gs_utils 24 from py.utils import url_utils 25 import buildbot_globals 26 import gm_json 27 28 29 GM_SUMMARIES_BUCKET = buildbot_globals.Get('gm_summaries_bucket') 30 DEFAULT_ACTUALS_BASE_URL = ( 31 'http://storage.googleapis.com/%s' % GM_SUMMARIES_BUCKET) 32 DEFAULT_JSON_FILENAME = 'actual-results.json' 33 34 35 class Download(object): 36 37 def __init__(self, actuals_base_url=DEFAULT_ACTUALS_BASE_URL, 38 json_filename=DEFAULT_JSON_FILENAME, 39 gm_actuals_root_url=gm_json.GM_ACTUALS_ROOT_HTTP_URL): 40 """ 41 Args: 42 actuals_base_url: URL pointing at the root directory 43 containing all actual-results.json files, e.g., 44 http://domain.name/path/to/dir OR 45 file:///absolute/path/to/localdir 46 json_filename: The JSON filename to read from within each directory. 47 gm_actuals_root_url: Base URL under which the actually-generated-by-bots 48 GM images are stored. 49 """ 50 self._actuals_base_url = actuals_base_url 51 self._json_filename = json_filename 52 self._gm_actuals_root_url = gm_actuals_root_url 53 self._image_filename_re = re.compile(gm_json.IMAGE_FILENAME_PATTERN) 54 55 def fetch(self, builder_name, dest_dir): 56 """ Downloads actual GM results for a particular builder. 57 58 Args: 59 builder_name: which builder to download results of 60 dest_dir: path to directory where the image files will be written; 61 if the directory does not exist yet, it will be created 62 63 TODO(epoger): Display progress info. Right now, it can take a long time 64 to download all of the results, and there is no indication of progress. 65 66 TODO(epoger): Download multiple images in parallel to speed things up. 67 """ 68 json_url = posixpath.join(self._actuals_base_url, builder_name, 69 self._json_filename) 70 json_contents = urllib2.urlopen(json_url).read() 71 results_dict = gm_json.LoadFromString(json_contents) 72 73 actual_results_dict = results_dict[gm_json.JSONKEY_ACTUALRESULTS] 74 for result_type in sorted(actual_results_dict.keys()): 75 results_of_this_type = actual_results_dict[result_type] 76 if not results_of_this_type: 77 continue 78 for image_name in sorted(results_of_this_type.keys()): 79 (test, config) = self._image_filename_re.match(image_name).groups() 80 (hash_type, hash_digest) = results_of_this_type[image_name] 81 source_url = gm_json.CreateGmActualUrl( 82 test_name=test, hash_type=hash_type, hash_digest=hash_digest, 83 gm_actuals_root_url=self._gm_actuals_root_url) 84 dest_path = os.path.join(dest_dir, config, test + '.png') 85 url_utils.copy_contents(source_url=source_url, dest_path=dest_path, 86 create_subdirs_if_needed=True) 87 88 89 def get_builders_list(summaries_bucket=GM_SUMMARIES_BUCKET): 90 """ Returns the list of builders we have actual results for. 91 92 Args: 93 summaries_bucket: Google Cloud Storage bucket containing the summary 94 JSON files 95 """ 96 dirs, _ = gs_utils.GSUtils().list_bucket_contents(bucket=GM_SUMMARIES_BUCKET) 97 return dirs 98 99 100 def main(): 101 parser = optparse.OptionParser() 102 required_params = [] 103 parser.add_option('--actuals-base-url', 104 action='store', type='string', 105 default=DEFAULT_ACTUALS_BASE_URL, 106 help=('Base URL from which to read files containing JSON ' 107 'summaries of actual GM results; defaults to ' 108 '"%default".')) 109 required_params.append('builder') 110 # TODO(epoger): Before https://codereview.chromium.org/309653005 , when this 111 # tool downloaded the JSON summaries from skia-autogen, it had the ability 112 # to get results as of a specific revision number. We should add similar 113 # functionality when retrieving the summaries from Google Storage. 114 parser.add_option('--builder', 115 action='store', type='string', 116 help=('REQUIRED: Which builder to download results for. ' 117 'To see a list of builders, run with the ' 118 '--list-builders option set.')) 119 required_params.append('dest_dir') 120 parser.add_option('--dest-dir', 121 action='store', type='string', 122 help=('REQUIRED: Directory where all images should be ' 123 'written. If this directory does not exist yet, it ' 124 'will be created.')) 125 parser.add_option('--json-filename', 126 action='store', type='string', 127 default=DEFAULT_JSON_FILENAME, 128 help=('JSON summary filename to read for each builder; ' 129 'defaults to "%default".')) 130 parser.add_option('--list-builders', action='store_true', 131 help=('List all available builders.')) 132 (params, remaining_args) = parser.parse_args() 133 134 if params.list_builders: 135 print '\n'.join(get_builders_list()) 136 return 137 138 # Make sure all required options were set, 139 # and that there were no items left over in the command line. 140 for required_param in required_params: 141 if not getattr(params, required_param): 142 raise Exception('required option \'%s\' was not set' % required_param) 143 if len(remaining_args) is not 0: 144 raise Exception('extra items specified in the command line: %s' % 145 remaining_args) 146 147 downloader = Download(actuals_base_url=params.actuals_base_url) 148 downloader.fetch(builder_name=params.builder, 149 dest_dir=params.dest_dir) 150 151 152 153 if __name__ == '__main__': 154 main() 155
