1 ''' 2 Downloads the actual gm results most recently generated by the Skia buildbots, 3 and adds any new ones to SVN control. 4 5 Launch with --help to see more information. 6 7 8 Copyright 2011 Google Inc. 9 10 Use of this source code is governed by a BSD-style license that can be 11 found in the LICENSE file. 12 ''' 13 14 # common Python modules 15 import fnmatch 16 import optparse 17 import os 18 import re 19 import shutil 20 import sys 21 import tempfile 22 23 # modules declared within this same directory 24 import compare_baselines 25 import svn 26 27 USAGE_STRING = 'Usage: %s [options] [baseline_subdir]...' 28 HELP_STRING = ''' 29 30 Downloads the actual gm results most recently generated by the Skia buildbots, 31 and adds any new ones to SVN control. 32 33 If no baseline_subdir is given, then this tool will download the most-recently 34 generated actual gm results for ALL platforms. 35 36 ''' + compare_baselines.HOWTO_STRING 37 38 # Base URL of SVN repository where buildbots store actual gm image results. 39 GM_ACTUAL_URL = 'http://skia-autogen.googlecode.com/svn/gm-actual' 40 41 # GM baseline image URL in regular Skia SVN repository 42 GM_BASELINE_URL = 'https://skia.googlecode.com/svn/gm-expected' 43 44 GM_EXPECTED_DIR = 'gm-expected' 45 46 OPTION_ADD_NEW_FILES = '--add-new-files' 47 OPTION_BUILDER_SUFFIX = '--builder-suffix' 48 DEFAULT_BUILDER_SUFFIX = '32' 49 OPTION_IGNORE_LOCAL_MODS = '--ignore-local-mods' 50 51 def GetLatestResultsSvnUrl(svn, baseline_subdir, builder_suffix): 52 """Return SVN URL from which we can check out the MOST RECENTLY generated images for this 53 baseline type. 54 55 @param svn an Svn object we can use to call ListSubdirs() 56 @param baseline_subdir indicates which platform we want images for 57 @param builder_suffix if multiple builders uploaded actual GM images for this baseline type, 58 choose the one whose builder_name matches this suffix 59 """ 60 root_url = '%s/%s' % (GM_ACTUAL_URL, baseline_subdir) 61 subdirs = sorted(svn.ListSubdirs(root_url)) 62 num_subdirs = len(subdirs) 63 print('Within actual-results root URL %s, found these %d subdirs (presumably builder_names): %s' 64 % (root_url, num_subdirs, subdirs)) 65 66 selected_subdir = None 67 if num_subdirs == 0: 68 print 'Found no builder_name subdirs, so reading actual images from the root_url itself.' 69 return root_url 70 elif num_subdirs == 1: 71 selected_subdir = subdirs[0] 72 print 'Found exactly one subdir in actual-results root_url: %s' % selected_subdir 73 else: 74 for possible_subdir in subdirs: 75 if possible_subdir.endswith(builder_suffix): 76 selected_subdir = possible_subdir 77 print 'Selected the first subdir ending in "%s": %s' % ( 78 builder_suffix, selected_subdir) 79 break 80 81 if selected_subdir: 82 return '%s/%s/%s' % (root_url, selected_subdir, baseline_subdir) 83 else: 84 raise Exception('none of these subdirs of %s ended in "%s": %s' % ( 85 root_url, builder_suffix, subdirs)) 86 87 def GetBaselineSvnUrl(baseline_subdir): 88 """Return SVN URL from which we can check out the baseline images for this 89 baseline type. 90 91 @param baseline_subdir indicates which platform we want baselines for 92 """ 93 return '%s/%s' % (GM_BASELINE_URL, baseline_subdir) 94 95 def CopyMatchingFiles(source_dir, dest_dir, filename_pattern, only_copy_updates=False): 96 """Copy all files from source_dir that match filename_pattern, and save them (with their 97 original filenames) in dest_dir. 98 99 @param source_dir 100 @param dest_dir where to save the copied files 101 @param filename_pattern only copy files that match this Unix-style filename 102 pattern (e.g., '*.jpg') 103 @param only_copy_updates if True, only copy files that are already present in dest_dir 104 """ 105 all_filenames = os.listdir(source_dir) 106 matching_filenames = fnmatch.filter(all_filenames, filename_pattern) 107 for filename in matching_filenames: 108 source_path = os.path.join(source_dir, filename) 109 dest_path = os.path.join(dest_dir, filename) 110 if only_copy_updates and not os.path.isfile(dest_path): 111 continue 112 shutil.copyfile(source_path, dest_path) 113 114 def DownloadBaselinesForOnePlatform(baseline_subdir): 115 """Download most recently generated baseline images for a single platform, 116 and add any new ones to SVN control. 117 118 @param baseline_subdir 119 """ 120 # Create repo_to_modify to handle the SVN repository we will add files to. 121 gm_dir = os.path.join(os.pardir, GM_EXPECTED_DIR) # Shouldn't assume we're in trunk... 122 try: 123 os.makedirs(gm_dir) 124 except: 125 pass 126 repo_to_modify = svn.Svn(gm_dir) 127 repo_to_modify.Checkout(GetBaselineSvnUrl(baseline_subdir), baseline_subdir) 128 129 # If there are any locally modified files in that directory, exit 130 # (so that we don't risk overwriting the user's previous work). 131 new_and_modified_files = repo_to_modify.GetNewAndModifiedFiles() 132 if not options.ignore_local_mods: 133 if new_and_modified_files: 134 raise Exception('Exiting because there are already new and/or ' 135 'modified files in %s. To continue in spite of ' 136 'that, run with %s option.' % ( 137 baseline_subdir, OPTION_IGNORE_LOCAL_MODS)) 138 139 # Download actual gm images into a separate repo in a temporary directory. 140 actual_dir = tempfile.mkdtemp() 141 actual_repo = svn.Svn(actual_dir) 142 print 'Using %s as a temp dir' % actual_dir 143 actual_url = GetLatestResultsSvnUrl(svn=actual_repo, baseline_subdir=baseline_subdir, 144 builder_suffix=options.builder_suffix) 145 print 'Reading actual buildbot GM results from %s' % actual_url 146 actual_repo.Checkout(actual_url, '.') 147 148 # Copy any of those files we are interested in into repo_to_modify, 149 # and then delete the temporary directory. 150 CopyMatchingFiles(source_dir=actual_dir, 151 dest_dir=os.path.join(gm_dir, baseline_subdir), 152 filename_pattern='*.png', 153 only_copy_updates=(not options.add_new_files)) 154 shutil.rmtree(actual_dir) 155 actual_repo = None 156 157 # Add any new files to SVN control (if we are running with add_new_files). 158 if options.add_new_files: 159 new_files = repo_to_modify.GetNewFiles() 160 if new_files: 161 repo_to_modify.AddFiles(sorted(new_files)) 162 163 # Set the mimetype property on any new/modified image files in 164 # baseline_subdir. (We used to set the mimetype property on *all* image 165 # files in the directory, even those whose content wasn't changing, 166 # but that caused confusion. See 167 # http://code.google.com/p/skia/issues/detail?id=618 .) 168 modified_files = repo_to_modify.GetNewAndModifiedFiles() 169 repo_to_modify.SetProperty(sorted(fnmatch.filter(modified_files, '*.png')), 170 svn.PROPERTY_MIMETYPE, 'image/png') 171 repo_to_modify.SetProperty(sorted(fnmatch.filter(modified_files, '*.pdf')), 172 svn.PROPERTY_MIMETYPE, 'application/pdf') 173 174 def RaiseUsageException(): 175 raise Exception('%s\nRun with --help for more detail.' % ( 176 USAGE_STRING % __file__)) 177 178 def Main(options, args): 179 """Allow other scripts to call this script with fake command-line args. 180 """ 181 # If no platforms are specified, do 'em all. 182 num_args = len(args) 183 if num_args == 0: 184 # TODO(epoger): automate the default set of platforms. We want to ensure 185 # that the user gets all of the platforms that the bots are running, 186 # not just whatever subdirectories he happens to have checked out... 187 # See http://code.google.com/p/skia/issues/detail?id=678 188 # Now that we have added Svn.ListSubdirs(), we should be able to do this 189 # pretty easily... 190 # 191 # For now, I generate this list using these Unix commands: 192 # svn ls http://skia.googlecode.com/svn/gm-expected | grep ^base | sort >/tmp/baselines 193 # svn ls http://skia-autogen.googlecode.com/svn/gm-actual | grep ^base | sort >/tmp/actual 194 # comm -1 -2 /tmp/baselines /tmp/actual 195 args = [ 196 'base-android-galaxy-nexus', 197 'base-android-nexus-7', 198 'base-android-nexus-s', 199 'base-android-xoom', 200 'base-macmini', 201 'base-macmini-lion-float', 202 'base-shuttle-win7-intel-float', 203 'base-shuttle_ubuntu12_ati5770', 204 'base-shuttle-win7-intel-angle', 205 'base-shuttle-win7-intel-directwrite', 206 ] 207 208 # Trim all subdir names. 209 baseline_subdirs = [] 210 for arg in args: 211 baseline_subdirs.append(arg.rstrip(os.sep)) 212 213 # Process the subdirs, one at a time. 214 for baseline_subdir in baseline_subdirs: 215 DownloadBaselinesForOnePlatform(baseline_subdir=baseline_subdir) 216 217 if __name__ == '__main__': 218 parser = optparse.OptionParser(USAGE_STRING % '%prog' + HELP_STRING) 219 parser.add_option(OPTION_ADD_NEW_FILES, 220 action='store_true', default=False, 221 help='in addition to downloading new versions of ' 222 'existing baselines, also download baselines that are ' 223 'not under SVN control yet') 224 parser.add_option(OPTION_BUILDER_SUFFIX, 225 action='store', type='string', default=DEFAULT_BUILDER_SUFFIX, 226 help='if multiple builders have uploaded actual GM images ' 227 'for this platform, download the images uploaded by the ' 228 'builder whose name ends in this suffix; defaults to ' 229 '"%s".' % DEFAULT_BUILDER_SUFFIX) 230 parser.add_option(OPTION_IGNORE_LOCAL_MODS, 231 action='store_true', default=False, 232 help='allow tool to run even if there are already ' 233 'local modifications in the baseline_subdir') 234 (options, args) = parser.parse_args() 235 Main(options, args) 236
