1 #!/usr/bin/python 2 ''' 3 Copyright 2012 Google Inc. 4 5 Use of this source code is governed by a BSD-style license that can be 6 found in the LICENSE file. 7 ''' 8 9 ''' 10 Generates a visual diff of all pending changes in the local SVN (or git!) 11 checkout. 12 13 Launch with --help to see more information. 14 15 TODO(epoger): Now that this tool supports either git or svn, rename it. 16 TODO(epoger): Fix indentation in this file (2-space indents, not 4-space). 17 ''' 18 19 # common Python modules 20 import optparse 21 import os 22 import posixpath 23 import re 24 import shutil 25 import subprocess 26 import sys 27 import tempfile 28 import urllib2 29 30 # Imports from within Skia 31 # 32 # We need to add the 'gm' directory, so that we can import gm_json.py within 33 # that directory. That script allows us to parse the actual-results.json file 34 # written out by the GM tool. 35 # Make sure that the 'gm' dir is in the PYTHONPATH, but add it at the *end* 36 # so any dirs that are already in the PYTHONPATH will be preferred. 37 # 38 # This assumes that the 'gm' directory has been checked out as a sibling of 39 # the 'tools' directory containing this script, which will be the case if 40 # 'trunk' was checked out as a single unit. 41 GM_DIRECTORY = os.path.realpath( 42 os.path.join(os.path.dirname(os.path.dirname(__file__)), 'gm')) 43 if GM_DIRECTORY not in sys.path: 44 sys.path.append(GM_DIRECTORY) 45 import gm_json 46 import jsondiff 47 import svn 48 49 USAGE_STRING = 'Usage: %s [options]' 50 HELP_STRING = ''' 51 52 Generates a visual diff of all pending changes in the local SVN/git checkout. 53 54 This includes a list of all files that have been added, deleted, or modified 55 (as far as SVN/git knows about). For any image modifications, pixel diffs will 56 be generated. 57 58 ''' 59 60 IMAGE_FILENAME_RE = re.compile(gm_json.IMAGE_FILENAME_PATTERN) 61 62 TRUNK_PATH = os.path.join(os.path.dirname(__file__), os.pardir) 63 64 OPTION_DEST_DIR = '--dest-dir' 65 OPTION_PATH_TO_SKDIFF = '--path-to-skdiff' 66 OPTION_SOURCE_DIR = '--source-dir' 67 68 def RunCommand(command): 69 """Run a command, raising an exception if it fails. 70 71 @param command the command as a single string 72 """ 73 print 'running command [%s]...' % command 74 retval = os.system(command) 75 if retval is not 0: 76 raise Exception('command [%s] failed' % command) 77 78 def FindPathToSkDiff(user_set_path=None): 79 """Return path to an existing skdiff binary, or raise an exception if we 80 cannot find one. 81 82 @param user_set_path if None, the user did not specify a path, so look in 83 some likely places; otherwise, only check at this path 84 """ 85 if user_set_path is not None: 86 if os.path.isfile(user_set_path): 87 return user_set_path 88 raise Exception('unable to find skdiff at user-set path %s' % 89 user_set_path) 90 trunk_path = os.path.join(os.path.dirname(__file__), os.pardir) 91 92 extension = '' 93 if os.name is 'nt': 94 extension = '.exe' 95 96 possible_paths = [os.path.join(trunk_path, 'out', 'Release', 97 'skdiff' + extension), 98 os.path.join(trunk_path, 'out', 'Debug', 99 'skdiff' + extension)] 100 for try_path in possible_paths: 101 if os.path.isfile(try_path): 102 return try_path 103 raise Exception('cannot find skdiff in paths %s; maybe you need to ' 104 'specify the %s option or build skdiff?' % ( 105 possible_paths, OPTION_PATH_TO_SKDIFF)) 106 107 def _DownloadUrlToFile(source_url, dest_path): 108 """Download source_url, and save its contents to dest_path. 109 Raises an exception if there were any problems.""" 110 try: 111 reader = urllib2.urlopen(source_url) 112 writer = open(dest_path, 'wb') 113 writer.write(reader.read()) 114 writer.close() 115 except BaseException as e: 116 raise Exception( 117 '%s: unable to download source_url %s to dest_path %s' % ( 118 e, source_url, dest_path)) 119 120 def _CreateGSUrl(imagename, hash_type, hash_digest): 121 """Return the HTTP URL we can use to download this particular version of 122 the actually-generated GM image with this imagename. 123 124 imagename: name of the test image, e.g. 'perlinnoise_msaa4.png' 125 hash_type: string indicating the hash type used to generate hash_digest, 126 e.g. gm_json.JSONKEY_HASHTYPE_BITMAP_64BITMD5 127 hash_digest: the hash digest of the image to retrieve 128 """ 129 return gm_json.CreateGmActualUrl( 130 test_name=IMAGE_FILENAME_RE.match(imagename).group(1), 131 hash_type=hash_type, 132 hash_digest=hash_digest) 133 134 def _CallJsonDiff(old_json_path, new_json_path, 135 old_flattened_dir, new_flattened_dir, 136 filename_prefix): 137 """Using jsondiff.py, write the images that differ between two GM 138 expectations summary files (old and new) into old_flattened_dir and 139 new_flattened_dir. 140 141 filename_prefix: prefix to prepend to filenames of all images we write 142 into the flattened directories 143 """ 144 json_differ = jsondiff.GMDiffer() 145 diff_dict = json_differ.GenerateDiffDict(oldfile=old_json_path, 146 newfile=new_json_path) 147 print 'Downloading %d before-and-after image pairs...' % len(diff_dict) 148 for (imagename, results) in diff_dict.iteritems(): 149 # TODO(epoger): Currently, this assumes that all images have been 150 # checksummed using gm_json.JSONKEY_HASHTYPE_BITMAP_64BITMD5 151 152 old_checksum = results['old'] 153 if old_checksum: 154 old_image_url = _CreateGSUrl( 155 imagename=imagename, 156 hash_type=gm_json.JSONKEY_HASHTYPE_BITMAP_64BITMD5, 157 hash_digest=old_checksum) 158 _DownloadUrlToFile( 159 source_url=old_image_url, 160 dest_path=os.path.join(old_flattened_dir, 161 filename_prefix + imagename)) 162 163 new_checksum = results['new'] 164 if new_checksum: 165 new_image_url = _CreateGSUrl( 166 imagename=imagename, 167 hash_type=gm_json.JSONKEY_HASHTYPE_BITMAP_64BITMD5, 168 hash_digest=new_checksum) 169 _DownloadUrlToFile( 170 source_url=new_image_url, 171 dest_path=os.path.join(new_flattened_dir, 172 filename_prefix + imagename)) 173 174 def _RunCommand(args): 175 """Run a command (from self._directory) and return stdout as a single 176 string. 177 178 @param args a list of arguments 179 """ 180 proc = subprocess.Popen(args, 181 stdout=subprocess.PIPE, 182 stderr=subprocess.PIPE) 183 (stdout, stderr) = proc.communicate() 184 if proc.returncode is not 0: 185 raise Exception('command "%s" failed: %s' % (args, stderr)) 186 return stdout 187 188 def _GitGetModifiedFiles(): 189 """Returns a list of locally modified files within the current working dir. 190 191 TODO(epoger): Move this into a git utility package? 192 """ 193 return _RunCommand(['git', 'ls-files', '-m']).splitlines() 194 195 def _GitExportBaseVersionOfFile(file_within_repo, dest_path): 196 """Retrieves a copy of the base version of a file within the repository. 197 198 @param file_within_repo path to the file within the repo whose base 199 version you wish to obtain 200 @param dest_path destination to which to write the base content 201 202 TODO(epoger): Move this into a git utility package? 203 """ 204 # TODO(epoger): Replace use of "git show" command with lower-level git 205 # commands? senorblanco points out that "git show" is a "porcelain" 206 # command, intended for human use, as opposed to the "plumbing" commands 207 # generally more suitable for scripting. (See 208 # http://git-scm.com/book/en/Git-Internals-Plumbing-and-Porcelain ) 209 # 210 # For now, though, "git show" is the most straightforward implementation 211 # I could come up with. I tried using "git cat-file", but I had trouble 212 # getting it to work as desired. 213 # Note that git expects / rather than \ as a path separator even on 214 # windows. 215 args = ['git', 'show', posixpath.join('HEAD:.', file_within_repo)] 216 with open(dest_path, 'wb') as outfile: 217 proc = subprocess.Popen(args, stdout=outfile) 218 proc.communicate() 219 if proc.returncode is not 0: 220 raise Exception('command "%s" failed' % args) 221 222 def SvnDiff(path_to_skdiff, dest_dir, source_dir): 223 """Generates a visual diff of all pending changes in source_dir. 224 225 @param path_to_skdiff 226 @param dest_dir existing directory within which to write results 227 @param source_dir 228 """ 229 # Validate parameters, filling in default values if necessary and possible. 230 path_to_skdiff = os.path.abspath(FindPathToSkDiff(path_to_skdiff)) 231 if not dest_dir: 232 dest_dir = tempfile.mkdtemp() 233 dest_dir = os.path.abspath(dest_dir) 234 235 os.chdir(source_dir) 236 svn_repo = svn.Svn('.') 237 using_svn = True 238 try: 239 svn_repo.GetInfo() 240 except: 241 using_svn = False 242 243 # Prepare temporary directories. 244 modified_flattened_dir = os.path.join(dest_dir, 'modified_flattened') 245 original_flattened_dir = os.path.join(dest_dir, 'original_flattened') 246 diff_dir = os.path.join(dest_dir, 'diffs') 247 for dir in [modified_flattened_dir, original_flattened_dir, diff_dir] : 248 shutil.rmtree(dir, ignore_errors=True) 249 os.mkdir(dir) 250 251 # Get a list of all locally modified (including added/deleted) files, 252 # descending subdirectories. 253 if using_svn: 254 modified_file_paths = svn_repo.GetFilesWithStatus( 255 svn.STATUS_ADDED | svn.STATUS_DELETED | svn.STATUS_MODIFIED) 256 else: 257 modified_file_paths = _GitGetModifiedFiles() 258 259 # For each modified file: 260 # 1. copy its current contents into modified_flattened_dir 261 # 2. copy its original contents into original_flattened_dir 262 for modified_file_path in modified_file_paths: 263 if modified_file_path.endswith('.json'): 264 # Special handling for JSON files, in the hopes that they 265 # contain GM result summaries. 266 original_file = tempfile.NamedTemporaryFile(delete = False) 267 original_file.close() 268 if using_svn: 269 svn_repo.ExportBaseVersionOfFile( 270 modified_file_path, original_file.name) 271 else: 272 _GitExportBaseVersionOfFile( 273 modified_file_path, original_file.name) 274 modified_dir = os.path.dirname(modified_file_path) 275 platform_prefix = (re.sub(re.escape(os.sep), '__', 276 os.path.splitdrive(modified_dir)[1]) 277 + '__') 278 _CallJsonDiff(old_json_path=original_file.name, 279 new_json_path=modified_file_path, 280 old_flattened_dir=original_flattened_dir, 281 new_flattened_dir=modified_flattened_dir, 282 filename_prefix=platform_prefix) 283 os.remove(original_file.name) 284 else: 285 dest_filename = re.sub(re.escape(os.sep), '__', modified_file_path) 286 # If the file had STATUS_DELETED, it won't exist anymore... 287 if os.path.isfile(modified_file_path): 288 shutil.copyfile(modified_file_path, 289 os.path.join(modified_flattened_dir, 290 dest_filename)) 291 if using_svn: 292 svn_repo.ExportBaseVersionOfFile( 293 modified_file_path, 294 os.path.join(original_flattened_dir, dest_filename)) 295 else: 296 _GitExportBaseVersionOfFile( 297 modified_file_path, 298 os.path.join(original_flattened_dir, dest_filename)) 299 300 # Run skdiff: compare original_flattened_dir against modified_flattened_dir 301 RunCommand('%s %s %s %s' % (path_to_skdiff, original_flattened_dir, 302 modified_flattened_dir, diff_dir)) 303 print '\nskdiff results are ready in file://%s/index.html' % diff_dir 304 305 def RaiseUsageException(): 306 raise Exception('%s\nRun with --help for more detail.' % ( 307 USAGE_STRING % __file__)) 308 309 def Main(options, args): 310 """Allow other scripts to call this script with fake command-line args. 311 """ 312 num_args = len(args) 313 if num_args != 0: 314 RaiseUsageException() 315 SvnDiff(path_to_skdiff=options.path_to_skdiff, dest_dir=options.dest_dir, 316 source_dir=options.source_dir) 317 318 if __name__ == '__main__': 319 parser = optparse.OptionParser(USAGE_STRING % '%prog' + HELP_STRING) 320 parser.add_option(OPTION_DEST_DIR, 321 action='store', type='string', default=None, 322 help='existing directory within which to write results; ' 323 'if not set, will create a temporary directory which ' 324 'will remain in place after this script completes') 325 parser.add_option(OPTION_PATH_TO_SKDIFF, 326 action='store', type='string', default=None, 327 help='path to already-built skdiff tool; if not set, ' 328 'will search for it in typical directories near this ' 329 'script') 330 parser.add_option(OPTION_SOURCE_DIR, 331 action='store', type='string', 332 default=os.path.join('expectations', 'gm'), 333 help='root directory within which to compare all ' + 334 'files; defaults to "%default"') 335 (options, args) = parser.parse_args() 336 Main(options, args) 337
