Home | History | Annotate | Download | only in tools 
      1 #!/usr/bin/python
      2 
      3 '''
      4 Copyright 2013 Google Inc.
      5 
      6 Use of this source code is governed by a BSD-style license that can be
      7 found in the LICENSE file.
      8 '''
      9 
     10 '''
     11 Gathers diffs between 2 JSON expectations files, or between actual and
     12 expected results within a single JSON actual-results file,
     13 and generates an old-vs-new diff dictionary.
     14 
     15 TODO(epoger): Fix indentation in this file (2-space indents, not 4-space).
     16 '''
     17 
     18 # System-level imports
     19 import argparse
     20 import json
     21 import os
     22 import sys
     23 import urllib2
     24 
     25 # Imports from within Skia
     26 #
     27 # We need to add the 'gm' directory, so that we can import gm_json.py within
     28 # that directory.  That script allows us to parse the actual-results.json file
     29 # written out by the GM tool.
     30 # Make sure that the 'gm' dir is in the PYTHONPATH, but add it at the *end*
     31 # so any dirs that are already in the PYTHONPATH will be preferred.
     32 #
     33 # This assumes that the 'gm' directory has been checked out as a sibling of
     34 # the 'tools' directory containing this script, which will be the case if
     35 # 'trunk' was checked out as a single unit.
     36 GM_DIRECTORY = os.path.realpath(
     37     os.path.join(os.path.dirname(os.path.dirname(__file__)), 'gm'))
     38 if GM_DIRECTORY not in sys.path:
     39     sys.path.append(GM_DIRECTORY)
     40 import gm_json
     41 
     42 
     43 # Object that generates diffs between two JSON gm result files.
     44 class GMDiffer(object):
     45 
     46     def __init__(self):
     47         pass
     48 
     49     def _GetFileContentsAsString(self, filepath):
     50         """Returns the full contents of a file, as a single string.
     51         If the filename looks like a URL, download its contents.
     52         If the filename is None, return None."""
     53         if filepath is None:
     54             return None
     55         elif filepath.startswith('http:') or filepath.startswith('https:'):
     56             return urllib2.urlopen(filepath).read()
     57         else:
     58             return open(filepath, 'r').read()
     59 
     60     def _GetExpectedResults(self, contents):
     61         """Returns the dictionary of expected results from a JSON string,
     62         in this form:
     63 
     64         {
     65           'test1' : 14760033689012826769,
     66           'test2' : 9151974350149210736,
     67           ...
     68         }
     69 
     70         We make these simplifying assumptions:
     71         1. Each test has either 0 or 1 allowed results.
     72         2. All expectations are of type JSONKEY_HASHTYPE_BITMAP_64BITMD5.
     73 
     74         Any tests which violate those assumptions will cause an exception to
     75         be raised.
     76 
     77         Any tests for which we have no expectations will be left out of the
     78         returned dictionary.
     79         """
     80         result_dict = {}
     81         json_dict = gm_json.LoadFromString(contents)
     82         all_expectations = json_dict[gm_json.JSONKEY_EXPECTEDRESULTS]
     83 
     84         # Prevent https://code.google.com/p/skia/issues/detail?id=1588
     85         if not all_expectations:
     86             return result_dict
     87 
     88         for test_name in all_expectations.keys():
     89             test_expectations = all_expectations[test_name]
     90             allowed_digests = test_expectations[
     91                 gm_json.JSONKEY_EXPECTEDRESULTS_ALLOWEDDIGESTS]
     92             if allowed_digests:
     93                 num_allowed_digests = len(allowed_digests)
     94                 if num_allowed_digests > 1:
     95                     raise ValueError(
     96                         'test %s has %d allowed digests' % (
     97                             test_name, num_allowed_digests))
     98                 digest_pair = allowed_digests[0]
     99                 if digest_pair[0] != gm_json.JSONKEY_HASHTYPE_BITMAP_64BITMD5:
    100                     raise ValueError(
    101                         'test %s has unsupported hashtype %s' % (
    102                             test_name, digest_pair[0]))
    103                 result_dict[test_name] = digest_pair[1]
    104         return result_dict
    105 
    106     def _GetActualResults(self, contents):
    107         """Returns the dictionary of actual results from a JSON string,
    108         in this form:
    109 
    110         {
    111           'test1' : 14760033689012826769,
    112           'test2' : 9151974350149210736,
    113           ...
    114         }
    115 
    116         We make these simplifying assumptions:
    117         1. All results are of type JSONKEY_HASHTYPE_BITMAP_64BITMD5.
    118 
    119         Any tests which violate those assumptions will cause an exception to
    120         be raised.
    121 
    122         Any tests for which we have no actual results will be left out of the
    123         returned dictionary.
    124         """
    125         result_dict = {}
    126         json_dict = gm_json.LoadFromString(contents)
    127         all_result_types = json_dict[gm_json.JSONKEY_ACTUALRESULTS]
    128         for result_type in all_result_types.keys():
    129             results_of_this_type = all_result_types[result_type]
    130             if results_of_this_type:
    131                 for test_name in results_of_this_type.keys():
    132                     digest_pair = results_of_this_type[test_name]
    133                     if digest_pair[0] != gm_json.JSONKEY_HASHTYPE_BITMAP_64BITMD5:
    134                         raise ValueError(
    135                             'test %s has unsupported hashtype %s' % (
    136                                 test_name, digest_pair[0]))
    137                     result_dict[test_name] = digest_pair[1]
    138         return result_dict
    139 
    140     def _DictionaryDiff(self, old_dict, new_dict):
    141         """Generate a dictionary showing the diffs between old_dict and new_dict.
    142         Any entries which are identical across them will be left out."""
    143         diff_dict = {}
    144         all_keys = set(old_dict.keys() + new_dict.keys())
    145         for key in all_keys:
    146             if old_dict.get(key) != new_dict.get(key):
    147                 new_entry = {}
    148                 new_entry['old'] = old_dict.get(key)
    149                 new_entry['new'] = new_dict.get(key)
    150                 diff_dict[key] = new_entry
    151         return diff_dict
    152 
    153     def GenerateDiffDict(self, oldfile, newfile=None):
    154         """Generate a dictionary showing the diffs:
    155         old = expectations within oldfile
    156         new = expectations within newfile
    157 
    158         If newfile is not specified, then 'new' is the actual results within
    159         oldfile.
    160         """
    161         return self.GenerateDiffDictFromStrings(self._GetFileContentsAsString(oldfile),
    162                                                 self._GetFileContentsAsString(newfile))
    163 
    164     def GenerateDiffDictFromStrings(self, oldjson, newjson=None):
    165         """Generate a dictionary showing the diffs:
    166         old = expectations within oldjson
    167         new = expectations within newjson
    168 
    169         If newfile is not specified, then 'new' is the actual results within
    170         oldfile.
    171         """
    172         old_results = self._GetExpectedResults(oldjson)
    173         if newjson:
    174             new_results = self._GetExpectedResults(newjson)
    175         else:
    176             new_results = self._GetActualResults(oldjson)
    177         return self._DictionaryDiff(old_results, new_results)
    178 
    179 
    180 def _Main():
    181     parser = argparse.ArgumentParser()
    182     parser.add_argument(
    183         'old',
    184         help='Path to JSON file whose expectations to display on ' +
    185         'the "old" side of the diff. This can be a filepath on ' +
    186         'local storage, or a URL.')
    187     parser.add_argument(
    188         'new', nargs='?',
    189         help='Path to JSON file whose expectations to display on ' +
    190         'the "new" side of the diff; if not specified, uses the ' +
    191         'ACTUAL results from the "old" JSON file. This can be a ' +
    192         'filepath on local storage, or a URL.')
    193     args = parser.parse_args()
    194     differ = GMDiffer()
    195     diffs = differ.GenerateDiffDict(oldfile=args.old, newfile=args.new)
    196     json.dump(diffs, sys.stdout, sort_keys=True, indent=2)
    197 
    198 
    199 if __name__ == '__main__':
    200     _Main()
    201