Home | History | Annotate | Download | only in rebaseline_server
      1 #!/usr/bin/python
      2 
      3 """
      4 Copyright 2014 Google Inc.
      5 
      6 Use of this source code is governed by a BSD-style license that can be
      7 found in the LICENSE file.
      8 
      9 ImagePairSet class; see its docstring below.
     10 """
     11 
     12 # System-level imports
     13 import posixpath
     14 
     15 # Must fix up PYTHONPATH before importing from within Skia
     16 import rs_fixpypath  # pylint: disable=W0611
     17 
     18 # Imports from within Skia
     19 import column
     20 import imagediffdb
     21 from py.utils import gs_utils
     22 
     23 # Keys used within dictionary representation of ImagePairSet.
     24 # NOTE: Keep these in sync with static/constants.js
     25 KEY__ROOT__EXTRACOLUMNHEADERS = 'extraColumnHeaders'
     26 KEY__ROOT__EXTRACOLUMNORDER = 'extraColumnOrder'
     27 KEY__ROOT__HEADER = 'header'
     28 KEY__ROOT__IMAGEPAIRS = 'imagePairs'
     29 KEY__ROOT__IMAGESETS = 'imageSets'
     30 KEY__IMAGESETS__FIELD__BASE_URL = 'baseUrl'
     31 KEY__IMAGESETS__FIELD__DESCRIPTION = 'description'
     32 KEY__IMAGESETS__SET__DIFFS = 'diffs'
     33 KEY__IMAGESETS__SET__IMAGE_A = 'imageA'
     34 KEY__IMAGESETS__SET__IMAGE_B = 'imageB'
     35 KEY__IMAGESETS__SET__WHITEDIFFS = 'whiteDiffs'
     36 
     37 DEFAULT_DESCRIPTIONS = ('setA', 'setB')
     38 
     39 
     40 class ImagePairSet(object):
     41   """A collection of ImagePairs, representing two arbitrary sets of images.
     42 
     43   These could be:
     44   - images generated before and after a code patch
     45   - expected and actual images for some tests
     46   - or any other pairwise set of images.
     47   """
     48 
     49   def __init__(self, diff_base_url, descriptions=None):
     50     """
     51     Args:
     52       diff_base_url: base URL indicating where diff images can be loaded from
     53       descriptions: a (string, string) tuple describing the two image sets.
     54           If not specified, DEFAULT_DESCRIPTIONS will be used.
     55     """
     56     self._column_header_factories = {}
     57     self._descriptions = descriptions or DEFAULT_DESCRIPTIONS
     58     self._extra_column_tallies = {}  # maps column_id -> values
     59                                      #                -> instances_per_value
     60     self._imageA_base_url = None
     61     self._imageB_base_url = None
     62     self._diff_base_url = diff_base_url
     63 
     64     # We build self._image_pair_objects incrementally as calls come into
     65     # add_image_pair(); self._image_pair_dicts is filled in lazily (so that
     66     # we put off asking ImageDiffDB for results as long as possible).
     67     self._image_pair_objects = []
     68     self._image_pair_dicts = None
     69 
     70   def add_image_pair(self, image_pair):
     71     """Adds an ImagePair; this may be repeated any number of times."""
     72     # Special handling when we add the first ImagePair...
     73     if not self._image_pair_objects:
     74       self._imageA_base_url = image_pair.imageA_base_url
     75       self._imageB_base_url = image_pair.imageB_base_url
     76 
     77     if(image_pair.imageA_base_url != self._imageA_base_url):
     78       raise Exception('added ImagePair with base_url "%s" instead of "%s"' % (
     79           image_pair.imageA_base_url, self._imageA_base_url))
     80     if(image_pair.imageB_base_url != self._imageB_base_url):
     81       raise Exception('added ImagePair with base_url "%s" instead of "%s"' % (
     82           image_pair.imageB_base_url, self._imageB_base_url))
     83     self._image_pair_objects.append(image_pair)
     84     extra_columns_dict = image_pair.extra_columns_dict
     85     if extra_columns_dict:
     86       for column_id, value in extra_columns_dict.iteritems():
     87         self._add_extra_column_value_to_summary(column_id, value)
     88 
     89   def set_column_header_factory(self, column_id, column_header_factory):
     90     """Overrides the default settings for one of the extraColumn headers.
     91 
     92     Args:
     93       column_id: string; unique ID of this column (must match a key within
     94           an ImagePair's extra_columns dictionary)
     95       column_header_factory: a ColumnHeaderFactory object
     96     """
     97     self._column_header_factories[column_id] = column_header_factory
     98 
     99   def get_column_header_factory(self, column_id):
    100     """Returns the ColumnHeaderFactory object for a particular extraColumn.
    101 
    102     Args:
    103       column_id: string; unique ID of this column (must match a key within
    104           an ImagePair's extra_columns dictionary)
    105     """
    106     column_header_factory = self._column_header_factories.get(column_id, None)
    107     if not column_header_factory:
    108       column_header_factory = column.ColumnHeaderFactory(header_text=column_id)
    109       self._column_header_factories[column_id] = column_header_factory
    110     return column_header_factory
    111 
    112   def ensure_extra_column_values_in_summary(self, column_id, values):
    113     """Ensure this column_id/value pair is part of the extraColumns summary.
    114 
    115     Args:
    116       column_id: string; unique ID of this column
    117       value: string; a possible value for this column
    118     """
    119     for value in values:
    120       self._add_extra_column_value_to_summary(
    121           column_id=column_id, value=value, addend=0)
    122 
    123   def _add_extra_column_value_to_summary(self, column_id, value, addend=1):
    124     """Records one column_id/value extraColumns pair found within an ImagePair.
    125 
    126     We use this information to generate tallies within the column header
    127     (how many instances we saw of a particular value, within a particular
    128     extraColumn).
    129 
    130     Args:
    131       column_id: string; unique ID of this column (must match a key within
    132           an ImagePair's extra_columns dictionary)
    133       value: string; a possible value for this column
    134       addend: integer; how many instances to add to the tally
    135     """
    136     known_values_for_column = self._extra_column_tallies.get(column_id, None)
    137     if not known_values_for_column:
    138       known_values_for_column = {}
    139       self._extra_column_tallies[column_id] = known_values_for_column
    140     instances_of_this_value = known_values_for_column.get(value, 0)
    141     instances_of_this_value += addend
    142     known_values_for_column[value] = instances_of_this_value
    143 
    144   def _column_headers_as_dict(self):
    145     """Returns all column headers as a dictionary."""
    146     asdict = {}
    147     for column_id, values_for_column in self._extra_column_tallies.iteritems():
    148       column_header_factory = self.get_column_header_factory(column_id)
    149       asdict[column_id] = column_header_factory.create_as_dict(
    150           values_for_column)
    151     return asdict
    152 
    153   def as_dict(self, column_ids_in_order=None):
    154     """Returns a dictionary describing this package of ImagePairs.
    155 
    156     Uses the KEY__* constants as keys.
    157 
    158     Args:
    159       column_ids_in_order: A list of all extracolumn IDs in the desired display
    160           order.  If unspecified, they will be displayed in alphabetical order.
    161           If specified, this list must contain all the extracolumn IDs!
    162           (It may contain extra column IDs; they will be ignored.)
    163     """
    164     all_column_ids = set(self._extra_column_tallies.keys())
    165     if column_ids_in_order == None:
    166       column_ids_in_order = sorted(all_column_ids)
    167     else:
    168       # Make sure the caller listed all column IDs, and throw away any extras.
    169       specified_column_ids = set(column_ids_in_order)
    170       forgotten_column_ids = all_column_ids - specified_column_ids
    171       assert not forgotten_column_ids, (
    172           'column_ids_in_order %s missing these column_ids: %s' % (
    173               column_ids_in_order, forgotten_column_ids))
    174       column_ids_in_order = [c for c in column_ids_in_order
    175                              if c in all_column_ids]
    176 
    177     key_description = KEY__IMAGESETS__FIELD__DESCRIPTION
    178     key_base_url = KEY__IMAGESETS__FIELD__BASE_URL
    179     if gs_utils.GSUtils.is_gs_url(self._imageA_base_url):
    180       valueA_base_url = self._convert_gs_url_to_http_url(self._imageA_base_url)
    181     else:
    182       valueA_base_url = self._imageA_base_url
    183     if gs_utils.GSUtils.is_gs_url(self._imageB_base_url):
    184       valueB_base_url = self._convert_gs_url_to_http_url(self._imageB_base_url)
    185     else:
    186       valueB_base_url = self._imageB_base_url
    187 
    188     # We've waited as long as we can to ask ImageDiffDB for details of the
    189     # image diffs, so that it has time to compute them.
    190     if self._image_pair_dicts == None:
    191       self._image_pair_dicts = [ip.as_dict() for ip in self._image_pair_objects]
    192 
    193     return {
    194         KEY__ROOT__EXTRACOLUMNHEADERS: self._column_headers_as_dict(),
    195         KEY__ROOT__EXTRACOLUMNORDER: column_ids_in_order,
    196         KEY__ROOT__IMAGEPAIRS: self._image_pair_dicts,
    197         KEY__ROOT__IMAGESETS: {
    198             KEY__IMAGESETS__SET__IMAGE_A: {
    199                 key_description: self._descriptions[0],
    200                 key_base_url: valueA_base_url,
    201             },
    202             KEY__IMAGESETS__SET__IMAGE_B: {
    203                 key_description: self._descriptions[1],
    204                 key_base_url: valueB_base_url,
    205             },
    206             KEY__IMAGESETS__SET__DIFFS: {
    207                 key_description: 'color difference per channel',
    208                 key_base_url: posixpath.join(
    209                     self._diff_base_url, imagediffdb.RGBDIFFS_SUBDIR),
    210             },
    211             KEY__IMAGESETS__SET__WHITEDIFFS: {
    212                 key_description: 'differing pixels in white',
    213                 key_base_url: posixpath.join(
    214                     self._diff_base_url, imagediffdb.WHITEDIFFS_SUBDIR),
    215             },
    216         },
    217     }
    218 
    219   @staticmethod
    220   def _convert_gs_url_to_http_url(gs_url):
    221     """Returns HTTP URL that can be used to download this Google Storage file.
    222 
    223     TODO(epoger): Create functionality like this within gs_utils.py instead of
    224     here?  See https://codereview.chromium.org/428493005/ ('create
    225     anyfile_utils.py for copying files between HTTP/GS/local filesystem')
    226 
    227     Args:
    228       gs_url: "gs://bucket/path" format URL
    229     """
    230     bucket, path = gs_utils.GSUtils.split_gs_url(gs_url)
    231     http_url = 'http://storage.cloud.google.com/' + bucket
    232     if path:
    233       http_url += '/' + path
    234     return http_url
    235