Home | History | Annotate | Download | only in tests
      1 #!/usr/bin/python
      2 
      3 """
      4 Copyright 2014 Google Inc.
      5 
      6 Use of this source code is governed by a BSD-style license that can be
      7 found in the LICENSE file.
      8 
      9 Test the render_pictures binary.
     10 """
     11 
     12 # System-level imports
     13 import copy
     14 import json
     15 import os
     16 import shutil
     17 import tempfile
     18 
     19 # Imports from within Skia
     20 import base_unittest
     21 
     22 # Maximum length of text diffs to show when tests fail
     23 MAX_DIFF_LENGTH = 30000
     24 
     25 EXPECTED_HEADER_CONTENTS = {
     26     "type" : "ChecksummedImages",
     27     "revision" : 1,
     28 }
     29 
     30 # Manually verified: 640x400 red rectangle with black border
     31 # Standard expectations will be set up in such a way that this image fails
     32 # the comparison.
     33 RED_WHOLEIMAGE = {
     34     "checksumAlgorithm" : "bitmap-64bitMD5",
     35     "checksumValue" : 11092453015575919668,
     36     "comparisonResult" : "failed",
     37     "filepath" : "red_skp.png",
     38 }
     39 
     40 # Manually verified: 640x400 green rectangle with black border
     41 # Standard expectations will be set up in such a way that this image passes
     42 # the comparison.
     43 GREEN_WHOLEIMAGE = {
     44     "checksumAlgorithm" : "bitmap-64bitMD5",
     45     "checksumValue" : 8891695120562235492,
     46     "comparisonResult" : "succeeded",
     47     "filepath" : "green_skp.png",
     48 }
     49 
     50 # Manually verified these 6 images, all 256x256 tiles,
     51 # consistent with a tiled version of the 640x400 red rect
     52 # with black borders.
     53 # Standard expectations will be set up in such a way that these images fail
     54 # the comparison.
     55 RED_TILES = [{
     56     "checksumAlgorithm" : "bitmap-64bitMD5",
     57     "checksumValue" : 5815827069051002745,
     58     "comparisonResult" : "failed",
     59     "filepath" : "red_skp-tile0.png",
     60 },{
     61     "checksumAlgorithm" : "bitmap-64bitMD5",
     62     "checksumValue" : 9323613075234140270,
     63     "comparisonResult" : "failed",
     64     "filepath" : "red_skp-tile1.png",
     65 }, {
     66     "checksumAlgorithm" : "bitmap-64bitMD5",
     67     "checksumValue" : 16670399404877552232,
     68     "comparisonResult" : "failed",
     69     "filepath" : "red_skp-tile2.png",
     70 }, {
     71     "checksumAlgorithm" : "bitmap-64bitMD5",
     72     "checksumValue" : 2507897274083364964,
     73     "comparisonResult" : "failed",
     74     "filepath" : "red_skp-tile3.png",
     75 }, {
     76     "checksumAlgorithm" : "bitmap-64bitMD5",
     77     "checksumValue" : 7325267995523877959,
     78     "comparisonResult" : "failed",
     79     "filepath" : "red_skp-tile4.png",
     80 }, {
     81     "checksumAlgorithm" : "bitmap-64bitMD5",
     82     "checksumValue" : 2181381724594493116,
     83     "comparisonResult" : "failed",
     84     "filepath" : "red_skp-tile5.png",
     85 }]
     86 
     87 # Manually verified these 6 images, all 256x256 tiles,
     88 # consistent with a tiled version of the 640x400 green rect
     89 # with black borders.
     90 # Standard expectations will be set up in such a way that these images pass
     91 # the comparison.
     92 GREEN_TILES = [{
     93     "checksumAlgorithm" : "bitmap-64bitMD5",
     94     "checksumValue" : 12587324416545178013,
     95     "comparisonResult" : "succeeded",
     96     "filepath" : "green_skp-tile0.png",
     97 }, {
     98     "checksumAlgorithm" : "bitmap-64bitMD5",
     99     "checksumValue" : 7624374914829746293,
    100     "comparisonResult" : "succeeded",
    101     "filepath" : "green_skp-tile1.png",
    102 }, {
    103     "checksumAlgorithm" : "bitmap-64bitMD5",
    104     "checksumValue" : 5686489729535631913,
    105     "comparisonResult" : "succeeded",
    106     "filepath" : "green_skp-tile2.png",
    107 }, {
    108     "checksumAlgorithm" : "bitmap-64bitMD5",
    109     "checksumValue" : 7980646035555096146,
    110     "comparisonResult" : "succeeded",
    111     "filepath" : "green_skp-tile3.png",
    112 }, {
    113     "checksumAlgorithm" : "bitmap-64bitMD5",
    114     "checksumValue" : 17817086664365875131,
    115     "comparisonResult" : "succeeded",
    116     "filepath" : "green_skp-tile4.png",
    117 }, {
    118     "checksumAlgorithm" : "bitmap-64bitMD5",
    119     "checksumValue" : 10673669813016809363,
    120     "comparisonResult" : "succeeded",
    121     "filepath" : "green_skp-tile5.png",
    122 }]
    123 
    124 
    125 def modified_dict(input_dict, modification_dict):
    126   """Returns a dict, with some modifications applied to it.
    127 
    128   Args:
    129     input_dict: a dictionary (which will be copied, not modified in place)
    130     modification_dict: a set of key/value pairs to overwrite in the dict
    131   """
    132   output_dict = input_dict.copy()
    133   output_dict.update(modification_dict)
    134   return output_dict
    135 
    136 
    137 def modified_list_of_dicts(input_list, modification_dict):
    138   """Returns a list of dicts, with some modifications applied to each dict.
    139 
    140   Args:
    141     input_list: a list of dictionaries; these dicts will be copied, not
    142         modified in place
    143     modification_dict: a set of key/value pairs to overwrite in each dict
    144         within input_list
    145   """
    146   output_list = []
    147   for input_dict in input_list:
    148     output_dict = modified_dict(input_dict, modification_dict)
    149     output_list.append(output_dict)
    150   return output_list
    151 
    152 
    153 class RenderPicturesTest(base_unittest.TestCase):
    154 
    155   def setUp(self):
    156     self.maxDiff = MAX_DIFF_LENGTH
    157     self._expectations_dir = tempfile.mkdtemp()
    158     self._input_skp_dir = tempfile.mkdtemp()
    159     # All output of render_pictures binary will go into this directory.
    160     self._output_dir = tempfile.mkdtemp()
    161 
    162   def tearDown(self):
    163     shutil.rmtree(self._expectations_dir)
    164     shutil.rmtree(self._input_skp_dir)
    165     shutil.rmtree(self._output_dir)
    166 
    167   def test_tiled_whole_image(self):
    168     """Run render_pictures with tiles and --writeWholeImage flag.
    169 
    170     TODO(epoger): This test generates undesired results!  The JSON summary
    171     includes both whole-image and tiled-images (as it should), but only
    172     whole-images are written out to disk.  See http://skbug.com/2463
    173     Once I fix that, I should add a similar test that exercises mismatchPath.
    174 
    175     TODO(epoger): I noticed that when this is run without --writePath being
    176     specified, this test writes red_skp.png and green_skp.png to the current
    177     directory.  We should fix that... if --writePath is not specified, this
    178     probably shouldn't write out red_skp.png and green_skp.png at all!
    179     See http://skbug.com/2464
    180     """
    181     output_json_path = os.path.join(self._output_dir, 'actuals.json')
    182     write_path_dir = self.create_empty_dir(
    183         path=os.path.join(self._output_dir, 'writePath'))
    184     self._generate_skps()
    185     expectations_path = self._create_expectations()
    186     self._run_render_pictures([
    187         '-r', self._input_skp_dir,
    188         '--bbh', 'grid', '256', '256',
    189         '--mode', 'tile', '256', '256',
    190         '--readJsonSummaryPath', expectations_path,
    191         '--writeJsonSummaryPath', output_json_path,
    192         '--writePath', write_path_dir,
    193         '--writeWholeImage'])
    194     expected_summary_dict = {
    195         "header" : EXPECTED_HEADER_CONTENTS,
    196         "actual-results" : {
    197             "red.skp": {
    198                 "tiled-images": RED_TILES,
    199                 "whole-image": RED_WHOLEIMAGE,
    200             },
    201             "green.skp": {
    202                 "tiled-images": GREEN_TILES,
    203                 "whole-image": GREEN_WHOLEIMAGE,
    204             }
    205         }
    206     }
    207     self._assert_json_contents(output_json_path, expected_summary_dict)
    208     self._assert_directory_contents(
    209         write_path_dir, ['red_skp.png', 'green_skp.png'])
    210 
    211   def test_missing_tile_and_whole_image(self):
    212     """test_tiled_whole_image, but missing expectations for some images.
    213     """
    214     output_json_path = os.path.join(self._output_dir, 'actuals.json')
    215     write_path_dir = self.create_empty_dir(
    216         path=os.path.join(self._output_dir, 'writePath'))
    217     self._generate_skps()
    218     expectations_path = self._create_expectations(missing_some_images=True)
    219     self._run_render_pictures([
    220         '-r', self._input_skp_dir,
    221         '--bbh', 'grid', '256', '256',
    222         '--mode', 'tile', '256', '256',
    223         '--readJsonSummaryPath', expectations_path,
    224         '--writeJsonSummaryPath', output_json_path,
    225         '--writePath', write_path_dir,
    226         '--writeWholeImage'])
    227     modified_red_tiles = copy.deepcopy(RED_TILES)
    228     modified_red_tiles[5]['comparisonResult'] = 'no-comparison'
    229     expected_summary_dict = {
    230         "header" : EXPECTED_HEADER_CONTENTS,
    231         "actual-results" : {
    232             "red.skp": {
    233                 "tiled-images": modified_red_tiles,
    234                 "whole-image": modified_dict(
    235                     RED_WHOLEIMAGE, {"comparisonResult" : "no-comparison"}),
    236             },
    237             "green.skp": {
    238                 "tiled-images": GREEN_TILES,
    239                 "whole-image": GREEN_WHOLEIMAGE,
    240             }
    241         }
    242     }
    243     self._assert_json_contents(output_json_path, expected_summary_dict)
    244 
    245   def _test_untiled(self, expectations_path=None, expected_summary_dict=None,
    246                     additional_args=None):
    247     """Base for multiple tests without tiles.
    248 
    249     Args:
    250       expectations_path: path we should pass using --readJsonSummaryPath, or
    251           None if we should create the default expectations file
    252       expected_summary_dict: dict we should compare against the output actual
    253           results summary, or None if we should use a default comparison dict
    254       additional_args: array of command-line args to add when we run
    255           render_pictures
    256     """
    257     output_json_path = os.path.join(self._output_dir, 'actuals.json')
    258     write_path_dir = self.create_empty_dir(
    259         path=os.path.join(self._output_dir, 'writePath'))
    260     self._generate_skps()
    261     if expectations_path == None:
    262       expectations_path = self._create_expectations()
    263     args = [
    264         '-r', self._input_skp_dir,
    265         '--readJsonSummaryPath', expectations_path,
    266         '--writePath', write_path_dir,
    267         '--writeJsonSummaryPath', output_json_path,
    268     ]
    269     if additional_args:
    270       args.extend(additional_args)
    271     self._run_render_pictures(args)
    272     if expected_summary_dict == None:
    273       expected_summary_dict = {
    274           "header" : EXPECTED_HEADER_CONTENTS,
    275           "actual-results" : {
    276               "red.skp": {
    277                   "whole-image": RED_WHOLEIMAGE,
    278               },
    279               "green.skp": {
    280                   "whole-image": GREEN_WHOLEIMAGE,
    281               }
    282           }
    283       }
    284     self._assert_json_contents(output_json_path, expected_summary_dict)
    285     self._assert_directory_contents(
    286         write_path_dir, ['red_skp.png', 'green_skp.png'])
    287 
    288   def test_untiled(self):
    289     """Basic test without tiles."""
    290     self._test_untiled()
    291 
    292   def test_untiled_empty_expectations_file(self):
    293     """Same as test_untiled, but with an empty expectations file."""
    294     expectations_path = os.path.join(self._expectations_dir, 'empty')
    295     with open(expectations_path, 'w') as fh:
    296       pass
    297     expected_summary_dict = {
    298         "header" : EXPECTED_HEADER_CONTENTS,
    299         "actual-results" : {
    300             "red.skp": {
    301                 "whole-image": modified_dict(
    302                     RED_WHOLEIMAGE, {"comparisonResult" : "no-comparison"}),
    303             },
    304             "green.skp": {
    305                 "whole-image": modified_dict(
    306                     GREEN_WHOLEIMAGE, {"comparisonResult" : "no-comparison"}),
    307             }
    308         }
    309     }
    310     self._test_untiled(expectations_path=expectations_path,
    311                        expected_summary_dict=expected_summary_dict)
    312 
    313   def test_untiled_writeChecksumBasedFilenames(self):
    314     """Same as test_untiled, but with --writeChecksumBasedFilenames."""
    315     output_json_path = os.path.join(self._output_dir, 'actuals.json')
    316     write_path_dir = self.create_empty_dir(
    317         path=os.path.join(self._output_dir, 'writePath'))
    318     self._generate_skps()
    319     self._run_render_pictures(['-r', self._input_skp_dir,
    320                                '--writeChecksumBasedFilenames',
    321                                '--writePath', write_path_dir,
    322                                '--writeJsonSummaryPath', output_json_path])
    323     expected_summary_dict = {
    324         "header" : EXPECTED_HEADER_CONTENTS,
    325         "actual-results" : {
    326             "red.skp": {
    327                 # Manually verified: 640x400 red rectangle with black border
    328                 "whole-image": {
    329                     "checksumAlgorithm" : "bitmap-64bitMD5",
    330                     "checksumValue" : 11092453015575919668,
    331                     "comparisonResult" : "no-comparison",
    332                     "filepath" : "red_skp/bitmap-64bitMD5_11092453015575919668.png",
    333                 },
    334             },
    335             "green.skp": {
    336                 # Manually verified: 640x400 green rectangle with black border
    337                 "whole-image": {
    338                     "checksumAlgorithm" : "bitmap-64bitMD5",
    339                     "checksumValue" : 8891695120562235492,
    340                     "comparisonResult" : "no-comparison",
    341                     "filepath" : "green_skp/bitmap-64bitMD5_8891695120562235492.png",
    342                 },
    343             }
    344         }
    345     }
    346     self._assert_json_contents(output_json_path, expected_summary_dict)
    347     self._assert_directory_contents(write_path_dir, ['red_skp', 'green_skp'])
    348     self._assert_directory_contents(
    349         os.path.join(write_path_dir, 'red_skp'),
    350         ['bitmap-64bitMD5_11092453015575919668.png'])
    351     self._assert_directory_contents(
    352         os.path.join(write_path_dir, 'green_skp'),
    353         ['bitmap-64bitMD5_8891695120562235492.png'])
    354 
    355   def test_untiled_validate(self):
    356     """Same as test_untiled, but with --validate."""
    357     self._test_untiled(additional_args=['--validate'])
    358 
    359   def test_untiled_without_writePath(self):
    360     """Same as test_untiled, but without --writePath."""
    361     output_json_path = os.path.join(self._output_dir, 'actuals.json')
    362     self._generate_skps()
    363     expectations_path = self._create_expectations()
    364     self._run_render_pictures([
    365         '-r', self._input_skp_dir,
    366         '--readJsonSummaryPath', expectations_path,
    367         '--writeJsonSummaryPath', output_json_path])
    368     expected_summary_dict = {
    369         "header" : EXPECTED_HEADER_CONTENTS,
    370         "actual-results" : {
    371             "red.skp": {
    372                 "whole-image": RED_WHOLEIMAGE,
    373             },
    374             "green.skp": {
    375                 "whole-image": GREEN_WHOLEIMAGE,
    376             }
    377         }
    378     }
    379     self._assert_json_contents(output_json_path, expected_summary_dict)
    380 
    381   def test_tiled(self):
    382     """Generate individual tiles."""
    383     output_json_path = os.path.join(self._output_dir, 'actuals.json')
    384     write_path_dir = self.create_empty_dir(
    385         path=os.path.join(self._output_dir, 'writePath'))
    386     self._generate_skps()
    387     expectations_path = self._create_expectations()
    388     self._run_render_pictures([
    389         '-r', self._input_skp_dir,
    390         '--bbh', 'grid', '256', '256',
    391         '--mode', 'tile', '256', '256',
    392         '--readJsonSummaryPath', expectations_path,
    393         '--writePath', write_path_dir,
    394         '--writeJsonSummaryPath', output_json_path])
    395     expected_summary_dict = {
    396         "header" : EXPECTED_HEADER_CONTENTS,
    397         "actual-results" : {
    398             "red.skp": {
    399                 "tiled-images": RED_TILES,
    400             },
    401             "green.skp": {
    402                 "tiled-images": GREEN_TILES,
    403             }
    404         }
    405     }
    406     self._assert_json_contents(output_json_path, expected_summary_dict)
    407     self._assert_directory_contents(
    408         write_path_dir,
    409         ['red_skp-tile0.png', 'red_skp-tile1.png', 'red_skp-tile2.png',
    410          'red_skp-tile3.png', 'red_skp-tile4.png', 'red_skp-tile5.png',
    411          'green_skp-tile0.png', 'green_skp-tile1.png', 'green_skp-tile2.png',
    412          'green_skp-tile3.png', 'green_skp-tile4.png', 'green_skp-tile5.png',
    413         ])
    414 
    415   def test_tiled_mismatches(self):
    416     """Same as test_tiled, but only write out mismatching images."""
    417     output_json_path = os.path.join(self._output_dir, 'actuals.json')
    418     mismatch_path_dir = self.create_empty_dir(
    419         path=os.path.join(self._output_dir, 'mismatchPath'))
    420     self._generate_skps()
    421     expectations_path = self._create_expectations()
    422     self._run_render_pictures([
    423         '-r', self._input_skp_dir,
    424         '--bbh', 'grid', '256', '256',
    425         '--mode', 'tile', '256', '256',
    426         '--readJsonSummaryPath', expectations_path,
    427         '--mismatchPath', mismatch_path_dir,
    428         '--writeJsonSummaryPath', output_json_path])
    429     expected_summary_dict = {
    430         "header" : EXPECTED_HEADER_CONTENTS,
    431         "actual-results" : {
    432             "red.skp": {
    433                 "tiled-images": RED_TILES,
    434             },
    435             "green.skp": {
    436                 "tiled-images": GREEN_TILES,
    437             }
    438         }
    439     }
    440     self._assert_json_contents(output_json_path, expected_summary_dict)
    441     self._assert_directory_contents(
    442         mismatch_path_dir,
    443         ['red_skp-tile0.png', 'red_skp-tile1.png', 'red_skp-tile2.png',
    444          'red_skp-tile3.png', 'red_skp-tile4.png', 'red_skp-tile5.png',
    445         ])
    446 
    447   def test_tiled_writeChecksumBasedFilenames(self):
    448     """Same as test_tiled, but with --writeChecksumBasedFilenames."""
    449     output_json_path = os.path.join(self._output_dir, 'actuals.json')
    450     write_path_dir = self.create_empty_dir(
    451         path=os.path.join(self._output_dir, 'writePath'))
    452     self._generate_skps()
    453     self._run_render_pictures(['-r', self._input_skp_dir,
    454                                '--bbh', 'grid', '256', '256',
    455                                '--mode', 'tile', '256', '256',
    456                                '--writeChecksumBasedFilenames',
    457                                '--writePath', write_path_dir,
    458                                '--writeJsonSummaryPath', output_json_path])
    459     expected_summary_dict = {
    460         "header" : EXPECTED_HEADER_CONTENTS,
    461         "actual-results" : {
    462             "red.skp": {
    463                 # Manually verified these 6 images, all 256x256 tiles,
    464                 # consistent with a tiled version of the 640x400 red rect
    465                 # with black borders.
    466                 "tiled-images": [{
    467                     "checksumAlgorithm" : "bitmap-64bitMD5",
    468                     "checksumValue" : 5815827069051002745,
    469                     "comparisonResult" : "no-comparison",
    470                     "filepath" : "red_skp/bitmap-64bitMD5_5815827069051002745.png",
    471                 }, {
    472                     "checksumAlgorithm" : "bitmap-64bitMD5",
    473                     "checksumValue" : 9323613075234140270,
    474                     "comparisonResult" : "no-comparison",
    475                     "filepath" : "red_skp/bitmap-64bitMD5_9323613075234140270.png",
    476                 }, {
    477                     "checksumAlgorithm" : "bitmap-64bitMD5",
    478                     "checksumValue" : 16670399404877552232,
    479                     "comparisonResult" : "no-comparison",
    480                     "filepath" : "red_skp/bitmap-64bitMD5_16670399404877552232.png",
    481                 }, {
    482                     "checksumAlgorithm" : "bitmap-64bitMD5",
    483                     "checksumValue" : 2507897274083364964,
    484                     "comparisonResult" : "no-comparison",
    485                     "filepath" : "red_skp/bitmap-64bitMD5_2507897274083364964.png",
    486                 }, {
    487                     "checksumAlgorithm" : "bitmap-64bitMD5",
    488                     "checksumValue" : 7325267995523877959,
    489                     "comparisonResult" : "no-comparison",
    490                     "filepath" : "red_skp/bitmap-64bitMD5_7325267995523877959.png",
    491                 }, {
    492                     "checksumAlgorithm" : "bitmap-64bitMD5",
    493                     "checksumValue" : 2181381724594493116,
    494                     "comparisonResult" : "no-comparison",
    495                     "filepath" : "red_skp/bitmap-64bitMD5_2181381724594493116.png",
    496                 }],
    497             },
    498             "green.skp": {
    499                 # Manually verified these 6 images, all 256x256 tiles,
    500                 # consistent with a tiled version of the 640x400 green rect
    501                 # with black borders.
    502                 "tiled-images": [{
    503                     "checksumAlgorithm" : "bitmap-64bitMD5",
    504                     "checksumValue" : 12587324416545178013,
    505                     "comparisonResult" : "no-comparison",
    506                     "filepath" : "green_skp/bitmap-64bitMD5_12587324416545178013.png",
    507                 }, {
    508                     "checksumAlgorithm" : "bitmap-64bitMD5",
    509                     "checksumValue" : 7624374914829746293,
    510                     "comparisonResult" : "no-comparison",
    511                     "filepath" : "green_skp/bitmap-64bitMD5_7624374914829746293.png",
    512                 }, {
    513                     "checksumAlgorithm" : "bitmap-64bitMD5",
    514                     "checksumValue" : 5686489729535631913,
    515                     "comparisonResult" : "no-comparison",
    516                     "filepath" : "green_skp/bitmap-64bitMD5_5686489729535631913.png",
    517                 }, {
    518                     "checksumAlgorithm" : "bitmap-64bitMD5",
    519                     "checksumValue" : 7980646035555096146,
    520                     "comparisonResult" : "no-comparison",
    521                     "filepath" : "green_skp/bitmap-64bitMD5_7980646035555096146.png",
    522                 }, {
    523                     "checksumAlgorithm" : "bitmap-64bitMD5",
    524                     "checksumValue" : 17817086664365875131,
    525                     "comparisonResult" : "no-comparison",
    526                     "filepath" : "green_skp/bitmap-64bitMD5_17817086664365875131.png",
    527                 }, {
    528                     "checksumAlgorithm" : "bitmap-64bitMD5",
    529                     "checksumValue" : 10673669813016809363,
    530                     "comparisonResult" : "no-comparison",
    531                     "filepath" : "green_skp/bitmap-64bitMD5_10673669813016809363.png",
    532                 }],
    533             }
    534         }
    535     }
    536     self._assert_json_contents(output_json_path, expected_summary_dict)
    537     self._assert_directory_contents(write_path_dir, ['red_skp', 'green_skp'])
    538     self._assert_directory_contents(
    539         os.path.join(write_path_dir, 'red_skp'),
    540         ['bitmap-64bitMD5_5815827069051002745.png',
    541          'bitmap-64bitMD5_9323613075234140270.png',
    542          'bitmap-64bitMD5_16670399404877552232.png',
    543          'bitmap-64bitMD5_2507897274083364964.png',
    544          'bitmap-64bitMD5_7325267995523877959.png',
    545          'bitmap-64bitMD5_2181381724594493116.png'])
    546     self._assert_directory_contents(
    547         os.path.join(write_path_dir, 'green_skp'),
    548         ['bitmap-64bitMD5_12587324416545178013.png',
    549          'bitmap-64bitMD5_7624374914829746293.png',
    550          'bitmap-64bitMD5_5686489729535631913.png',
    551          'bitmap-64bitMD5_7980646035555096146.png',
    552          'bitmap-64bitMD5_17817086664365875131.png',
    553          'bitmap-64bitMD5_10673669813016809363.png'])
    554 
    555   def _run_render_pictures(self, args):
    556     binary = self.find_path_to_program('render_pictures')
    557     return self.run_command([binary,
    558                              '--clone', '1',
    559                              '--config', '8888',
    560                              ] + args)
    561 
    562   def _create_expectations(self, missing_some_images=False,
    563                            rel_path='expectations.json'):
    564     """Creates expectations JSON file within self._expectations_dir .
    565 
    566     Args:
    567       missing_some_images: (bool) whether to remove expectations for a subset
    568           of the images
    569       rel_path: (string) relative path within self._expectations_dir to write
    570           the expectations into
    571 
    572     Returns: full path to the expectations file created.
    573     """
    574     expectations_dict = {
    575         "header" : EXPECTED_HEADER_CONTENTS,
    576         "expected-results" : {
    577             # red.skp: these should fail the comparison
    578             "red.skp": {
    579                 "tiled-images": modified_list_of_dicts(
    580                     RED_TILES, {'checksumValue': 11111}),
    581                 "whole-image": modified_dict(
    582                     RED_WHOLEIMAGE, {'checksumValue': 22222}),
    583             },
    584             # green.skp: these should pass the comparison
    585             "green.skp": {
    586                 "tiled-images": GREEN_TILES,
    587                 "whole-image": GREEN_WHOLEIMAGE,
    588             }
    589         }
    590     }
    591     if missing_some_images:
    592       del expectations_dict['expected-results']['red.skp']['whole-image']
    593       del expectations_dict['expected-results']['red.skp']['tiled-images'][-1]
    594     path = os.path.join(self._expectations_dir, rel_path)
    595     with open(path, 'w') as fh:
    596       json.dump(expectations_dict, fh)
    597     return path
    598 
    599   def _generate_skps(self):
    600     """Runs the skpmaker binary to generate files in self._input_skp_dir."""
    601     self._run_skpmaker(
    602         output_path=os.path.join(self._input_skp_dir, 'red.skp'), red=255)
    603     self._run_skpmaker(
    604         output_path=os.path.join(self._input_skp_dir, 'green.skp'), green=255)
    605 
    606   def _run_skpmaker(self, output_path, red=0, green=0, blue=0,
    607                     width=640, height=400):
    608     """Runs the skpmaker binary to generate SKP with known characteristics.
    609 
    610     Args:
    611       output_path: Filepath to write the SKP into.
    612       red: Value of red color channel in image, 0-255.
    613       green: Value of green color channel in image, 0-255.
    614       blue: Value of blue color channel in image, 0-255.
    615       width: Width of canvas to create.
    616       height: Height of canvas to create.
    617     """
    618     binary = self.find_path_to_program('skpmaker')
    619     return self.run_command([binary,
    620                              '--red', str(red),
    621                              '--green', str(green),
    622                              '--blue', str(blue),
    623                              '--width', str(width),
    624                              '--height', str(height),
    625                              '--writePath', str(output_path),
    626                              ])
    627 
    628   def _assert_directory_contents(self, dir_path, expected_filenames):
    629     """Asserts that files found in a dir are identical to expected_filenames.
    630 
    631     Args:
    632       dir_path: Path to a directory on local disk.
    633       expected_filenames: Set containing the expected filenames within the dir.
    634 
    635     Raises:
    636       AssertionError: contents of the directory are not identical to
    637                       expected_filenames.
    638     """
    639     self.assertEqual(set(os.listdir(dir_path)), set(expected_filenames))
    640 
    641   def _assert_json_contents(self, json_path, expected_dict):
    642     """Asserts that contents of a JSON file are identical to expected_dict.
    643 
    644     Args:
    645       json_path: Path to a JSON file.
    646       expected_dict: Dictionary indicating the expected contents of the JSON
    647                      file.
    648 
    649     Raises:
    650       AssertionError: contents of the JSON file are not identical to
    651                       expected_dict.
    652     """
    653     prettyprinted_expected_dict = json.dumps(expected_dict, sort_keys=True,
    654                                              indent=2)
    655     with open(json_path, 'r') as fh:
    656       prettyprinted_json_dict = json.dumps(json.load(fh), sort_keys=True,
    657                                            indent=2)
    658     self.assertMultiLineEqual(prettyprinted_expected_dict,
    659                               prettyprinted_json_dict)
    660 
    661 
    662 def main():
    663   base_unittest.main(RenderPicturesTest)
    664 
    665 
    666 if __name__ == '__main__':
    667   main()
    668