1 # Copyright (c) 2012 The Chromium Authors. All rights reserved. 2 # Use of this source code is governed by a BSD-style license that can be 3 # found in the LICENSE file. 4 from datetime import datetime 5 import glob 6 import optparse 7 import os 8 import re 9 10 import cloud_storage_test_base 11 import page_sets 12 import pixel_expectations 13 14 from telemetry import benchmark 15 from telemetry.core import bitmap 16 from telemetry.page import page_test 17 from telemetry.util import cloud_storage 18 19 20 test_data_dir = os.path.abspath(os.path.join( 21 os.path.dirname(__file__), '..', '..', 'data', 'gpu')) 22 23 default_reference_image_dir = os.path.join(test_data_dir, 'gpu_reference') 24 25 test_harness_script = r""" 26 var domAutomationController = {}; 27 28 domAutomationController._succeeded = false; 29 domAutomationController._finished = false; 30 31 domAutomationController.setAutomationId = function(id) {} 32 33 domAutomationController.send = function(msg) { 34 domAutomationController._finished = true; 35 36 if(msg.toLowerCase() == "success") { 37 domAutomationController._succeeded = true; 38 } else { 39 domAutomationController._succeeded = false; 40 } 41 } 42 43 window.domAutomationController = domAutomationController; 44 """ 45 46 class PixelTestFailure(Exception): 47 pass 48 49 def _DidTestSucceed(tab): 50 return tab.EvaluateJavaScript('domAutomationController._succeeded') 51 52 class _PixelValidator(cloud_storage_test_base.ValidatorBase): 53 def CustomizeBrowserOptions(self, options): 54 options.AppendExtraBrowserArgs('--enable-gpu-benchmarking') 55 56 def ValidateAndMeasurePage(self, page, tab, results): 57 if not _DidTestSucceed(tab): 58 raise page_test.Failure('Page indicated a failure') 59 60 if not tab.screenshot_supported: 61 raise page_test.Failure('Browser does not support screenshot capture') 62 63 screenshot = tab.Screenshot(5) 64 65 if not screenshot: 66 raise page_test.Failure('Could not capture screenshot') 67 68 if hasattr(page, 'test_rect'): 69 screenshot = screenshot.Crop( 70 page.test_rect[0], page.test_rect[1], 71 page.test_rect[2], page.test_rect[3]) 72 73 image_name = self._UrlToImageName(page.display_name) 74 75 if self.options.upload_refimg_to_cloud_storage: 76 if self._ConditionallyUploadToCloudStorage(image_name, page, tab, 77 screenshot): 78 # This is the new reference image; there's nothing to compare against. 79 ref_png = screenshot 80 else: 81 # There was a preexisting reference image, so we might as well 82 # compare against it. 83 ref_png = self._DownloadFromCloudStorage(image_name, page, tab) 84 elif self.options.download_refimg_from_cloud_storage: 85 # This bot doesn't have the ability to properly generate a 86 # reference image, so download it from cloud storage. 87 try: 88 ref_png = self._DownloadFromCloudStorage(image_name, page, tab) 89 except cloud_storage.NotFoundError as e: 90 # There is no reference image yet in cloud storage. This 91 # happens when the revision of the test is incremented or when 92 # a new test is added, because the trybots are not allowed to 93 # produce reference images, only the bots on the main 94 # waterfalls. Report this as a failure so the developer has to 95 # take action by explicitly suppressing the failure and 96 # removing the suppression once the reference images have been 97 # generated. Otherwise silent failures could happen for long 98 # periods of time. 99 raise page_test.Failure('Could not find image %s in cloud storage' % 100 image_name) 101 else: 102 # Legacy path using on-disk results. 103 ref_png = self._GetReferenceImage(self.options.reference_dir, 104 image_name, page.revision, screenshot) 105 106 # Test new snapshot against existing reference image 107 if not ref_png.IsEqual(screenshot, tolerance=2): 108 if self.options.test_machine_name: 109 self._UploadErrorImagesToCloudStorage(image_name, screenshot, ref_png) 110 else: 111 self._WriteErrorImages(self.options.generated_dir, image_name, 112 screenshot, ref_png) 113 raise page_test.Failure('Reference image did not match captured screen') 114 115 def _DeleteOldReferenceImages(self, ref_image_path, cur_revision): 116 if not cur_revision: 117 return 118 119 old_revisions = glob.glob(ref_image_path + "_*.png") 120 for rev_path in old_revisions: 121 m = re.match(r'^.*_(\d+)\.png$', rev_path) 122 if m and int(m.group(1)) < cur_revision: 123 print 'Found deprecated reference image. Deleting rev ' + m.group(1) 124 os.remove(rev_path) 125 126 def _GetReferenceImage(self, img_dir, img_name, cur_revision, screenshot): 127 if not cur_revision: 128 cur_revision = 0 129 130 image_path = os.path.join(img_dir, img_name) 131 132 self._DeleteOldReferenceImages(image_path, cur_revision) 133 134 image_path = image_path + '_' + str(cur_revision) + '.png' 135 136 try: 137 ref_png = bitmap.Bitmap.FromPngFile(image_path) 138 except IOError: 139 ref_png = None 140 141 if ref_png: 142 return ref_png 143 144 print 'Reference image not found. Writing tab contents as reference.' 145 146 self._WriteImage(image_path, screenshot) 147 return screenshot 148 149 class Pixel(cloud_storage_test_base.TestBase): 150 test = _PixelValidator 151 152 @classmethod 153 def AddTestCommandLineArgs(cls, group): 154 super(Pixel, cls).AddTestCommandLineArgs(group) 155 group.add_option('--reference-dir', 156 help='Overrides the default on-disk location for reference images ' 157 '(only used for local testing without a cloud storage account)', 158 default=default_reference_image_dir) 159 160 def CreatePageSet(self, options): 161 page_set = page_sets.PixelTestsPageSet() 162 for page in page_set.pages: 163 page.script_to_evaluate_on_commit = test_harness_script 164 return page_set 165 166 def CreateExpectations(self, page_set): 167 return pixel_expectations.PixelExpectations() 168
