1 # Copyright (c) 2012 The Chromium Authors. All rights reserved. 2 # Use of this source code is governed by a BSD-style license that can be 3 # found in the LICENSE file. 4 from datetime import datetime 5 import glob 6 import optparse 7 import os 8 import re 9 10 import cloud_storage_test_base 11 import page_sets 12 import pixel_expectations 13 14 from telemetry import test 15 from telemetry.core import bitmap 16 from telemetry.page import cloud_storage 17 from telemetry.page import page_test 18 19 test_data_dir = os.path.abspath(os.path.join( 20 os.path.dirname(__file__), '..', '..', 'data', 'gpu')) 21 22 default_reference_image_dir = os.path.join(test_data_dir, 'gpu_reference') 23 24 test_harness_script = r""" 25 var domAutomationController = {}; 26 27 domAutomationController._succeeded = false; 28 domAutomationController._finished = false; 29 30 domAutomationController.setAutomationId = function(id) {} 31 32 domAutomationController.send = function(msg) { 33 domAutomationController._finished = true; 34 35 if(msg.toLowerCase() == "success") { 36 domAutomationController._succeeded = true; 37 } else { 38 domAutomationController._succeeded = false; 39 } 40 } 41 42 window.domAutomationController = domAutomationController; 43 """ 44 45 class PixelTestFailure(Exception): 46 pass 47 48 def _DidTestSucceed(tab): 49 return tab.EvaluateJavaScript('domAutomationController._succeeded') 50 51 class _PixelValidator(cloud_storage_test_base.ValidatorBase): 52 def CustomizeBrowserOptions(self, options): 53 options.AppendExtraBrowserArgs('--enable-gpu-benchmarking') 54 55 def ValidatePage(self, page, tab, results): 56 if not _DidTestSucceed(tab): 57 raise page_test.Failure('Page indicated a failure') 58 59 if not tab.screenshot_supported: 60 raise page_test.Failure('Browser does not support screenshot capture') 61 62 screenshot = tab.Screenshot(5) 63 64 if not screenshot: 65 raise page_test.Failure('Could not capture screenshot') 66 67 if hasattr(page, 'test_rect'): 68 screenshot = screenshot.Crop( 69 page.test_rect[0], page.test_rect[1], 70 page.test_rect[2], page.test_rect[3]) 71 72 image_name = self._UrlToImageName(page.display_name) 73 74 if self.options.upload_refimg_to_cloud_storage: 75 if self._ConditionallyUploadToCloudStorage(image_name, page, tab, 76 screenshot): 77 # This is the new reference image; there's nothing to compare against. 78 ref_png = screenshot 79 else: 80 # There was a preexisting reference image, so we might as well 81 # compare against it. 82 ref_png = self._DownloadFromCloudStorage(image_name, page, tab) 83 elif self.options.download_refimg_from_cloud_storage: 84 # This bot doesn't have the ability to properly generate a 85 # reference image, so download it from cloud storage. 86 try: 87 ref_png = self._DownloadFromCloudStorage(image_name, page, tab) 88 except cloud_storage.NotFoundError as e: 89 # There is no reference image yet in cloud storage. This 90 # happens when the revision of the test is incremented or when 91 # a new test is added, because the trybots are not allowed to 92 # produce reference images, only the bots on the main 93 # waterfalls. Report this as a failure so the developer has to 94 # take action by explicitly suppressing the failure and 95 # removing the suppression once the reference images have been 96 # generated. Otherwise silent failures could happen for long 97 # periods of time. 98 raise page_test.Failure('Could not find image %s in cloud storage' % 99 image_name) 100 else: 101 # Legacy path using on-disk results. 102 ref_png = self._GetReferenceImage(self.options.reference_dir, 103 image_name, page.revision, screenshot) 104 105 # Test new snapshot against existing reference image 106 if not ref_png.IsEqual(screenshot, tolerance=2): 107 if self.options.test_machine_name: 108 self._UploadErrorImagesToCloudStorage(image_name, screenshot, ref_png) 109 else: 110 self._WriteErrorImages(self.options.generated_dir, image_name, 111 screenshot, ref_png) 112 raise page_test.Failure('Reference image did not match captured screen') 113 114 def _DeleteOldReferenceImages(self, ref_image_path, cur_revision): 115 if not cur_revision: 116 return 117 118 old_revisions = glob.glob(ref_image_path + "_*.png") 119 for rev_path in old_revisions: 120 m = re.match(r'^.*_(\d+)\.png$', rev_path) 121 if m and int(m.group(1)) < cur_revision: 122 print 'Found deprecated reference image. Deleting rev ' + m.group(1) 123 os.remove(rev_path) 124 125 def _GetReferenceImage(self, img_dir, img_name, cur_revision, screenshot): 126 if not cur_revision: 127 cur_revision = 0 128 129 image_path = os.path.join(img_dir, img_name) 130 131 self._DeleteOldReferenceImages(image_path, cur_revision) 132 133 image_path = image_path + '_' + str(cur_revision) + '.png' 134 135 try: 136 ref_png = bitmap.Bitmap.FromPngFile(image_path) 137 except IOError: 138 ref_png = None 139 140 if ref_png: 141 return ref_png 142 143 print 'Reference image not found. Writing tab contents as reference.' 144 145 self._WriteImage(image_path, screenshot) 146 return screenshot 147 148 class Pixel(cloud_storage_test_base.TestBase): 149 test = _PixelValidator 150 page_set = page_sets.PixelTestsPageSet 151 152 @classmethod 153 def AddTestCommandLineArgs(cls, group): 154 super(Pixel, cls).AddTestCommandLineArgs(group) 155 group.add_option('--reference-dir', 156 help='Overrides the default on-disk location for reference images ' 157 '(only used for local testing without a cloud storage account)', 158 default=default_reference_image_dir) 159 160 def CreatePageSet(self, options): 161 page_set = super(Pixel, self).CreatePageSet(options) 162 for page in page_set.pages: 163 page.script_to_evaluate_on_commit = test_harness_script 164 return page_set 165 166 def CreateExpectations(self, page_set): 167 return pixel_expectations.PixelExpectations() 168
