1 # Copyright 2013 The Chromium Authors. All rights reserved. 2 # Use of this source code is governed by a BSD-style license that can be 3 # found in the LICENSE file. 4 5 """Base classes for a test and validator which upload results 6 (reference images, error images) to cloud storage.""" 7 8 import os 9 import re 10 import tempfile 11 12 from telemetry import benchmark 13 from telemetry.core import bitmap 14 from telemetry.page import page_test 15 from telemetry.util import cloud_storage 16 17 18 test_data_dir = os.path.abspath(os.path.join( 19 os.path.dirname(__file__), '..', '..', 'data', 'gpu')) 20 21 default_generated_data_dir = os.path.join(test_data_dir, 'generated') 22 23 error_image_cloud_storage_bucket = 'chromium-browser-gpu-tests' 24 25 def _CompareScreenshotSamples(screenshot, expectations, device_pixel_ratio): 26 for expectation in expectations: 27 location = expectation["location"] 28 x = int(location[0] * device_pixel_ratio) 29 y = int(location[1] * device_pixel_ratio) 30 31 if x < 0 or y < 0 or x > screenshot.width or y > screenshot.height: 32 raise page_test.Failure( 33 'Expected pixel location [%d, %d] is out of range on [%d, %d] image' % 34 (x, y, screenshot.width, screenshot.height)) 35 36 actual_color = screenshot.GetPixelColor(x, y) 37 expected_color = bitmap.RgbaColor( 38 expectation["color"][0], 39 expectation["color"][1], 40 expectation["color"][2]) 41 if not actual_color.IsEqual(expected_color, expectation["tolerance"]): 42 raise page_test.Failure('Expected pixel at ' + str(location) + 43 ' to be ' + 44 str(expectation["color"]) + " but got [" + 45 str(actual_color.r) + ", " + 46 str(actual_color.g) + ", " + 47 str(actual_color.b) + "]") 48 49 class ValidatorBase(page_test.PageTest): 50 def __init__(self): 51 super(ValidatorBase, self).__init__() 52 # Parameters for cloud storage reference images. 53 self.vendor_id = None 54 self.device_id = None 55 self.vendor_string = None 56 self.device_string = None 57 self.msaa = False 58 59 ### 60 ### Routines working with the local disk (only used for local 61 ### testing without a cloud storage account -- the bots do not use 62 ### this code path). 63 ### 64 65 def _UrlToImageName(self, url): 66 image_name = re.sub(r'^(http|https|file)://(/*)', '', url) 67 image_name = re.sub(r'\.\./', '', image_name) 68 image_name = re.sub(r'(\.|/|-)', '_', image_name) 69 return image_name 70 71 def _WriteImage(self, image_path, png_image): 72 output_dir = os.path.dirname(image_path) 73 if not os.path.exists(output_dir): 74 os.makedirs(output_dir) 75 png_image.WritePngFile(image_path) 76 77 def _WriteErrorImages(self, img_dir, img_name, screenshot, ref_png): 78 full_image_name = img_name + '_' + str(self.options.build_revision) 79 full_image_name = full_image_name + '.png' 80 81 # Always write the failing image. 82 self._WriteImage( 83 os.path.join(img_dir, 'FAIL_' + full_image_name), screenshot) 84 85 if ref_png: 86 # Save the reference image. 87 # This ensures that we get the right revision number. 88 self._WriteImage( 89 os.path.join(img_dir, full_image_name), ref_png) 90 91 # Save the difference image. 92 diff_png = screenshot.Diff(ref_png) 93 self._WriteImage( 94 os.path.join(img_dir, 'DIFF_' + full_image_name), diff_png) 95 96 ### 97 ### Cloud storage code path -- the bots use this. 98 ### 99 100 def _ComputeGpuInfo(self, tab): 101 if ((self.vendor_id and self.device_id) or 102 (self.vendor_string and self.device_string)): 103 return 104 browser = tab.browser 105 if not browser.supports_system_info: 106 raise Exception('System info must be supported by the browser') 107 system_info = browser.GetSystemInfo() 108 if not system_info.gpu: 109 raise Exception('GPU information was absent') 110 device = system_info.gpu.devices[0] 111 if device.vendor_id and device.device_id: 112 self.vendor_id = device.vendor_id 113 self.device_id = device.device_id 114 elif device.vendor_string and device.device_string: 115 self.vendor_string = device.vendor_string 116 self.device_string = device.device_string 117 else: 118 raise Exception('GPU device information was incomplete') 119 self.msaa = not ( 120 'disable_multisampling' in system_info.gpu.driver_bug_workarounds) 121 122 def _FormatGpuInfo(self, tab): 123 self._ComputeGpuInfo(tab) 124 msaa_string = '_msaa' if self.msaa else '_non_msaa' 125 if self.vendor_id: 126 return '%s_%04x_%04x%s' % ( 127 self.options.os_type, self.vendor_id, self.device_id, msaa_string) 128 else: 129 return '%s_%s_%s%s' % ( 130 self.options.os_type, self.vendor_string, self.device_string, 131 msaa_string) 132 133 def _FormatReferenceImageName(self, img_name, page, tab): 134 return '%s_v%s_%s.png' % ( 135 img_name, 136 page.revision, 137 self._FormatGpuInfo(tab)) 138 139 def _UploadBitmapToCloudStorage(self, bucket, name, bitmap, public=False): 140 # This sequence of steps works on all platforms to write a temporary 141 # PNG to disk, following the pattern in bitmap_unittest.py. The key to 142 # avoiding PermissionErrors seems to be to not actually try to write to 143 # the temporary file object, but to re-open its name for all operations. 144 temp_file = tempfile.NamedTemporaryFile().name 145 bitmap.WritePngFile(temp_file) 146 cloud_storage.Insert(bucket, name, temp_file, publicly_readable=public) 147 148 def _ConditionallyUploadToCloudStorage(self, img_name, page, tab, screenshot): 149 """Uploads the screenshot to cloud storage as the reference image 150 for this test, unless it already exists. Returns True if the 151 upload was actually performed.""" 152 if not self.options.refimg_cloud_storage_bucket: 153 raise Exception('--refimg-cloud-storage-bucket argument is required') 154 cloud_name = self._FormatReferenceImageName(img_name, page, tab) 155 if not cloud_storage.Exists(self.options.refimg_cloud_storage_bucket, 156 cloud_name): 157 self._UploadBitmapToCloudStorage(self.options.refimg_cloud_storage_bucket, 158 cloud_name, 159 screenshot) 160 return True 161 return False 162 163 def _DownloadFromCloudStorage(self, img_name, page, tab): 164 """Downloads the reference image for the given test from cloud 165 storage, returning it as a Telemetry Bitmap object.""" 166 # TODO(kbr): there's a race condition between the deletion of the 167 # temporary file and gsutil's overwriting it. 168 if not self.options.refimg_cloud_storage_bucket: 169 raise Exception('--refimg-cloud-storage-bucket argument is required') 170 temp_file = tempfile.NamedTemporaryFile().name 171 cloud_storage.Get(self.options.refimg_cloud_storage_bucket, 172 self._FormatReferenceImageName(img_name, page, tab), 173 temp_file) 174 return bitmap.Bitmap.FromPngFile(temp_file) 175 176 def _UploadErrorImagesToCloudStorage(self, image_name, screenshot, ref_img): 177 """For a failing run, uploads the failing image, reference image (if 178 supplied), and diff image (if reference image was supplied) to cloud 179 storage. This subsumes the functionality of the 180 archive_gpu_pixel_test_results.py script.""" 181 machine_name = re.sub('\W+', '_', self.options.test_machine_name) 182 upload_dir = '%s_%s_telemetry' % (self.options.build_revision, machine_name) 183 base_bucket = '%s/runs/%s' % (error_image_cloud_storage_bucket, upload_dir) 184 image_name_with_revision = '%s_%s.png' % ( 185 image_name, self.options.build_revision) 186 self._UploadBitmapToCloudStorage( 187 base_bucket + '/gen', image_name_with_revision, screenshot, 188 public=True) 189 if ref_img: 190 self._UploadBitmapToCloudStorage( 191 base_bucket + '/ref', image_name_with_revision, ref_img, public=True) 192 diff_img = screenshot.Diff(ref_img) 193 self._UploadBitmapToCloudStorage( 194 base_bucket + '/diff', image_name_with_revision, diff_img, 195 public=True) 196 print ('See http://%s.commondatastorage.googleapis.com/' 197 'view_test_results.html?%s for this run\'s test results') % ( 198 error_image_cloud_storage_bucket, upload_dir) 199 200 def _ValidateScreenshotSamples(self, url, 201 screenshot, expectations, device_pixel_ratio): 202 """Samples the given screenshot and verifies pixel color values. 203 The sample locations and expected color values are given in expectations. 204 In case any of the samples do not match the expected color, it raises 205 a Failure and dumps the screenshot locally or cloud storage depending on 206 what machine the test is being run.""" 207 try: 208 _CompareScreenshotSamples(screenshot, expectations, device_pixel_ratio) 209 except page_test.Failure: 210 image_name = self._UrlToImageName(url) 211 if self.options.test_machine_name: 212 self._UploadErrorImagesToCloudStorage(image_name, screenshot, None) 213 else: 214 self._WriteErrorImages(self.options.generated_dir, image_name, 215 screenshot, None) 216 raise 217 218 219 class TestBase(benchmark.Benchmark): 220 @classmethod 221 def AddTestCommandLineArgs(cls, group): 222 group.add_option('--build-revision', 223 help='Chrome revision being tested.', 224 default="unknownrev") 225 group.add_option('--upload-refimg-to-cloud-storage', 226 dest='upload_refimg_to_cloud_storage', 227 action='store_true', default=False, 228 help='Upload resulting images to cloud storage as reference images') 229 group.add_option('--download-refimg-from-cloud-storage', 230 dest='download_refimg_from_cloud_storage', 231 action='store_true', default=False, 232 help='Download reference images from cloud storage') 233 group.add_option('--refimg-cloud-storage-bucket', 234 help='Name of the cloud storage bucket to use for reference images; ' 235 'required with --upload-refimg-to-cloud-storage and ' 236 '--download-refimg-from-cloud-storage. Example: ' 237 '"chromium-gpu-archive/reference-images"') 238 group.add_option('--os-type', 239 help='Type of operating system on which the pixel test is being run, ' 240 'used only to distinguish different operating systems with the same ' 241 'graphics card. Any value is acceptable, but canonical values are ' 242 '"win", "mac", and "linux", and probably, eventually, "chromeos" ' 243 'and "android").', 244 default='') 245 group.add_option('--test-machine-name', 246 help='Name of the test machine. Specifying this argument causes this ' 247 'script to upload failure images and diffs to cloud storage directly, ' 248 'instead of relying on the archive_gpu_pixel_test_results.py script.', 249 default='') 250 group.add_option('--generated-dir', 251 help='Overrides the default on-disk location for generated test images ' 252 '(only used for local testing without a cloud storage account)', 253 default=default_generated_data_dir) 254
