1 # Copyright 2013 The Chromium Authors. All rights reserved. 2 # Use of this source code is governed by a BSD-style license that can be 3 # found in the LICENSE file. 4 5 import logging 6 import optparse 7 import os 8 import shutil 9 import sys 10 import zipfile 11 12 from telemetry import decorators 13 from telemetry.core import browser_finder 14 from telemetry.core import command_line 15 from telemetry.core import util 16 from telemetry.page import page_runner 17 from telemetry.page import cloud_storage 18 from telemetry.page import page_set 19 from telemetry.page import page_test 20 from telemetry.page import test_expectations 21 from telemetry.results import page_test_results 22 23 24 Disabled = decorators.Disabled 25 Enabled = decorators.Enabled 26 27 28 class Test(command_line.Command): 29 """Base class for a Telemetry test or benchmark. 30 31 A test packages a PageTest/PageMeasurement and a PageSet together. 32 """ 33 options = {} 34 35 @classmethod 36 def Name(cls): 37 name = cls.__module__.split('.')[-1] 38 if hasattr(cls, 'tag'): 39 name += '.' + cls.tag 40 if hasattr(cls, 'page_set'): 41 name += '.' + cls.page_set.Name() 42 return name 43 44 @classmethod 45 def AddCommandLineArgs(cls, parser): 46 cls.PageTestClass().AddCommandLineArgs(parser) 47 48 if hasattr(cls, 'AddTestCommandLineArgs'): 49 group = optparse.OptionGroup(parser, '%s test options' % cls.Name()) 50 cls.AddTestCommandLineArgs(group) 51 parser.add_option_group(group) 52 53 @classmethod 54 def SetArgumentDefaults(cls, parser): 55 cls.PageTestClass().SetArgumentDefaults(parser) 56 parser.set_defaults(**cls.options) 57 58 @classmethod 59 def ProcessCommandLineArgs(cls, parser, args): 60 cls.PageTestClass().ProcessCommandLineArgs(parser, args) 61 62 def CustomizeBrowserOptions(self, options): 63 """Add browser options that are required by this benchmark.""" 64 65 def Run(self, args): 66 """Run this test with the given options.""" 67 self.CustomizeBrowserOptions(args.browser_options) 68 69 test = self.PageTestClass()() 70 test.__name__ = self.__class__.__name__ 71 72 if hasattr(self, '_disabled_strings'): 73 test._disabled_strings = self._disabled_strings 74 if hasattr(self, '_enabled_strings'): 75 test._enabled_strings = self._enabled_strings 76 77 ps = self.CreatePageSet(args) 78 expectations = self.CreateExpectations(ps) 79 80 self._DownloadGeneratedProfileArchive(args) 81 82 results = page_test_results.PageTestResults() 83 try: 84 results = page_runner.Run(test, ps, expectations, args) 85 except page_test.TestNotSupportedOnPlatformFailure as failure: 86 logging.warning(str(failure)) 87 88 results.PrintSummary() 89 return len(results.failures) + len(results.errors) 90 91 def _DownloadGeneratedProfileArchive(self, options): 92 """Download and extract profile directory archive if one exists.""" 93 archive_name = getattr(self, 'generated_profile_archive', None) 94 95 # If attribute not specified, nothing to do. 96 if not archive_name: 97 return 98 99 # If profile dir specified on command line, nothing to do. 100 if options.browser_options.profile_dir: 101 logging.warning("Profile directory specified on command line: %s, this" 102 "overrides the benchmark's default profile directory.", 103 options.browser_options.profile_dir) 104 return 105 106 # Download profile directory from cloud storage. 107 found_browser = browser_finder.FindBrowser(options) 108 test_data_dir = os.path.join(util.GetChromiumSrcDir(), 'tools', 'perf', 109 'generated_profiles', 110 found_browser.target_os) 111 generated_profile_archive_path = os.path.normpath( 112 os.path.join(test_data_dir, archive_name)) 113 114 try: 115 cloud_storage.GetIfChanged(generated_profile_archive_path, 116 cloud_storage.PUBLIC_BUCKET) 117 except (cloud_storage.CredentialsError, 118 cloud_storage.PermissionError) as e: 119 if os.path.exists(generated_profile_archive_path): 120 # If the profile directory archive exists, assume the user has their 121 # own local copy simply warn. 122 logging.warning('Could not download Profile archive: %s', 123 generated_profile_archive_path) 124 else: 125 # If the archive profile directory doesn't exist, this is fatal. 126 logging.error('Can not run without required profile archive: %s. ' 127 'If you believe you have credentials, follow the ' 128 'instructions below.', 129 generated_profile_archive_path) 130 logging.error(e) 131 sys.exit(-1) 132 133 # Unzip profile directory. 134 extracted_profile_dir_path = ( 135 os.path.splitext(generated_profile_archive_path)[0]) 136 if not os.path.isfile(generated_profile_archive_path): 137 raise Exception("Profile directory archive not downloaded: ", 138 generated_profile_archive_path) 139 with zipfile.ZipFile(generated_profile_archive_path) as f: 140 try: 141 f.extractall(os.path.dirname(generated_profile_archive_path)) 142 except e: 143 # Cleanup any leftovers from unzipping. 144 if os.path.exists(extracted_profile_dir_path): 145 shutil.rmtree(extracted_profile_dir_path) 146 logging.error("Error extracting profile directory zip file: %s", e) 147 sys.exit(-1) 148 149 # Run with freshly extracted profile directory. 150 logging.info("Using profile archive directory: %s", 151 extracted_profile_dir_path) 152 options.browser_options.profile_dir = extracted_profile_dir_path 153 154 @classmethod 155 def PageTestClass(cls): 156 """Get the PageTest for this Test. 157 158 If the Test has no PageTest, raises NotImplementedError. 159 """ 160 if not hasattr(cls, 'test'): 161 raise NotImplementedError('This test has no "test" attribute.') 162 if not issubclass(cls.test, page_test.PageTest): 163 raise TypeError('"%s" is not a PageTest.' % cls.test.__name__) 164 return cls.test 165 166 @classmethod 167 def PageSetClass(cls): 168 """Get the PageSet for this Test. 169 170 If the Test has no PageSet, raises NotImplementedError. 171 """ 172 if not hasattr(cls, 'page_set'): 173 raise NotImplementedError('This test has no "page_set" attribute.') 174 if not issubclass(cls.page_set, page_set.PageSet): 175 raise TypeError('"%s" is not a PageSet.' % cls.page_set.__name__) 176 return cls.page_set 177 178 @classmethod 179 def CreatePageSet(cls, options): # pylint: disable=W0613 180 """Get the page set this test will run on. 181 182 By default, it will create a page set from the file at this test's 183 page_set attribute. Override to generate a custom page set. 184 """ 185 return cls.PageSetClass()() 186 187 @classmethod 188 def CreateExpectations(cls, ps): # pylint: disable=W0613 189 """Get the expectations this test will run with. 190 191 By default, it will create an empty expectations set. Override to generate 192 custom expectations. 193 """ 194 if hasattr(cls, 'expectations'): 195 return cls.expectations 196 else: 197 return test_expectations.TestExpectations() 198 199 200 def AddCommandLineArgs(parser): 201 page_runner.AddCommandLineArgs(parser) 202 203 204 def ProcessCommandLineArgs(parser, args): 205 page_runner.ProcessCommandLineArgs(parser, args) 206
