1 #!/usr/bin/env python 2 # Copyright (c) 2011 The Chromium Authors. All rights reserved. 3 # Use of this source code is governed by a BSD-style license that can be 4 # found in the LICENSE file. 5 6 """Dromaeo benchmark automation script. 7 8 Script runs dromaeo tests in browsers specified by --browser switch and saves 9 results to a spreadsheet on docs.google.com. 10 11 Prerequisites: 12 1. Install Google Data APIs Python Client Library from 13 http://code.google.com/p/gdata-python-client. 14 2. Checkout Dromaeo benchmark from 15 http://src.chromium.org/svn/trunk/src/chrome/test/data/dromaeo and provide 16 local path to it in --dromaeo_home switch. 17 3. Create a spreadsheet at http://docs.google.com and specify its name in 18 --spreadsheet switch 19 20 Benchmark results are presented in the following format: 21 browser | date time 22 test 1 name|m11|...|m1n|test 1 average mean| |e11|...|e1n|test 1 average error 23 test 2 name|m21|...|m2n|test 2 average mean| |e21|...|e2n|test 2 average error 24 ... 25 26 Here mij is mean run/s in individual dromaeo test i during benchmark run j, 27 eij is error in individual dromaeo test i during benchmark run j. 28 29 Example usage: 30 dromaeo_benchmark_runner.py -b "E:\chromium\src\chrome\Release\chrome.exe" 31 -b "C:\Program Files (x86)\Safari\safari.exe" 32 -b "C:\Program Files (x86)\Opera 10.50 pre-alpha\opera.exe" -n 1 33 -d "E:\chromium\src\chrome\test\data\dromaeo" -f dom -e example (at] gmail.com 34 35 """ 36 37 import getpass 38 import json 39 import os 40 import re 41 import subprocess 42 import time 43 import urlparse 44 from optparse import OptionParser 45 from BaseHTTPServer import HTTPServer 46 import SimpleHTTPServer 47 import gdata.spreadsheet.service 48 49 max_spreadsheet_columns = 20 50 test_props = ['mean', 'error'] 51 52 53 def ParseArguments(): 54 parser = OptionParser() 55 parser.add_option("-b", "--browser", 56 action="append", dest="browsers", 57 help="list of browsers to test") 58 parser.add_option("-n", "--run_count", dest="run_count", type="int", 59 default=5, help="number of runs") 60 parser.add_option("-d", "--dromaeo_home", dest="dromaeo_home", 61 help="directory with your dromaeo files") 62 parser.add_option("-p", "--port", dest="port", type="int", 63 default=8080, help="http server port") 64 parser.add_option("-f", "--filter", dest="filter", 65 default="dom", help="dromaeo suite filter") 66 parser.add_option("-e", "--email", dest="email", 67 help="your google docs account") 68 parser.add_option("-s", "--spreadsheet", dest="spreadsheet_title", 69 default="dromaeo", 70 help="your google docs spreadsheet name") 71 72 options = parser.parse_args()[0] 73 74 if not options.dromaeo_home: 75 raise Exception('please specify dromaeo_home') 76 77 return options 78 79 80 def KillProcessByName(process_name): 81 process = subprocess.Popen('wmic process get processid, executablepath', 82 stdout=subprocess.PIPE) 83 stdout = str(process.communicate()[0]) 84 match = re.search(re.escape(process_name) + '\s+(\d+)', stdout) 85 if match: 86 pid = match.group(1) 87 subprocess.call('taskkill /pid %s' % pid) 88 89 90 class SpreadsheetWriter(object): 91 "Utility class for storing benchmarking results in Google spreadsheets." 92 93 def __init__(self, email, spreadsheet_title): 94 '''Login to google docs and search for spreadsheet''' 95 96 self.token_file = os.path.expanduser("~/.dromaeo_bot_auth_token") 97 self.gd_client = gdata.spreadsheet.service.SpreadsheetsService() 98 99 authenticated = False 100 if os.path.exists(self.token_file): 101 token = '' 102 try: 103 file = open(self.token_file, 'r') 104 token = file.read() 105 file.close() 106 self.gd_client.SetClientLoginToken(token) 107 self.gd_client.GetSpreadsheetsFeed() 108 authenticated = True 109 except (IOError, gdata.service.RequestError): 110 pass 111 if not authenticated: 112 self.gd_client.email = email 113 self.gd_client.password = getpass.getpass('Password for %s: ' % email) 114 self.gd_client.source = 'python robot for dromaeo' 115 self.gd_client.ProgrammaticLogin() 116 token = self.gd_client.GetClientLoginToken() 117 try: 118 file = open(self.token_file, 'w') 119 file.write(token) 120 file.close() 121 except (IOError): 122 pass 123 os.chmod(self.token_file, 0600) 124 125 # Search for the spreadsheet with title = spreadsheet_title. 126 spreadsheet_feed = self.gd_client.GetSpreadsheetsFeed() 127 for spreadsheet in spreadsheet_feed.entry: 128 if spreadsheet.title.text == spreadsheet_title: 129 self.spreadsheet_key = spreadsheet.id.text.rsplit('/', 1)[1] 130 if not self.spreadsheet_key: 131 raise Exception('Spreadsheet %s not found' % spreadsheet_title) 132 133 # Get the key of the first worksheet in spreadsheet. 134 worksheet_feed = self.gd_client.GetWorksheetsFeed(self.spreadsheet_key) 135 self.worksheet_key = worksheet_feed.entry[0].id.text.rsplit('/', 1)[1] 136 137 def _InsertRow(self, row): 138 row = dict([('c' + str(i), row[i]) for i in xrange(len(row))]) 139 self.gd_client.InsertRow(row, self.spreadsheet_key, self.worksheet_key) 140 141 def _InsertBlankRow(self): 142 self._InsertRow('-' * self.columns_count) 143 144 def PrepareSpreadsheet(self, run_count): 145 """Update cells in worksheet topmost row with service information. 146 147 Calculate column count corresponding to run_count and create worksheet 148 column titles [c0, c1, ...] in the topmost row to speed up spreadsheet 149 updates (it allows to insert a whole row with a single request) 150 """ 151 152 # Calculate the number of columns we need to present all test results. 153 self.columns_count = (run_count + 2) * len(test_props) 154 if self.columns_count > max_spreadsheet_columns: 155 # Google spreadsheet has just max_spreadsheet_columns columns. 156 max_run_count = max_spreadsheet_columns / len(test_props) - 2 157 raise Exception('maximum run count is %i' % max_run_count) 158 # Create worksheet column titles [c0, c1, ..., cn]. 159 for i in xrange(self.columns_count): 160 self.gd_client.UpdateCell(1, i + 1, 'c' + str(i), self.spreadsheet_key, 161 self.worksheet_key) 162 163 def WriteColumnTitles(self, run_count): 164 "Create titles for test results (mean 1, mean 2, ..., average mean, ...)" 165 row = [] 166 for prop in test_props: 167 row.append('') 168 for i in xrange(run_count): 169 row.append('%s %i' % (prop, i + 1)) 170 row.append('average ' + prop) 171 self._InsertRow(row) 172 173 def WriteBrowserBenchmarkTitle(self, browser_name): 174 "Create browser benchmark title (browser name, date time)" 175 self._InsertBlankRow() 176 self._InsertRow([browser_name, time.strftime('%d.%m.%Y %H:%M:%S')]) 177 178 def WriteBrowserBenchmarkResults(self, test_name, test_data): 179 "Insert a row with single test results" 180 row = [] 181 for prop in test_props: 182 if not row: 183 row.append(test_name) 184 else: 185 row.append('') 186 row.extend([str(x) for x in test_data[prop]]) 187 row.append(str(sum(test_data[prop]) / len(test_data[prop]))) 188 self._InsertRow(row) 189 190 191 class DromaeoHandler(SimpleHTTPServer.SimpleHTTPRequestHandler): 192 193 def do_POST(self): 194 self.send_response(200) 195 self.end_headers() 196 self.wfile.write("<HTML>POST OK.<BR><BR>"); 197 length = int(self.headers.getheader('content-length')) 198 parameters = urlparse.parse_qs(self.rfile.read(length)) 199 self.server.got_post = True 200 self.server.post_data = parameters['data'] 201 202 203 class BenchmarkResults(object): 204 "Storage class for dromaeo benchmark results" 205 206 def __init__(self): 207 self.data = {} 208 209 def ProcessBrowserPostData(self, data): 210 "Convert dromaeo test results in internal format" 211 tests = json.loads(data[0]) 212 for test in tests: 213 test_name = test['name'] 214 if test_name not in self.data: 215 # Test is encountered for the first time. 216 self.data[test_name] = dict([(prop, []) for prop in test_props]) 217 # Append current run results. 218 for prop in test_props: 219 value = -1 220 if prop in test: value = test[prop] # workaround for Opera 10.5 221 self.data[test_name][prop].append(value) 222 223 224 def main(): 225 options = ParseArguments() 226 227 # Start sever with dromaeo. 228 os.chdir(options.dromaeo_home) 229 server = HTTPServer(('', options.port), DromaeoHandler) 230 231 # Open and prepare spreadsheet on google docs. 232 spreadsheet_writer = SpreadsheetWriter(options.email, 233 options.spreadsheet_title) 234 spreadsheet_writer.PrepareSpreadsheet(options.run_count) 235 spreadsheet_writer.WriteColumnTitles(options.run_count) 236 237 for browser in options.browsers: 238 browser_name = os.path.splitext(os.path.basename(browser))[0] 239 spreadsheet_writer.WriteBrowserBenchmarkTitle(browser_name) 240 benchmark_results = BenchmarkResults() 241 for run_number in xrange(options.run_count): 242 print '%s run %i' % (browser_name, run_number + 1) 243 # Run browser. 244 test_page = 'http://localhost:%i/index.html?%s&automated&post_json' % ( 245 options.port, options.filter) 246 browser_process = subprocess.Popen('%s "%s"' % (browser, test_page)) 247 server.got_post = False 248 server.post_data = None 249 # Wait until POST request from browser. 250 while not server.got_post: 251 server.handle_request() 252 benchmark_results.ProcessBrowserPostData(server.post_data) 253 # Kill browser. 254 KillProcessByName(browser) 255 browser_process.wait() 256 257 # Insert test results into spreadsheet. 258 for (test_name, test_data) in benchmark_results.data.iteritems(): 259 spreadsheet_writer.WriteBrowserBenchmarkResults(test_name, test_data) 260 261 server.socket.close() 262 return 0 263 264 265 if __name__ == '__main__': 266 sys.exit(main()) 267
