1 #!/usr/bin/env python 2 # Copyright 2015 The Chromium Authors. All rights reserved. 3 # Use of this source code is governed by a BSD-style license that can be 4 # found in the LICENSE file. 5 6 import argparse 7 import json 8 import logging 9 import os 10 import re 11 import shutil 12 import stat 13 import subprocess 14 import sys 15 import tempfile 16 import time 17 import urllib2 18 import zipfile 19 20 from hooks import install 21 22 from catapult_base import xvfb 23 24 # URL on omahaproxy.appspot.com which lists the current version for the os 25 # and channel. 26 VERSION_LOOKUP_URL = 'https://omahaproxy.appspot.com/all?os=%s&channel=%s' 27 28 # URL on omahaproxy.appspot.com which looks up base positions from versions. 29 BASE_POS_LOOKUP_URL = 'http://omahaproxy.appspot.com/revision.json?version=%s' 30 31 # URL on cloud storage which looks up the chromium download url from base pos. 32 CLOUD_STORAGE_LOOKUP_URL = ('https://www.googleapis.com/storage/v1/b/' 33 'chromium-browser-snapshots/o?delimiter=/&prefix=' 34 '%s/%s&fields=items(kind,mediaLink,metadata,name,' 35 'size,updated),kind,prefixes,nextPageToken') 36 37 # URL on cloud storage to download chromium at a base pos from. 38 CLOUD_STORAGE_DOWNLOAD_URL = ('https://www.googleapis.com/download/storage/v1/b' 39 '/chromium-browser-snapshots/o/%s%%2F%s%%2F' 40 'chrome-%s.zip?alt=media') 41 42 # URL in cloud storage to download Chrome zip from. 43 CLOUDSTORAGE_URL = ('https://commondatastorage.googleapis.com/chrome-unsigned' 44 '/desktop-W15K3Y/%s/%s/chrome-%s.zip') 45 46 # Default port to run on if not auto-assigning from OS 47 DEFAULT_PORT = '8111' 48 49 # Mapping of sys.platform -> platform-specific names and paths. 50 PLATFORM_MAPPING = { 51 'linux2': { 52 'omaha': 'linux', 53 'prefix': 'Linux_x64', 54 'zip_prefix': 'linux', 55 'chromepath': 'chrome-linux/chrome' 56 }, 57 'win32': { 58 'omaha': 'win', 59 'prefix': 'Win', 60 'zip_prefix': 'win32', 61 'chromepath': 'chrome-win32\\chrome.exe', 62 }, 63 'darwin': { 64 'omaha': 'mac', 65 'prefix': 'Mac', 66 'zip_prefix': 'mac', 67 'chromepath': ('chrome-mac/Chromium.app/Contents/MacOS/Chromium'), 68 'version_path': 'chrome-mac/Chromium.app/Contents/Versions/', 69 'additional_paths': [ 70 ('chrome-mac/Chromium.app/Contents/Versions/%VERSION%/' 71 'Chromium Helper.app/Contents/MacOS/Chromium Helper'), 72 ], 73 }, 74 } 75 76 77 def IsDepotToolsPath(path): 78 return os.path.isfile(os.path.join(path, 'gclient')) 79 80 81 def FindDepotTools(): 82 # Check if depot_tools is already in PYTHONPATH 83 for path in sys.path: 84 if path.rstrip(os.sep).endswith('depot_tools') and IsDepotToolsPath(path): 85 return path 86 87 # Check if depot_tools is in the path 88 for path in os.environ['PATH'].split(os.pathsep): 89 if IsDepotToolsPath(path): 90 return path.rstrip(os.sep) 91 92 return None 93 94 95 def DownloadChromium(channel): 96 """ 97 Gets the version of Chrome current for the given channel from omahaproxy, then 98 follows instructions for downloading a prebuilt version of chromium from the 99 commit at the branch cut for that version. This downloads a chromium binary 100 which does not have any commits merged onto the branch. It is close to the 101 released Chrome, but not exact. Downloading the released Chrome is not 102 supported. 103 https://www.chromium.org/getting-involved/download-chromium 104 """ 105 # Get the version for the current channel from omahaproxy 106 platform_data = PLATFORM_MAPPING[sys.platform] 107 omaha_platform = platform_data['omaha'] 108 version_lookup_url = VERSION_LOOKUP_URL % (omaha_platform, channel) 109 print 'Getting version from %s' % version_lookup_url 110 response = urllib2.urlopen(version_lookup_url, timeout=120) 111 version = response.readlines()[1].split(',')[2] 112 113 # Get the base position for that version from omahaproxy 114 base_pos_lookup_url = BASE_POS_LOOKUP_URL % version 115 print 'Getting base_pos from %s' % base_pos_lookup_url 116 response = urllib2.urlopen(base_pos_lookup_url, timeout=120) 117 base_pos = json.load(response)['chromium_base_position'] 118 119 # Find the build from that base position in cloud storage. If it's not found, 120 # decrement base position until one is found. 121 cloud_storage_lookup_url = CLOUD_STORAGE_LOOKUP_URL % ( 122 platform_data['prefix'], base_pos) 123 download_url = None 124 while not download_url: 125 print 'Getting download url from %s' % cloud_storage_lookup_url 126 response = urllib2.urlopen(cloud_storage_lookup_url, timeout=120) 127 prefixes = json.load(response).get('prefixes') 128 if prefixes: 129 download_url = CLOUD_STORAGE_DOWNLOAD_URL % ( 130 platform_data['prefix'], base_pos, platform_data['zip_prefix']) 131 break 132 base_pos = int(base_pos) - 1 133 cloud_storage_lookup_url = CLOUD_STORAGE_LOOKUP_URL % ( 134 platform_data['prefix'], base_pos) 135 136 print 'Approximating Chrome %s with chromium from base position %s.' % ( 137 version, base_pos) 138 print 'Downloading from %s' % download_url 139 140 tmpdir = tempfile.mkdtemp() 141 zip_path = os.path.join(tmpdir, 'chrome.zip') 142 with open(zip_path, 'wb') as local_file: 143 local_file.write(urllib2.urlopen(download_url, timeout=600).read()) 144 zf = zipfile.ZipFile(zip_path) 145 zf.extractall(path=tmpdir) 146 return tmpdir, version, download_url 147 148 149 def GetLocalChromePath(path_from_command_line): 150 if path_from_command_line: 151 return path_from_command_line 152 153 if sys.platform == 'darwin': # Mac 154 chrome_path = ( 155 '/Applications/Google Chrome.app/Contents/MacOS/Google Chrome') 156 if os.path.isfile(chrome_path): 157 return chrome_path 158 elif sys.platform.startswith('linux'): 159 found = False 160 try: 161 with open(os.devnull, 'w') as devnull: 162 found = subprocess.call(['google-chrome', '--version'], 163 stdout=devnull, stderr=devnull) == 0 164 except OSError: 165 pass 166 if found: 167 return 'google-chrome' 168 elif sys.platform == 'win32': 169 search_paths = [os.getenv('PROGRAMFILES(X86)'), 170 os.getenv('PROGRAMFILES'), 171 os.getenv('LOCALAPPDATA')] 172 chrome_path = os.path.join('Google', 'Chrome', 'Application', 'chrome.exe') 173 for search_path in search_paths: 174 test_path = os.path.join(search_path, chrome_path) 175 if os.path.isfile(test_path): 176 return test_path 177 return None 178 179 180 def Main(argv): 181 try: 182 parser = argparse.ArgumentParser( 183 description='Run dev_server tests for a project.') 184 parser.add_argument('--chrome_path', type=str, 185 help='Path to Chrome browser binary.') 186 parser.add_argument('--no-use-local-chrome', 187 dest='use_local_chrome', action='store_false') 188 parser.add_argument( 189 '--no-install-hooks', dest='install_hooks', action='store_false') 190 parser.add_argument('--tests', type=str, 191 help='Set of tests to run (tracing or perf_insights)') 192 parser.add_argument('--channel', type=str, default='stable', 193 help='Chrome channel to run (stable or canary)') 194 parser.add_argument('--presentation-json', type=str, 195 help='Recipe presentation-json output file path') 196 parser.set_defaults(install_hooks=True) 197 parser.set_defaults(use_local_chrome=True) 198 args = parser.parse_args(argv[1:]) 199 200 if args.install_hooks: 201 install.InstallHooks() 202 203 platform_data = PLATFORM_MAPPING[sys.platform] 204 user_data_dir = tempfile.mkdtemp() 205 tmpdir = None 206 xvfb_process = None 207 208 server_path = os.path.join(os.path.dirname( 209 os.path.abspath(__file__)), os.pardir, 'bin', 'run_dev_server') 210 # TODO(anniesullie): Make OS selection of port work on Windows. See #1235. 211 if sys.platform == 'win32': 212 port = DEFAULT_PORT 213 else: 214 port = '0' 215 server_command = [server_path, '--no-install-hooks', '--port', port] 216 if sys.platform.startswith('win'): 217 server_command = ['python.exe'] + server_command 218 print "Starting dev_server..." 219 server_process = subprocess.Popen( 220 server_command, stdout=subprocess.PIPE, stderr=subprocess.PIPE, 221 bufsize=1) 222 time.sleep(1) 223 if sys.platform != 'win32': 224 output = server_process.stderr.readline() 225 port = re.search( 226 r'Now running on http://127.0.0.1:([\d]+)', output).group(1) 227 228 chrome_info = None 229 if args.use_local_chrome: 230 chrome_path = GetLocalChromePath(args.chrome_path) 231 if not chrome_path: 232 logging.error('Could not find path to chrome.') 233 sys.exit(1) 234 chrome_info = 'with command `%s`' % chrome_path 235 else: 236 channel = args.channel 237 if sys.platform == 'linux2' and channel == 'canary': 238 channel = 'dev' 239 assert channel in ['stable', 'beta', 'dev', 'canary'] 240 241 242 tmpdir, version, download_url = DownloadChromium(channel) 243 if xvfb.ShouldStartXvfb(): 244 xvfb_process = xvfb.StartXvfb() 245 chrome_path = os.path.join( 246 tmpdir, platform_data['chromepath']) 247 os.chmod(chrome_path, os.stat(chrome_path).st_mode | stat.S_IEXEC) 248 # On Mac, we need to update a file with the version in the path, and 249 # the version we downloaded could be slightly different than what we 250 # requested. Update it. 251 if platform_data.get('version_path'): 252 contents = os.listdir( 253 os.path.join(tmpdir, platform_data['version_path'])) 254 for path in contents: 255 if re.match(r'\d+\.\d+\.\d+\.\d+', path): 256 version = path 257 if platform_data.get('additional_paths'): 258 for path in platform_data.get('additional_paths'): 259 path = path.replace('%VERSION%', version) 260 path = os.path.join(tmpdir, path) 261 if os.path.exists(path): 262 os.chmod(path, os.stat(path).st_mode | stat.S_IEXEC) 263 chrome_info = version 264 chrome_command = [ 265 chrome_path, 266 '--user-data-dir=%s' % user_data_dir, 267 '--no-sandbox', 268 '--no-experiments', 269 '--no-first-run', 270 '--noerrdialogs', 271 '--window-size=1280,1024', 272 ('http://localhost:%s/%s/tests.html?' % (port, args.tests)) + 273 'headless=true&testTypeToRun=all', 274 ] 275 print "Starting Chrome %s..." % chrome_info 276 chrome_process = subprocess.Popen( 277 chrome_command, stdout=subprocess.PIPE, stderr=subprocess.PIPE) 278 print "Waiting for tests to finish..." 279 server_out, server_err = server_process.communicate() 280 print "Killing Chrome..." 281 if sys.platform == 'win32': 282 # Use taskkill on Windows to make sure Chrome and all subprocesses are 283 # killed. 284 subprocess.call(['taskkill', '/F', '/T', '/PID', str(chrome_process.pid)]) 285 else: 286 chrome_process.kill() 287 if server_process.returncode != 0: 288 logging.error('Tests failed!') 289 logging.error('Server stderr:') 290 logging.error(server_err) 291 logging.error('Server stdout:') 292 logging.error(server_out) 293 else: 294 print server_out 295 if args.presentation_json: 296 with open(args.presentation_json, 'w') as recipe_out: 297 # Add a link to the buildbot status for the step saying which version 298 # of Chrome the test ran on. The actual linking feature is not used, 299 # but there isn't a way to just add text. 300 link_name = 'Chrome Version %s' % version 301 presentation_info = {'links': {link_name: download_url}} 302 json.dump(presentation_info, recipe_out) 303 finally: 304 # Wait for Chrome to be killed before deleting temp Chrome dir. Only have 305 # this timing issue on Windows. 306 if sys.platform == 'win32': 307 time.sleep(5) 308 if tmpdir: 309 try: 310 shutil.rmtree(tmpdir) 311 shutil.rmtree(user_data_dir) 312 except OSError as e: 313 logging.error('Error cleaning up temp dirs %s and %s: %s', 314 tmpdir, user_data_dir, e) 315 if xvfb_process: 316 xvfb_process.kill() 317 318 sys.exit(server_process.returncode) 319
