1 # Copyright (c) 2013 The Chromium Authors. All rights reserved. 2 # Use of this source code is governed by a BSD-style license that can be 3 # found in the LICENSE file. 4 5 """Set of operations/utilities related to checking out the depot, and 6 outputting annotations on the buildbot waterfall. These are intended to be 7 used by the bisection scripts.""" 8 9 import errno 10 import os 11 import shutil 12 import subprocess 13 import sys 14 15 GCLIENT_SPEC_DATA = [ 16 { "name" : "src", 17 "url" : "https://chromium.googlesource.com/chromium/src.git", 18 "deps_file" : ".DEPS.git", 19 "managed" : True, 20 "custom_deps" : { 21 "src/data/page_cycler": "https://chrome-internal.googlesource.com/" 22 "chrome/data/page_cycler/.git", 23 "src/data/dom_perf": "https://chrome-internal.googlesource.com/" 24 "chrome/data/dom_perf/.git", 25 "src/data/mach_ports": "https://chrome-internal.googlesource.com/" 26 "chrome/data/mach_ports/.git", 27 "src/tools/perf/data": "https://chrome-internal.googlesource.com/" 28 "chrome/tools/perf/data/.git", 29 "src/third_party/adobe/flash/binaries/ppapi/linux": 30 "https://chrome-internal.googlesource.com/" 31 "chrome/deps/adobe/flash/binaries/ppapi/linux/.git", 32 "src/third_party/adobe/flash/binaries/ppapi/linux_x64": 33 "https://chrome-internal.googlesource.com/" 34 "chrome/deps/adobe/flash/binaries/ppapi/linux_x64/.git", 35 "src/third_party/adobe/flash/binaries/ppapi/mac": 36 "https://chrome-internal.googlesource.com/" 37 "chrome/deps/adobe/flash/binaries/ppapi/mac/.git", 38 "src/third_party/adobe/flash/binaries/ppapi/mac_64": 39 "https://chrome-internal.googlesource.com/" 40 "chrome/deps/adobe/flash/binaries/ppapi/mac_64/.git", 41 "src/third_party/adobe/flash/binaries/ppapi/win": 42 "https://chrome-internal.googlesource.com/" 43 "chrome/deps/adobe/flash/binaries/ppapi/win/.git", 44 "src/third_party/adobe/flash/binaries/ppapi/win_x64": 45 "https://chrome-internal.googlesource.com/" 46 "chrome/deps/adobe/flash/binaries/ppapi/win_x64/.git", 47 }, 48 "safesync_url": "", 49 }, 50 ] 51 GCLIENT_SPEC_ANDROID = "\ntarget_os = ['android']" 52 GCLIENT_CUSTOM_DEPS_V8 = {"src/v8_bleeding_edge": "git://github.com/v8/v8.git"} 53 FILE_DEPS_GIT = '.DEPS.git' 54 55 REPO_PARAMS = [ 56 'https://chrome-internal.googlesource.com/chromeos/manifest-internal/', 57 '--repo-url', 58 'https://git.chromium.org/external/repo.git' 59 ] 60 61 REPO_SYNC_COMMAND = 'git checkout -f $(git rev-list --max-count=1 '\ 62 '--before=%d remotes/m/master)' 63 64 ORIGINAL_ENV = {} 65 66 def OutputAnnotationStepStart(name): 67 """Outputs appropriate annotation to signal the start of a step to 68 a trybot. 69 70 Args: 71 name: The name of the step. 72 """ 73 print 74 print '@@@SEED_STEP %s@@@' % name 75 print '@@@STEP_CURSOR %s@@@' % name 76 print '@@@STEP_STARTED@@@' 77 print 78 sys.stdout.flush() 79 80 81 def OutputAnnotationStepClosed(): 82 """Outputs appropriate annotation to signal the closing of a step to 83 a trybot.""" 84 print 85 print '@@@STEP_CLOSED@@@' 86 print 87 sys.stdout.flush() 88 89 90 def CreateAndChangeToSourceDirectory(working_directory): 91 """Creates a directory 'bisect' as a subdirectory of 'working_directory'. If 92 the function is successful, the current working directory will change to that 93 of the new 'bisect' directory. 94 95 Returns: 96 True if the directory was successfully created (or already existed). 97 """ 98 cwd = os.getcwd() 99 os.chdir(working_directory) 100 try: 101 os.mkdir('bisect') 102 except OSError, e: 103 if e.errno != errno.EEXIST: 104 return False 105 os.chdir('bisect') 106 return True 107 108 109 def SubprocessCall(cmd, cwd=None): 110 """Runs a subprocess with specified parameters. 111 112 Args: 113 params: A list of parameters to pass to gclient. 114 cwd: Working directory to run from. 115 116 Returns: 117 The return code of the call. 118 """ 119 if os.name == 'nt': 120 # "HOME" isn't normally defined on windows, but is needed 121 # for git to find the user's .netrc file. 122 if not os.getenv('HOME'): 123 os.environ['HOME'] = os.environ['USERPROFILE'] 124 shell = os.name == 'nt' 125 return subprocess.call(cmd, shell=shell, cwd=cwd) 126 127 128 def RunGClient(params, cwd=None): 129 """Runs gclient with the specified parameters. 130 131 Args: 132 params: A list of parameters to pass to gclient. 133 cwd: Working directory to run from. 134 135 Returns: 136 The return code of the call. 137 """ 138 cmd = ['gclient'] + params 139 140 return SubprocessCall(cmd, cwd=cwd) 141 142 143 def RunRepo(params): 144 """Runs cros repo command with specified parameters. 145 146 Args: 147 params: A list of parameters to pass to gclient. 148 149 Returns: 150 The return code of the call. 151 """ 152 cmd = ['repo'] + params 153 154 return SubprocessCall(cmd) 155 156 157 def RunRepoSyncAtTimestamp(timestamp): 158 """Syncs all git depots to the timestamp specified using repo forall. 159 160 Args: 161 params: Unix timestamp to sync to. 162 163 Returns: 164 The return code of the call. 165 """ 166 repo_sync = REPO_SYNC_COMMAND % timestamp 167 cmd = ['forall', '-c', REPO_SYNC_COMMAND % timestamp] 168 return RunRepo(cmd) 169 170 171 def RunGClientAndCreateConfig(opts, custom_deps=None, cwd=None): 172 """Runs gclient and creates a config containing both src and src-internal. 173 174 Args: 175 opts: The options parsed from the command line through parse_args(). 176 custom_deps: A dictionary of additional dependencies to add to .gclient. 177 cwd: Working directory to run from. 178 179 Returns: 180 The return code of the call. 181 """ 182 spec = GCLIENT_SPEC_DATA 183 184 if custom_deps: 185 for k, v in custom_deps.iteritems(): 186 spec[0]['custom_deps'][k] = v 187 188 # Cannot have newlines in string on windows 189 spec = 'solutions =' + str(spec) 190 spec = ''.join([l for l in spec.splitlines()]) 191 192 if opts.target_platform == 'android': 193 spec += GCLIENT_SPEC_ANDROID 194 195 return_code = RunGClient( 196 ['config', '--spec=%s' % spec, '--git-deps'], cwd=cwd) 197 return return_code 198 199 200 def IsDepsFileBlink(): 201 """Reads .DEPS.git and returns whether or not we're using blink. 202 203 Returns: 204 True if blink, false if webkit. 205 """ 206 locals = {'Var': lambda _: locals["vars"][_], 207 'From': lambda *args: None} 208 execfile(FILE_DEPS_GIT, {}, locals) 209 return 'blink.git' in locals['vars']['webkit_url'] 210 211 212 def RemoveThirdPartyWebkitDirectory(): 213 """Removes third_party/WebKit. 214 215 Returns: 216 True on success. 217 """ 218 try: 219 path_to_dir = os.path.join(os.getcwd(), 'third_party', 'WebKit') 220 if os.path.exists(path_to_dir): 221 shutil.rmtree(path_to_dir) 222 except OSError, e: 223 if e.errno != errno.ENOENT: 224 return False 225 return True 226 227 228 def RunGClientAndSync(cwd=None): 229 """Runs gclient and does a normal sync. 230 231 Args: 232 cwd: Working directory to run from. 233 234 Returns: 235 The return code of the call. 236 """ 237 params = ['sync', '--verbose', '--nohooks', '--reset', '--force'] 238 return RunGClient(params, cwd=cwd) 239 240 241 def SetupGitDepot(opts): 242 """Sets up the depot for the bisection. The depot will be located in a 243 subdirectory called 'bisect'. 244 245 Args: 246 opts: The options parsed from the command line through parse_args(). 247 248 Returns: 249 True if gclient successfully created the config file and did a sync, False 250 otherwise. 251 """ 252 name = 'Setting up Bisection Depot' 253 254 if opts.output_buildbot_annotations: 255 OutputAnnotationStepStart(name) 256 257 passed = False 258 259 if not RunGClientAndCreateConfig(opts): 260 passed_deps_check = True 261 if os.path.isfile(os.path.join('src', FILE_DEPS_GIT)): 262 cwd = os.getcwd() 263 os.chdir('src') 264 if not IsDepsFileBlink(): 265 passed_deps_check = RemoveThirdPartyWebkitDirectory() 266 else: 267 passed_deps_check = True 268 os.chdir(cwd) 269 270 if passed_deps_check: 271 RunGClient(['revert']) 272 if not RunGClientAndSync(): 273 passed = True 274 275 if opts.output_buildbot_annotations: 276 print 277 OutputAnnotationStepClosed() 278 279 return passed 280 281 282 def SetupCrosRepo(): 283 """Sets up cros repo for bisecting chromeos. 284 285 Returns: 286 Returns 0 on success. 287 """ 288 cwd = os.getcwd() 289 try: 290 os.mkdir('cros') 291 except OSError, e: 292 if e.errno != errno.EEXIST: 293 return False 294 os.chdir('cros') 295 296 cmd = ['init', '-u'] + REPO_PARAMS 297 298 passed = False 299 300 if not RunRepo(cmd): 301 if not RunRepo(['sync']): 302 passed = True 303 os.chdir(cwd) 304 305 return passed 306 307 308 def CopyAndSaveOriginalEnvironmentVars(): 309 """Makes a copy of the current environment variables.""" 310 # TODO: Waiting on crbug.com/255689, will remove this after. 311 vars_to_remove = [] 312 for k, v in os.environ.iteritems(): 313 if 'ANDROID' in k: 314 vars_to_remove.append(k) 315 vars_to_remove.append('CHROME_SRC') 316 vars_to_remove.append('CHROMIUM_GYP_FILE') 317 vars_to_remove.append('GYP_CROSSCOMPILE') 318 vars_to_remove.append('GYP_DEFINES') 319 vars_to_remove.append('GYP_GENERATORS') 320 vars_to_remove.append('GYP_GENERATOR_FLAGS') 321 vars_to_remove.append('OBJCOPY') 322 for k in vars_to_remove: 323 if os.environ.has_key(k): 324 del os.environ[k] 325 326 global ORIGINAL_ENV 327 ORIGINAL_ENV = os.environ.copy() 328 329 330 def SetupAndroidBuildEnvironment(opts): 331 """Sets up the android build environment. 332 333 Args: 334 opts: The options parsed from the command line through parse_args(). 335 path_to_file: Path to the bisect script's directory. 336 337 Returns: 338 True if successful. 339 """ 340 341 # Revert the environment variables back to default before setting them up 342 # with envsetup.sh. 343 env_vars = os.environ.copy() 344 for k, _ in env_vars.iteritems(): 345 del os.environ[k] 346 for k, v in ORIGINAL_ENV.iteritems(): 347 os.environ[k] = v 348 349 path_to_file = os.path.join('build', 'android', 'envsetup.sh') 350 proc = subprocess.Popen(['bash', '-c', 'source %s && env' % path_to_file], 351 stdout=subprocess.PIPE, 352 stderr=subprocess.PIPE, 353 cwd='src') 354 (out, _) = proc.communicate() 355 356 for line in out.splitlines(): 357 (k, _, v) = line.partition('=') 358 os.environ[k] = v 359 return not proc.returncode 360 361 362 def SetupPlatformBuildEnvironment(opts): 363 """Performs any platform specific setup. 364 365 Args: 366 opts: The options parsed from the command line through parse_args(). 367 path_to_file: Path to the bisect script's directory. 368 369 Returns: 370 True if successful. 371 """ 372 if opts.target_platform == 'android': 373 CopyAndSaveOriginalEnvironmentVars() 374 return SetupAndroidBuildEnvironment(opts) 375 elif opts.target_platform == 'cros': 376 return SetupCrosRepo() 377 378 return True 379 380 381 def CreateBisectDirectoryAndSetupDepot(opts): 382 """Sets up a subdirectory 'bisect' and then retrieves a copy of the depot 383 there using gclient. 384 385 Args: 386 opts: The options parsed from the command line through parse_args(). 387 reset: Whether to reset any changes to the depot. 388 389 Returns: 390 Returns 0 on success, otherwise 1. 391 """ 392 if not CreateAndChangeToSourceDirectory(opts.working_directory): 393 print 'Error: Could not create bisect directory.' 394 print 395 return 1 396 397 if not SetupGitDepot(opts): 398 print 'Error: Failed to grab source.' 399 print 400 return 1 401 402 return 0 403
