1 # Copyright 2017 The Chromium Authors. All rights reserved. 2 # Use of this source code is governed by a BSD-style license that can be 3 # found in the LICENSE file. 4 5 6 from recipe_engine import recipe_api 7 8 UPLOAD_ATTEMPTS = 5 9 10 class GSUtilApi(recipe_api.RecipeApi): 11 def __call__(self, step_name, *args): 12 """Run gsutil with the given args.""" 13 return self.m.step(step_name, cmd=['gsutil'] + list(args)) 14 15 def cp(self, name, src, dst, extra_args=None, multithread=False): 16 """Attempt to upload or download files to/from Google Cloud Storage (GCS). 17 18 Args: 19 name: string. Will be used to fill out the step name. 20 src: string. Absolute path for a local file or gcs file (e.g. gs://...) 21 dst: string. Same as src. 22 extra_args: optional list of args to be passed to gsutil. e.g. [-Z] asks 23 all files be compressed with gzip after upload and before download. 24 multi_thread: if the -m argument should be used to copy multiple items 25 at once (e.g. gsutil -m cp foo* gs://bar/dir) 26 27 If the operation fails, it will be retried multiple times. 28 """ 29 cmd = ['cp'] 30 if multithread: 31 cmd = ['-m'] + cmd 32 if extra_args: 33 cmd.extend(extra_args) 34 cmd.extend([src, dst]) 35 36 name = 'upload %s' % name 37 for i in xrange(UPLOAD_ATTEMPTS): 38 step_name = name 39 if i > 0: 40 step_name += ' (attempt %d)' % (i+1) 41 try: 42 self(step_name, *cmd) 43 break 44 except self.m.step.StepFailure: 45 if i == UPLOAD_ATTEMPTS - 1: 46 raise 47
