Home | History | Annotate | Download | only in lib 
      1 #!/usr/bin/env python
      2 #
      3 # Copyright 2016 - The Android Open Source Project
      4 #
      5 # Licensed under the Apache License, Version 2.0 (the "License");
      6 # you may not use this file except in compliance with the License.
      7 # You may obtain a copy of the License at
      8 #
      9 #     http://www.apache.org/licenses/LICENSE-2.0
     10 #
     11 # Unless required by applicable law or agreed to in writing, software
     12 # distributed under the License is distributed on an "AS IS" BASIS,
     13 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
     14 # See the License for the specific language governing permissions and
     15 # limitations under the License.
     16 
     17 """A client that talks to Google Cloud Storage APIs."""
     18 
     19 import io
     20 import logging
     21 import os
     22 
     23 import apiclient
     24 
     25 from acloud.internal.lib import base_cloud_client
     26 from acloud.internal.lib import utils
     27 from acloud.public import errors
     28 
     29 logger = logging.getLogger(__name__)
     30 
     31 
     32 class StorageClient(base_cloud_client.BaseCloudApiClient):
     33     """Client that talks to Google Cloud Storages."""
     34 
     35     # API settings, used by BaseCloudApiClient.
     36     API_NAME = "storage"
     37     API_VERSION = "v1"
     38     SCOPE = "https://www.googleapis.com/auth/devstorage.read_write"
     39     GET_OBJ_MAX_RETRY = 3
     40     GET_OBJ_RETRY_SLEEP = 5
     41 
     42     # Other class variables.
     43     OBJECT_URL_FMT = "https://storage.googleapis.com/%s/%s"
     44 
     45     def Get(self, bucket_name, object_name):
     46         """Get object in a bucket.
     47 
     48         Args:
     49             bucket_name: String, google cloud storage bucket name.
     50             object_name: String, full path to the object within the bucket.
     51 
     52         Returns:
     53             A dictronary representing an object resource.
     54         """
     55         request = self.service.objects().get(bucket=bucket_name,
     56                                              object=object_name)
     57         return self.Execute(request)
     58 
     59     def List(self, bucket_name, prefix=None):
     60         """Lists objects in a bucket.
     61 
     62         Args:
     63             bucket_name: String, google cloud storage bucket name.
     64             prefix: String, Filter results to objects whose names begin with
     65                     this prefix.
     66 
     67         Returns:
     68             A list of google storage objects whose names match the prefix.
     69             Each element is dictionary that wraps all the information about an object.
     70         """
     71         logger.debug("Listing storage bucket: %s, prefix: %s", bucket_name,
     72                      prefix)
     73         items = self.ListWithMultiPages(
     74             api_resource=self.service.objects().list,
     75             bucket=bucket_name,
     76             prefix=prefix)
     77         return items
     78 
     79     def Upload(self, local_src, bucket_name, object_name, mime_type):
     80         """Uploads a file.
     81 
     82         Args:
     83             local_src: string, a local path to a file to be uploaded.
     84             bucket_name: string, google cloud storage bucket name.
     85             object_name: string, the name of the remote file in storage.
     86             mime_type: string, mime-type of the file.
     87 
     88         Returns:
     89             URL to the inserted artifact in storage.
     90         """
     91         logger.info("Uploading file: src: %s, bucket: %s, object: %s",
     92                     local_src, bucket_name, object_name)
     93         try:
     94             with io.FileIO(local_src, mode="rb") as fh:
     95                 media = apiclient.http.MediaIoBaseUpload(fh, mime_type)
     96                 request = self.service.objects().insert(bucket=bucket_name,
     97                                                         name=object_name,
     98                                                         media_body=media)
     99                 response = self.Execute(request)
    100             logger.info("Uploaded artifact: %s", response["selfLink"])
    101             return response
    102         except OSError as e:
    103             logger.error("Uploading artifact fails: %s", str(e))
    104             raise errors.DriverError(str(e))
    105 
    106     def Delete(self, bucket_name, object_name):
    107         """Deletes a file.
    108 
    109         Args:
    110             bucket_name: string, google cloud storage bucket name.
    111             object_name: string, the name of the remote file in storage.
    112         """
    113         logger.info("Deleting file: bucket: %s, object: %s", bucket_name,
    114                     object_name)
    115         request = self.service.objects().delete(bucket=bucket_name,
    116                                                 object=object_name)
    117         self.Execute(request)
    118         logger.info("Deleted file: bucket: %s, object: %s", bucket_name,
    119                     object_name)
    120 
    121     def DeleteFiles(self, bucket_name, object_names):
    122         """Deletes multiple files.
    123 
    124         Args:
    125             bucket_name: string, google cloud storage bucket name.
    126             object_names: A list of strings, each of which is a name of a remote file.
    127 
    128         Returns:
    129             A tuple, (deleted, failed, error_msgs)
    130             deleted: A list of names of objects that have been deleted.
    131             faild: A list of names of objects that we fail to delete.
    132             error_msgs: A list of failure messages.
    133         """
    134         deleted = []
    135         failed = []
    136         error_msgs = []
    137         for object_name in object_names:
    138             try:
    139                 self.Delete(bucket_name, object_name)
    140                 deleted.append(object_name)
    141             except errors.DriverError as e:
    142                 failed.append(object_name)
    143                 error_msgs.append(str(e))
    144         return deleted, failed, error_msgs
    145 
    146     def GetUrl(self, bucket_name, object_name):
    147         """Get information about a file object.
    148 
    149         Args:
    150             bucket_name: string, google cloud storage bucket name.
    151             object_name: string, name of the file to look for.
    152 
    153         Returns:
    154             Value of "selfLink" field from the response, which represents
    155             a url to the file.
    156 
    157         Raises:
    158             errors.ResourceNotFoundError: when file is not found.
    159         """
    160         item = utils.RetryExceptionType(
    161                 errors.ResourceNotFoundError,
    162                 max_retries=self.GET_OBJ_MAX_RETRY, functor=self.Get,
    163                 sleep_multiplier=self.GET_OBJ_RETRY_SLEEP,
    164                 bucket_name=bucket_name, object_name=object_name)
    165         return item["selfLink"]
    166