Home | History | Annotate | Download | only in lib 
      1 #!/usr/bin/env python
      2 #
      3 # Copyright 2016 - The Android Open Source Project
      4 #
      5 # Licensed under the Apache License, Version 2.0 (the "License");
      6 # you may not use this file except in compliance with the License.
      7 # You may obtain a copy of the License at
      8 #
      9 #     http://www.apache.org/licenses/LICENSE-2.0
     10 #
     11 # Unless required by applicable law or agreed to in writing, software
     12 # distributed under the License is distributed on an "AS IS" BASIS,
     13 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
     14 # See the License for the specific language governing permissions and
     15 # limitations under the License.
     16 
     17 """Base Cloud API Client.
     18 
     19 BasicCloudApiCliend does basic setup for a cloud API.
     20 """
     21 import httplib
     22 import logging
     23 import os
     24 import socket
     25 import ssl
     26 
     27 from apiclient import errors as gerrors
     28 from apiclient.discovery import build
     29 import apiclient.http
     30 import httplib2
     31 from oauth2client import client
     32 
     33 from acloud.internal.lib import utils
     34 from acloud.public import errors
     35 
     36 logger = logging.getLogger(__name__)
     37 
     38 
     39 class BaseCloudApiClient(object):
     40     """A class that does basic setup for a cloud API."""
     41 
     42     # To be overriden by subclasses.
     43     API_NAME = ""
     44     API_VERSION = "v1"
     45     SCOPE = ""
     46 
     47     # Defaults for retry.
     48     RETRY_COUNT = 5
     49     RETRY_BACKOFF_FACTOR = 1.5
     50     RETRY_SLEEP_MULTIPLIER = 2
     51     RETRY_HTTP_CODES = [
     52         # 403 is to retry the "Rate Limit Exceeded" error.
     53         # We could retry on a finer-grained error message later if necessary.
     54         403,
     55         500,  # Internal Server Error
     56         502,  # Bad Gateway
     57         503,  # Service Unavailable
     58     ]
     59     RETRIABLE_ERRORS = (httplib.HTTPException, httplib2.HttpLib2Error,
     60                         socket.error, ssl.SSLError)
     61     RETRIABLE_AUTH_ERRORS = (client.AccessTokenRefreshError, )
     62 
     63     def __init__(self, oauth2_credentials):
     64         """Initialize.
     65 
     66         Args:
     67             oauth2_credentials: An oauth2client.OAuth2Credentials instance.
     68         """
     69         self._service = self.InitResourceHandle(oauth2_credentials)
     70 
     71     @classmethod
     72     def InitResourceHandle(cls, oauth2_credentials):
     73         """Authenticate and initialize a Resource object.
     74 
     75         Authenticate http and create a Resource object with methods
     76         for interacting with the service.
     77 
     78         Args:
     79             oauth2_credentials: An oauth2client.OAuth2Credentials instance.
     80 
     81         Returns:
     82             An apiclient.discovery.Resource object
     83         """
     84         http_auth = oauth2_credentials.authorize(httplib2.Http())
     85         return utils.RetryExceptionType(
     86                 exception_types=cls.RETRIABLE_AUTH_ERRORS,
     87                 max_retries=cls.RETRY_COUNT,
     88                 functor=build,
     89                 sleep_multiplier=cls.RETRY_SLEEP_MULTIPLIER,
     90                 retry_backoff_factor=cls.RETRY_BACKOFF_FACTOR,
     91                 serviceName=cls.API_NAME,
     92                 version=cls.API_VERSION,
     93                 http=http_auth)
     94 
     95     def _ShouldRetry(self, exception, retry_http_codes,
     96                      other_retriable_errors):
     97         """Check if exception is retriable.
     98 
     99         Args:
    100             exception: An instance of Exception.
    101             retry_http_codes: a list of integers, retriable HTTP codes of
    102                               HttpError
    103             other_retriable_errors: a tuple of error types to retry other than
    104                                     HttpError.
    105 
    106         Returns:
    107             Boolean, True if retriable, False otherwise.
    108         """
    109         if isinstance(exception, other_retriable_errors):
    110             return True
    111 
    112         if isinstance(exception, errors.HttpError):
    113             if exception.code in retry_http_codes:
    114                 return True
    115             else:
    116                 logger.debug("_ShouldRetry: Exception code %s not in %s: %s",
    117                              exception.code, retry_http_codes, str(exception))
    118 
    119         logger.debug(
    120             "_ShouldRetry: Exception %s is not one of %s: %s", type(exception),
    121             list(other_retriable_errors) + [errors.HttpError], str(exception))
    122         return False
    123 
    124     def _TranslateError(self, exception):
    125         """Translate the exception to a desired type.
    126 
    127         Args:
    128             exception: An instance of Exception.
    129 
    130         Returns:
    131             gerrors.HttpError will be translated to errors.HttpError.
    132             If the error code is errors.HTTP_NOT_FOUND_CODE, it will
    133             be translated to errors.ResourceNotFoundError.
    134             Unrecognized error type will not be translated and will
    135             be returned as is.
    136         """
    137         if isinstance(exception, gerrors.HttpError):
    138             exception = errors.HttpError.CreateFromHttpError(exception)
    139             if exception.code == errors.HTTP_NOT_FOUND_CODE:
    140                 exception = errors.ResourceNotFoundError(exception.code,
    141                                                          str(exception))
    142         return exception
    143 
    144     def ExecuteOnce(self, api):
    145         """Execute an api and parse the errors.
    146 
    147         Args:
    148             api: An apiclient.http.HttpRequest, representing the api to execute.
    149 
    150         Returns:
    151             Execution result of the api.
    152 
    153         Raises:
    154             errors.ResourceNotFoundError: For 404 error.
    155             errors.HttpError: For other types of http error.
    156         """
    157         try:
    158             return api.execute()
    159         except gerrors.HttpError as e:
    160             raise self._TranslateError(e)
    161 
    162     def Execute(self,
    163                 api,
    164                 retry_http_codes=None,
    165                 max_retry=None,
    166                 sleep=None,
    167                 backoff_factor=None,
    168                 other_retriable_errors=None):
    169         """Execute an api with retry.
    170 
    171         Call ExecuteOnce and retry on http error with given codes.
    172 
    173         Args:
    174             api: An apiclient.http.HttpRequest, representing the api to execute:
    175             retry_http_codes: A list of http codes to retry.
    176             max_retry: See utils.Retry.
    177             sleep: See utils.Retry.
    178             backoff_factor: See utils.Retry.
    179             other_retriable_errors: A tuple of error types that should be retried
    180                                     other than errors.HttpError.
    181 
    182         Returns:
    183           Execution result of the api.
    184 
    185         Raises:
    186           See ExecuteOnce.
    187         """
    188         retry_http_codes = (self.RETRY_HTTP_CODES if retry_http_codes is None
    189                             else retry_http_codes)
    190         max_retry = (self.RETRY_COUNT if max_retry is None else max_retry)
    191         sleep = (self.RETRY_SLEEP_MULTIPLIER if sleep is None else sleep)
    192         backoff_factor = (self.RETRY_BACKOFF_FACTOR if backoff_factor is None
    193                           else backoff_factor)
    194         other_retriable_errors = (self.RETRIABLE_ERRORS
    195                                   if other_retriable_errors is None else
    196                                   other_retriable_errors)
    197 
    198         def _Handler(exc):
    199             """Check if |exc| is a retriable exception.
    200 
    201             Args:
    202                 exc: An exception.
    203 
    204             Returns:
    205                 True if exc is an errors.HttpError and code exists in |retry_http_codes|
    206                 False otherwise.
    207             """
    208             if self._ShouldRetry(exc, retry_http_codes,
    209                                  other_retriable_errors):
    210                 logger.debug("Will retry error: %s", str(exc))
    211                 return True
    212             return False
    213 
    214         return utils.Retry(
    215              _Handler, max_retries=max_retry, functor=self.ExecuteOnce,
    216              sleep_multiplier=sleep, retry_backoff_factor=backoff_factor,
    217              api=api)
    218 
    219     def BatchExecuteOnce(self, requests):
    220         """Execute requests in a batch.
    221 
    222         Args:
    223             requests: A dictionary where key is request id and value
    224                       is an http request.
    225 
    226         Returns:
    227             results, a dictionary in the following format
    228             {request_id: (response, exception)}
    229             request_ids are those from requests; response
    230             is the http response for the request or None on error;
    231             exception is an instance of DriverError or None if no error.
    232         """
    233         results = {}
    234 
    235         def _CallBack(request_id, response, exception):
    236             results[request_id] = (response, self._TranslateError(exception))
    237 
    238         batch = apiclient.http.BatchHttpRequest()
    239         for request_id, request in requests.iteritems():
    240             batch.add(request=request,
    241                       callback=_CallBack,
    242                       request_id=request_id)
    243         batch.execute()
    244         return results
    245 
    246     def BatchExecute(self,
    247                      requests,
    248                      retry_http_codes=None,
    249                      max_retry=None,
    250                      sleep=None,
    251                      backoff_factor=None,
    252                      other_retriable_errors=None):
    253         """Batch execute multiple requests with retry.
    254 
    255         Call BatchExecuteOnce and retry on http error with given codes.
    256 
    257         Args:
    258             requests: A dictionary where key is request id picked by caller,
    259                       and value is a apiclient.http.HttpRequest.
    260             retry_http_codes: A list of http codes to retry.
    261             max_retry: See utils.Retry.
    262             sleep: See utils.Retry.
    263             backoff_factor: See utils.Retry.
    264             other_retriable_errors: A tuple of error types that should be retried
    265                                     other than errors.HttpError.
    266 
    267         Returns:
    268             results, a dictionary in the following format
    269             {request_id: (response, exception)}
    270             request_ids are those from requests; response
    271             is the http response for the request or None on error;
    272             exception is an instance of DriverError or None if no error.
    273         """
    274         executor = utils.BatchHttpRequestExecutor(
    275             self.BatchExecuteOnce,
    276             requests=requests,
    277             retry_http_codes=retry_http_codes or self.RETRY_HTTP_CODES,
    278             max_retry=max_retry or self.RETRY_COUNT,
    279             sleep=sleep or self.RETRY_SLEEP_MULTIPLIER,
    280             backoff_factor=backoff_factor or self.RETRY_BACKOFF_FACTOR,
    281             other_retriable_errors=other_retriable_errors or
    282             self.RETRIABLE_ERRORS)
    283         executor.Execute()
    284         return executor.GetResults()
    285 
    286     def ListWithMultiPages(self, api_resource, *args, **kwargs):
    287         """Call an api that list a type of resource.
    288 
    289         Multiple google services support listing a type of
    290         resource (e.g list gce instances, list storage objects).
    291         The querying pattern is similar --
    292         Step 1: execute the api and get a response object like,
    293         {
    294             "items": [..list of resource..],
    295             # The continuation token that can be used
    296             # to get the next page.
    297             "nextPageToken": "A String",
    298         }
    299         Step 2: execute the api again with the nextPageToken to
    300         retrieve more pages and get a response object.
    301 
    302         Step 3: Repeat Step 2 until no more page.
    303 
    304         This method encapsulates the generic logic of
    305         calling such listing api.
    306 
    307         Args:
    308             api_resource: An apiclient.discovery.Resource object
    309                 used to create an http request for the listing api.
    310             *args: Arguments used to create the http request.
    311             **kwargs: Keyword based arguments to create the http
    312                       request.
    313 
    314         Returns:
    315             A list of items.
    316         """
    317         items = []
    318         next_page_token = None
    319         while True:
    320             api = api_resource(pageToken=next_page_token, *args, **kwargs)
    321             response = self.Execute(api)
    322             items.extend(response.get("items", []))
    323             next_page_token = response.get("nextPageToken")
    324             if not next_page_token:
    325                 break
    326         return items
    327 
    328     @property
    329     def service(self):
    330         """Return self._service as a property."""
    331         return self._service
    332