gsutil/gslib/cloud_api.py

# -*- coding: utf-8 -*-
# Copyright 2013 Google Inc. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Gsutil API for interacting with cloud storage providers."""

from __future__ import absolute_import


class CloudApi(object):
  """Abstract base class for interacting with cloud storage providers.

  Implementations of the gsutil Cloud API are not guaranteed to be thread-safe.
  Behavior when calling a gsutil Cloud API instance simultaneously across
  threads is undefined and doing so will likely cause errors. Therefore,
  a separate instance of the gsutil Cloud API should be instantiated per-thread.
  """

  def __init__(self, bucket_storage_uri_class, logger, provider=None,
               debug=0, trace_token=None):
    """Performs necessary setup for interacting with the cloud storage provider.

    Args:
      bucket_storage_uri_class: boto storage_uri class, used by APIs that
                                provide boto translation or mocking.
      logger: logging.logger for outputting log messages.
      provider: Default provider prefix describing cloud storage provider to
                connect to.
      debug: Debug level for the API implementation (0..3).
      trace_token: Google internal trace token to pass to the API
                   implementation (string).
    """
    self.bucket_storage_uri_class = bucket_storage_uri_class
    self.logger = logger
    self.provider = provider
    self.debug = debug
    self.trace_token = trace_token

  def GetBucket(self, bucket_name, provider=None, fields=None):
    """Gets Bucket metadata.

    Args:
      bucket_name: Name of the bucket.
      provider: Cloud storage provider to connect to.  If not present,
                class-wide default is used.
      fields: If present, return only these Bucket metadata fields, for
              example, ['logging', 'defaultObjectAcl']

    Raises:
      ArgumentException for errors during input validation.
      ServiceException for errors interacting with cloud storage providers.

    Returns:
      Bucket object.
    """
    raise NotImplementedError('GetBucket must be overloaded')

  def ListBuckets(self, project_id=None, provider=None, fields=None):
    """Lists bucket metadata for the given project.

    Args:
      project_id: Project owning the buckets, default from config if None.
      provider: Cloud storage provider to connect to.  If not present,
                class-wide default is used.
      fields: If present, return only these metadata fields for the listing,
              for example:
              ['items/logging', 'items/defaultObjectAcl'].
              Note that the WildcardIterator class should be used to list
              buckets instead of calling this function directly.  It amends
              the fields definition from get-like syntax such as
              ['logging', 'defaultObjectAcl'] so that the caller does not
              need to prepend 'items/' or specify fields necessary for listing
              (like nextPageToken).

    Raises:
      ArgumentException for errors during input validation.
      ServiceException for errors interacting with cloud storage providers.

    Returns:
      Iterator over Bucket objects.
    """
    raise NotImplementedError('ListBuckets must be overloaded')

  def PatchBucket(self, bucket_name, metadata, canned_acl=None,
                  canned_def_acl=None, preconditions=None, provider=None,
                  fields=None):
    """Updates bucket metadata for the bucket with patch semantics.

    Args:
      bucket_name: Name of bucket to update.
      metadata: Bucket object defining metadata to be updated.
      canned_acl: Canned ACL to apply to the bucket.
      canned_def_acl: Canned default object ACL to apply to the bucket.
      preconditions: Preconditions for the request.
      provider: Cloud storage provider to connect to.  If not present,
                class-wide default is used.
      fields: If present, return only these Bucket metadata fields.

    Raises:
      ArgumentException for errors during input validation.
      ServiceException for errors interacting with cloud storage providers.

    Returns:
      Bucket object describing new bucket metadata.
    """
    raise NotImplementedError('PatchBucket must be overloaded')

  def CreateBucket(self, bucket_name, project_id=None, metadata=None,
                   provider=None, fields=None):
    """Creates a new bucket with the specified metadata.

    Args:
      bucket_name: Name of the new bucket.
      project_id: Project owner of the new bucket, default from config if None.
      metadata: Bucket object defining new bucket metadata.
      provider: Cloud storage provider to connect to.  If not present,
                class-wide default is used.
      fields: If present, return only these Bucket metadata fields.

    Raises:
      ArgumentException for errors during input validation.
      ServiceException for errors interacting with cloud storage providers.

    Returns:
      Bucket object describing new bucket metadata.
    """
    raise NotImplementedError('CreateBucket must be overloaded')

  def DeleteBucket(self, bucket_name, preconditions=None, provider=None):
    """Deletes a bucket.

    Args:
      bucket_name: Name of the bucket to delete.
      preconditions: Preconditions for the request.
      provider: Cloud storage provider to connect to.  If not present,
                class-wide default is used.

    Raises:
      ArgumentException for errors during input validation.
      ServiceException for errors interacting with cloud storage providers.

    Returns:
      None.
    """
    raise NotImplementedError('DeleteBucket must be overloaded')

  class CsObjectOrPrefixType(object):
    """Enum class for describing CsObjectOrPrefix types."""
    OBJECT = 'object'  # Cloud object
    PREFIX = 'prefix'  # Cloud bucket subdirectory

  class CsObjectOrPrefix(object):
    """Container class for ListObjects results."""

    def __init__(self, data, datatype):
      """Stores a ListObjects result.

      Args:
        data: Root object, either an apitools Object or a string Prefix.
        datatype: CsObjectOrPrefixType of data.
      """
      self.data = data
      self.datatype = datatype

  def ListObjects(self, bucket_name, prefix=None, delimiter=None,
                  all_versions=None, provider=None, fields=None):
    """Lists objects (with metadata) and prefixes in a bucket.

    Args:
      bucket_name: Bucket containing the objects.
      prefix: Prefix for directory-like behavior.
      delimiter: Delimiter for directory-like behavior.
      all_versions: If true, list all object versions.
      provider: Cloud storage provider to connect to.  If not present,
                class-wide default is used.
      fields: If present, return only these metadata fields for the listing,
              for example:
              ['items/acl', 'items/updated', 'prefixes'].
              Note that the WildcardIterator class should be used to list
              objects instead of calling this function directly.  It amends
              the fields definition from get-like syntax such as
              ['acl', 'updated'] so that the caller does not need to
              prepend 'items/' or specify any fields necessary for listing
              (such as prefixes or nextPageToken).

    Raises:
      ArgumentException for errors during input validation.
      ServiceException for errors interacting with cloud storage providers.

    Returns:
      Iterator over CsObjectOrPrefix wrapper class.
    """
    raise NotImplementedError('ListObjects must be overloaded')

  def GetObjectMetadata(self, bucket_name, object_name, generation=None,
                        provider=None, fields=None):
    """Gets object metadata.

    Args:
      bucket_name: Bucket containing the object.
      object_name: Object name.
      generation: Generation of the object to retrieve.
      provider: Cloud storage provider to connect to.  If not present,
                class-wide default is used.
      fields: If present, return only these Object metadata fields, for
              example, ['acl', 'updated'].

    Raises:
      ArgumentException for errors during input validation.
      ServiceException for errors interacting with cloud storage providers.

    Returns:
      Object object.
    """
    raise NotImplementedError('GetObjectMetadata must be overloaded')

  def PatchObjectMetadata(self, bucket_name, object_name, metadata,
                          canned_acl=None, generation=None, preconditions=None,
                          provider=None, fields=None):
    """Updates object metadata with patch semantics.

    Args:
      bucket_name: Bucket containing the object.
      object_name: Object name for object.
      metadata: Object object defining metadata to be updated.
      canned_acl: Canned ACL to be set on the object.
      generation: Generation (or version) of the object to update.
      preconditions: Preconditions for the request.
      provider: Cloud storage provider to connect to.  If not present,
                class-wide default is used.
      fields: If present, return only these Object metadata fields.

    Raises:
      ArgumentException for errors during input validation.
      ServiceException for errors interacting with cloud storage providers.

    Returns:
      Updated object metadata.
    """
    raise NotImplementedError('PatchObjectMetadata must be overloaded')

  class DownloadStrategy(object):
    """Enum class for specifying download strategy."""
    ONE_SHOT = 'oneshot'
    RESUMABLE = 'resumable'

  def GetObjectMedia(self, bucket_name, object_name, download_stream,
                     provider=None, generation=None, object_size=None,
                     download_strategy=DownloadStrategy.ONE_SHOT, start_byte=0,
                     end_byte=None, progress_callback=None,
                     serialization_data=None, digesters=None):
    """Gets object data.

    Args:
      bucket_name: Bucket containing the object.
      object_name: Object name.
      download_stream: Stream to send the object data to.
      provider: Cloud storage provider to connect to.  If not present,
                class-wide default is used.
      generation: Generation of the object to retrieve.
      object_size: Total size of the object being downloaded.
      download_strategy: Cloud API download strategy to use for download.
      start_byte: Starting point for download (for resumable downloads and
                  range requests). Can be set to negative to request a range
                  of bytes (python equivalent of [:-3])
      end_byte: Ending byte number, inclusive, for download (for range
                requests). If None, download the rest of the object.
      progress_callback: Optional callback function for progress notifications.
                         Receives calls with arguments
                         (bytes_transferred, total_size).
      serialization_data: Implementation-specific JSON string of a dict
                          containing serialization information for the download.
      digesters: Dict of {string : digester}, where string is a name of a hash
                 algorithm, and digester is a validation digester that supports
                 update(bytes) and digest() using that algorithm.
                 Implementation can set the digester value to None to indicate
                 bytes were not successfully digested on-the-fly.

    Raises:
      ArgumentException for errors during input validation.
      ServiceException for errors interacting with cloud storage providers.

    Returns:
      Content-encoding string if it was detected that the server sent an encoded
      object during transfer, None otherwise.
    """
    raise NotImplementedError('GetObjectMedia must be overloaded')

  def UploadObject(self, upload_stream, object_metadata, canned_acl=None,
                   size=None, preconditions=None, progress_callback=None,
                   provider=None, fields=None):
    """Uploads object data and metadata.

    Args:
      upload_stream: Seekable stream of object data.
      object_metadata: Object metadata for new object.  Must include bucket
                       and object name.
      canned_acl: Optional canned ACL to apply to object. Overrides ACL set
                  in object_metadata.
      size: Optional object size.
      preconditions: Preconditions for the request.
      progress_callback: Optional callback function for progress notifications.
                         Receives calls with arguments
                         (bytes_transferred, total_size).
      provider: Cloud storage provider to connect to.  If not present,
                class-wide default is used.
      fields: If present, return only these Object metadata fields.

    Raises:
      ArgumentException for errors during input validation.
      ServiceException for errors interacting with cloud storage providers.

    Returns:
      Object object for newly created destination object.
    """
    raise NotImplementedError('UploadObject must be overloaded')

  def UploadObjectStreaming(self, upload_stream, object_metadata,
                            canned_acl=None, preconditions=None,
                            progress_callback=None, provider=None,
                            fields=None):
    """Uploads object data and metadata.

    Args:
      upload_stream: Stream of object data. May not be seekable.
      object_metadata: Object metadata for new object.  Must include bucket
                       and object name.
      canned_acl: Optional canned ACL to apply to object. Overrides ACL set
                  in object_metadata.
      preconditions: Preconditions for the request.
      progress_callback: Optional callback function for progress notifications.
                         Receives calls with arguments
                         (bytes_transferred, total_size), but fills in only
                         bytes_transferred.
      provider: Cloud storage provider to connect to.  If not present,
                class-wide default is used.
      fields: If present, return only these Object metadata fields.

    Raises:
      ArgumentException for errors during input validation.
      ServiceException for errors interacting with cloud storage providers.

    Returns:
      Object object for newly created destination object.
    """
    raise NotImplementedError('UploadObjectStreaming must be overloaded')

  def UploadObjectResumable(
      self, upload_stream, object_metadata, canned_acl=None,
      size=None, preconditions=None, serialization_data=None,
      tracker_callback=None, progress_callback=None, provider=None,
      fields=None):
    """Uploads object data and metadata using a resumable upload strategy.

    Args:
      upload_stream: Seekable stream of object data.
      object_metadata: Object metadata for new object.  Must include bucket
                       and object name.
      canned_acl: Optional canned ACL to apply to object. Overrides ACL set
                  in object_metadata.
      size: Total size of the object.
      preconditions: Preconditions for the request.
      serialization_data: Dict of {'url' : UploadURL} allowing for uploads to
                          be resumed.
      tracker_callback: Callback function taking a upload URL string.
                        Guaranteed to be called when the implementation gets an
                        upload URL, allowing the caller to resume the upload
                        across process breaks by saving the upload URL in
                        a tracker file.
      progress_callback: Optional callback function for progress notifications.
                         Receives calls with arguments
                         (bytes_transferred, total_size).
      provider: Cloud storage provider to connect to.  If not present,
                class-wide default is used.
      fields: If present, return only these Object metadata fields when the
              upload is complete.

    Raises:
      ArgumentException for errors during input validation.
      ServiceException for errors interacting with cloud storage providers.

    Returns:
      Object object for newly created destination object.
    """
    raise NotImplementedError('UploadObjectResumable must be overloaded')

  def CopyObject(self, src_obj_metadata, dst_obj_metadata, src_generation=None,
                 canned_acl=None, preconditions=None, progress_callback=None,
                 max_bytes_per_call=None, provider=None, fields=None):
    """Copies an object in the cloud.

    Args:
      src_obj_metadata: Object metadata for source object.  Must include
                        bucket name, object name, and etag.
      dst_obj_metadata: Object metadata for new object.  Must include bucket
                        and object name.
      src_generation: Generation of the source object to copy.
      canned_acl: Optional canned ACL to apply to destination object. Overrides
                  ACL set in dst_obj_metadata.
      preconditions: Destination object preconditions for the request.
      progress_callback: Optional callback function for progress notifications.
                         Receives calls with arguments
                         (bytes_transferred, total_size).
      max_bytes_per_call: Integer describing maximum number of bytes
                          to rewrite per service call.
      provider: Cloud storage provider to connect to.  If not present,
                class-wide default is used.
      fields: If present, return only these Object metadata fields.

    Raises:
      ArgumentException for errors during input validation.
      ServiceException for errors interacting with cloud storage providers.

    Returns:
      Object object for newly created destination object.
    """
    raise NotImplementedError('CopyObject must be overloaded')

  def ComposeObject(self, src_objs_metadata, dst_obj_metadata,
                    preconditions=None, provider=None, fields=None):
    """Composes an object in the cloud.

    Args:
      src_objs_metadata: List of ComposeRequest.SourceObjectsValueListEntries
                         specifying the objects to compose.
      dst_obj_metadata: Metadata for the destination object including bucket
                        and object name.
      preconditions: Destination object preconditions for the request.
      provider: Cloud storage provider to connect to.  If not present,
                class-wide default is used.
      fields: If present, return only these Object metadata fields.

    Raises:
      ArgumentException for errors during input validation.
      ServiceException for errors interacting with cloud storage providers.

    Returns:
      Composed object metadata.
    """
    raise NotImplementedError('ComposeObject must be overloaded')

  def DeleteObject(self, bucket_name, object_name, preconditions=None,
                   generation=None, provider=None):
    """Deletes an object.

    Args:
      bucket_name: Name of the containing bucket.
      object_name: Name of the object to delete.
      preconditions: Preconditions for the request.
      generation: Generation (or version) of the object to delete; if None,
                  deletes the live object.
      provider: Cloud storage provider to connect to.  If not present,
                class-wide default is used.

    Raises:
      ArgumentException for errors during input validation.
      ServiceException for errors interacting with cloud storage providers.

    Returns:
      None.
    """
    raise NotImplementedError('DeleteObject must be overloaded')

  def WatchBucket(self, bucket_name, address, channel_id, token=None,
                  provider=None, fields=None):
    """Creates a notification subscription for changes to objects in a bucket.

    Args:
      bucket_name: Bucket containing the objects.
      address: Address to which to send notifications.
      channel_id: Unique ID string for the channel.
      token: If present, token string is delivered with each notification.
      provider: Cloud storage provider to connect to.  If not present,
                class-wide default is used.
      fields: If present, return only these Channel metadata fields.

    Raises:
      ArgumentException for errors during input validation.
      ServiceException for errors interacting with cloud storage providers.

    Returns:
      Channel object describing the notification subscription.
    """
    raise NotImplementedError('WatchBucket must be overloaded')

  def StopChannel(self, channel_id, resource_id, provider=None):
    """Stops a notification channel.

    Args:
      channel_id: Unique ID string for the channel.
      resource_id: Version-agnostic ID string for the channel.
      provider: Cloud storage provider to connect to.  If not present,
                class-wide default is used.

    Raises:
      ArgumentException for errors during input validation.
      ServiceException for errors interacting with cloud storage providers.

    Returns:
      None.
    """
    raise NotImplementedError('StopChannel must be overloaded')


class Preconditions(object):
  """Preconditions class for specifying preconditions to cloud API requests."""

  def __init__(self, gen_match=None, meta_gen_match=None):
    """Instantiates a Preconditions object.

    Args:
      gen_match: Perform request only if generation of target object
                 matches the given integer. Ignored for bucket requests.
      meta_gen_match: Perform request only if metageneration of target
                      object/bucket matches the given integer.
    """
    self.gen_match = gen_match
    self.meta_gen_match = meta_gen_match


class ArgumentException(Exception):
  """Exception raised when arguments to a Cloud API method are invalid.

    This exception is never raised as a result of a failed call to a cloud
    storage provider.
  """

  def __init__(self, reason):
    Exception.__init__(self)
    self.reason = reason

  def __repr__(self):
    return str(self)

  def __str__(self):
    return '%s: %s' % (self.__class__.__name__, self.reason)


class ProjectIdException(ArgumentException):
  """Exception raised when a Project ID argument is required but not present."""


class ServiceException(Exception):
  """Exception raised when a cloud storage provider request fails.

    This exception is raised only as a result of a failed remote call.
  """

  def __init__(self, reason, status=None, body=None):
    Exception.__init__(self)
    self.reason = reason
    self.status = status
    self.body = body

  def __repr__(self):
    return str(self)

  def __str__(self):
    message = '%s:' % self.__class__.__name__
    if self.status:
      message += ' %s' % self.status
    message += ' %s' % self.reason
    if self.body:
      message += '\n%s' % self.body
    return message


class RetryableServiceException(ServiceException):
  """Exception class for retryable exceptions."""


class ResumableDownloadException(RetryableServiceException):
  """Exception raised for res. downloads that can be retried later."""


class ResumableUploadException(RetryableServiceException):
  """Exception raised for res. uploads that can be retried w/ same upload ID."""


class ResumableUploadStartOverException(RetryableServiceException):
  """Exception raised for res. uploads that can be retried w/ new upload ID."""


class ResumableUploadAbortException(ServiceException):
  """Exception raised for resumable uploads that cannot be retried later."""


class AuthenticationException(ServiceException):
  """Exception raised for errors during the authentication process."""


class PreconditionException(ServiceException):
  """Exception raised for precondition failures."""


class NotFoundException(ServiceException):
  """Exception raised when a resource is not found (404)."""


class BucketNotFoundException(NotFoundException):
  """Exception raised when a bucket resource is not found (404)."""

  def __init__(self, reason, bucket_name, status=None, body=None):
    super(BucketNotFoundException, self).__init__(reason, status=status,
                                                  body=body)
    self.bucket_name = bucket_name


class NotEmptyException(ServiceException):
  """Exception raised when trying to delete a bucket is not empty."""


class BadRequestException(ServiceException):
  """Exception raised for malformed requests.

    Where it is possible to detect invalid arguments prior to sending them
    to the server, an ArgumentException should be raised instead.
  """


class AccessDeniedException(ServiceException):
  """Exception raised  when authenticated user has insufficient access rights.

    This is raised when the authentication process succeeded but the
    authenticated user does not have access rights to the requested resource.
  """