1 # -*- coding: utf-8 -*- 2 # Copyright 2013 Google Inc. All Rights Reserved. 3 # 4 # Licensed under the Apache License, Version 2.0 (the "License"); 5 # you may not use this file except in compliance with the License. 6 # You may obtain a copy of the License at 7 # 8 # http://www.apache.org/licenses/LICENSE-2.0 9 # 10 # Unless required by applicable law or agreed to in writing, software 11 # distributed under the License is distributed on an "AS IS" BASIS, 12 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 # See the License for the specific language governing permissions and 14 # limitations under the License. 15 """Gsutil API delegator for interacting with cloud storage providers.""" 16 17 from __future__ import absolute_import 18 19 import boto 20 from boto import config 21 from gslib.cloud_api import ArgumentException 22 from gslib.cloud_api import CloudApi 23 from gslib.cs_api_map import ApiMapConstants 24 from gslib.cs_api_map import ApiSelector 25 26 27 class CloudApiDelegator(CloudApi): 28 """Class that handles delegating requests to gsutil Cloud API implementations. 29 30 This class is responsible for determining at runtime which gsutil Cloud API 31 implementation should service the request based on the Cloud storage provider, 32 command-level API support, and configuration file override. 33 34 During initialization it takes as an argument a gsutil_api_map which maps 35 providers to their default and supported gsutil Cloud API implementations 36 (see comments in cs_api_map for details). 37 38 Instantiation of multiple delegators per-thread is required for multiprocess 39 and/or multithreaded operations. Calling methods on the same delegator in 40 multiple threads is unsafe. 41 """ 42 43 def __init__(self, bucket_storage_uri_class, gsutil_api_map, logger, 44 provider=None, debug=0, trace_token=None): 45 """Performs necessary setup for delegating cloud storage requests. 46 47 This function has different arguments than the gsutil Cloud API __init__ 48 function because of the delegation responsibilties of this class. 49 50 Args: 51 bucket_storage_uri_class: boto storage_uri class, used by APIs that 52 provide boto translation or mocking. 53 gsutil_api_map: Map of providers and API selector tuples to api classes 54 which can be used to communicate with those providers. 55 logger: logging.logger for outputting log messages. 56 provider: Default provider prefix describing cloud storage provider to 57 connect to. 58 debug: Debug level for the API implementation (0..3). 59 trace_token: Apiary trace token to pass to API. 60 """ 61 super(CloudApiDelegator, self).__init__(bucket_storage_uri_class, logger, 62 provider=provider, debug=debug, 63 trace_token=trace_token) 64 self.api_map = gsutil_api_map 65 self.prefer_api = boto.config.get('GSUtil', 'prefer_api', '').upper() 66 self.loaded_apis = {} 67 68 if not self.api_map[ApiMapConstants.API_MAP]: 69 raise ArgumentException('No apiclass supplied for gsutil Cloud API map.') 70 71 def _GetApi(self, provider): 72 """Returns a valid CloudApi for use by the caller. 73 74 This function lazy-loads connection and credentials using the API map 75 and credential store provided during class initialization. 76 77 Args: 78 provider: Provider to load API for. If None, class-wide default is used. 79 80 Raises: 81 ArgumentException if there is no matching API available in the API map. 82 83 Returns: 84 Valid API instance that can be used to communicate with the Cloud 85 Storage provider. 86 """ 87 provider = provider or self.provider 88 if not provider: 89 raise ArgumentException('No provider selected for _GetApi') 90 91 provider = str(provider) 92 if provider not in self.loaded_apis: 93 self.loaded_apis[provider] = {} 94 95 api_selector = self.GetApiSelector(provider) 96 if api_selector not in self.loaded_apis[provider]: 97 # Need to load the API. 98 self._LoadApi(provider, api_selector) 99 100 return self.loaded_apis[provider][api_selector] 101 102 def _LoadApi(self, provider, api_selector): 103 """Loads a CloudApi into the loaded_apis map for this class. 104 105 Args: 106 provider: Provider to load the API for. 107 api_selector: cs_api_map.ApiSelector defining the API type. 108 """ 109 if provider not in self.api_map[ApiMapConstants.API_MAP]: 110 raise ArgumentException( 111 'gsutil Cloud API map contains no entry for provider %s.' % provider) 112 if api_selector not in self.api_map[ApiMapConstants.API_MAP][provider]: 113 raise ArgumentException( 114 'gsutil Cloud API map does not support API %s for provider %s.' % 115 (api_selector, provider)) 116 self.loaded_apis[provider][api_selector] = ( 117 self.api_map[ApiMapConstants.API_MAP][provider][api_selector]( 118 self.bucket_storage_uri_class, 119 self.logger, 120 provider=provider, 121 debug=self.debug, 122 trace_token=self.trace_token)) 123 124 def GetApiSelector(self, provider=None): 125 """Returns a cs_api_map.ApiSelector based on input and configuration. 126 127 Args: 128 provider: Provider to return the ApiSelector for. If None, class-wide 129 default is used. 130 131 Returns: 132 cs_api_map.ApiSelector that will be used for calls to the delegator 133 for this provider. 134 """ 135 selected_provider = provider or self.provider 136 if not selected_provider: 137 raise ArgumentException('No provider selected for CloudApi') 138 139 if (selected_provider not in self.api_map[ApiMapConstants.DEFAULT_MAP] or 140 self.api_map[ApiMapConstants.DEFAULT_MAP][selected_provider] not in 141 self.api_map[ApiMapConstants.API_MAP][selected_provider]): 142 raise ArgumentException('No default api available for provider %s' % 143 selected_provider) 144 145 if selected_provider not in self.api_map[ApiMapConstants.SUPPORT_MAP]: 146 raise ArgumentException('No supported apis available for provider %s' % 147 selected_provider) 148 149 api = self.api_map[ApiMapConstants.DEFAULT_MAP][selected_provider] 150 151 # If we have only HMAC credentials for Google Cloud Storage, we must use 152 # the XML API as the JSON API does not support HMAC. 153 # 154 # Technically if we have only HMAC credentials, we should still be able to 155 # access public read resources via the JSON API, but the XML API can do 156 # that just as well. It is better to use it than inspect the credentials on 157 # every HTTP call. 158 if (provider == 'gs' and 159 not config.has_option('Credentials', 'gs_oauth2_refresh_token') and 160 not (config.has_option('Credentials', 'gs_service_client_id') 161 and config.has_option('Credentials', 'gs_service_key_file')) and 162 (config.has_option('Credentials', 'gs_access_key_id') 163 and config.has_option('Credentials', 'gs_secret_access_key'))): 164 api = ApiSelector.XML 165 # Try to force the user's preference to a supported API. 166 elif self.prefer_api in (self.api_map[ApiMapConstants.SUPPORT_MAP] 167 [selected_provider]): 168 api = self.prefer_api 169 return api 170 171 # For function docstrings, see CloudApi class. 172 def GetBucket(self, bucket_name, provider=None, fields=None): 173 return self._GetApi(provider).GetBucket(bucket_name, fields=fields) 174 175 def ListBuckets(self, project_id=None, provider=None, fields=None): 176 return self._GetApi(provider).ListBuckets(project_id=project_id, 177 fields=fields) 178 179 def PatchBucket(self, bucket_name, metadata, canned_acl=None, 180 canned_def_acl=None, preconditions=None, provider=None, 181 fields=None): 182 return self._GetApi(provider).PatchBucket( 183 bucket_name, metadata, canned_acl=canned_acl, 184 canned_def_acl=canned_def_acl, preconditions=preconditions, 185 fields=fields) 186 187 def CreateBucket(self, bucket_name, project_id=None, metadata=None, 188 provider=None, fields=None): 189 return self._GetApi(provider).CreateBucket( 190 bucket_name, project_id=project_id, metadata=metadata, fields=fields) 191 192 def DeleteBucket(self, bucket_name, preconditions=None, provider=None): 193 return self._GetApi(provider).DeleteBucket(bucket_name, 194 preconditions=preconditions) 195 196 def ListObjects(self, bucket_name, prefix=None, delimiter=None, 197 all_versions=None, provider=None, fields=None): 198 return self._GetApi(provider).ListObjects( 199 bucket_name, prefix=prefix, delimiter=delimiter, 200 all_versions=all_versions, fields=fields) 201 202 def GetObjectMetadata(self, bucket_name, object_name, generation=None, 203 provider=None, fields=None): 204 return self._GetApi(provider).GetObjectMetadata( 205 bucket_name, object_name, generation=generation, fields=fields) 206 207 def PatchObjectMetadata(self, bucket_name, object_name, metadata, 208 canned_acl=None, generation=None, preconditions=None, 209 provider=None, fields=None): 210 return self._GetApi(provider).PatchObjectMetadata( 211 bucket_name, object_name, metadata, canned_acl=canned_acl, 212 generation=generation, preconditions=preconditions, fields=fields) 213 214 def GetObjectMedia( 215 self, bucket_name, object_name, download_stream, provider=None, 216 generation=None, object_size=None, 217 download_strategy=CloudApi.DownloadStrategy.ONE_SHOT, 218 start_byte=0, end_byte=None, progress_callback=None, 219 serialization_data=None, digesters=None): 220 return self._GetApi(provider).GetObjectMedia( 221 bucket_name, object_name, download_stream, 222 download_strategy=download_strategy, start_byte=start_byte, 223 end_byte=end_byte, generation=generation, object_size=object_size, 224 progress_callback=progress_callback, 225 serialization_data=serialization_data, digesters=digesters) 226 227 def UploadObject(self, upload_stream, object_metadata, size=None, 228 canned_acl=None, preconditions=None, progress_callback=None, 229 provider=None, fields=None): 230 return self._GetApi(provider).UploadObject( 231 upload_stream, object_metadata, size=size, canned_acl=canned_acl, 232 preconditions=preconditions, progress_callback=progress_callback, 233 fields=fields) 234 235 def UploadObjectStreaming(self, upload_stream, object_metadata, 236 canned_acl=None, preconditions=None, 237 progress_callback=None, provider=None, fields=None): 238 return self._GetApi(provider).UploadObjectStreaming( 239 upload_stream, object_metadata, canned_acl=canned_acl, 240 preconditions=preconditions, progress_callback=progress_callback, 241 fields=fields) 242 243 def UploadObjectResumable( 244 self, upload_stream, object_metadata, canned_acl=None, preconditions=None, 245 provider=None, fields=None, size=None, serialization_data=None, 246 tracker_callback=None, progress_callback=None): 247 return self._GetApi(provider).UploadObjectResumable( 248 upload_stream, object_metadata, canned_acl=canned_acl, 249 preconditions=preconditions, size=size, fields=fields, 250 serialization_data=serialization_data, 251 tracker_callback=tracker_callback, progress_callback=progress_callback) 252 253 def CopyObject(self, src_obj_metadata, dst_obj_metadata, src_generation=None, 254 canned_acl=None, preconditions=None, progress_callback=None, 255 max_bytes_per_call=None, provider=None, fields=None): 256 return self._GetApi(provider).CopyObject( 257 src_obj_metadata, dst_obj_metadata, src_generation=src_generation, 258 canned_acl=canned_acl, preconditions=preconditions, 259 progress_callback=progress_callback, 260 max_bytes_per_call=max_bytes_per_call, fields=fields) 261 262 def ComposeObject(self, src_objs_metadata, dst_obj_metadata, 263 preconditions=None, provider=None, fields=None): 264 return self._GetApi(provider).ComposeObject( 265 src_objs_metadata, dst_obj_metadata, preconditions=preconditions, 266 fields=fields) 267 268 def DeleteObject(self, bucket_name, object_name, preconditions=None, 269 generation=None, provider=None): 270 return self._GetApi(provider).DeleteObject( 271 bucket_name, object_name, preconditions=preconditions, 272 generation=generation) 273 274 def WatchBucket(self, bucket_name, address, channel_id, token=None, 275 provider=None, fields=None): 276 return self._GetApi(provider).WatchBucket( 277 bucket_name, address, channel_id, token=token, fields=fields) 278 279 def StopChannel(self, channel_id, resource_id, provider=None): 280 return self._GetApi(provider).StopChannel(channel_id, resource_id) 281 282 def XmlPassThroughGetAcl(self, storage_url, def_obj_acl=False, provider=None): 283 """XML compatibility function for getting ACLs. 284 285 Args: 286 storage_url: StorageUrl object. 287 def_obj_acl: If true, get the default object ACL on a bucket. 288 provider: Cloud storage provider to connect to. If not present, 289 class-wide default is used. 290 291 Raises: 292 ArgumentException for errors during input validation. 293 ServiceException for errors interacting with cloud storage providers. 294 295 Returns: 296 ACL XML for the resource specified by storage_url. 297 """ 298 return self._GetApi(provider).XmlPassThroughGetAcl(storage_url, 299 def_obj_acl=def_obj_acl) 300 301 def XmlPassThroughSetAcl(self, acl_text, storage_url, canned=True, 302 def_obj_acl=False, provider=None): 303 """XML compatibility function for setting ACLs. 304 305 Args: 306 acl_text: XML ACL or canned ACL string. 307 storage_url: StorageUrl object. 308 canned: If true, acl_text is treated as a canned ACL string. 309 def_obj_acl: If true, set the default object ACL on a bucket. 310 provider: Cloud storage provider to connect to. If not present, 311 class-wide default is used. 312 313 Raises: 314 ArgumentException for errors during input validation. 315 ServiceException for errors interacting with cloud storage providers. 316 317 Returns: 318 None. 319 """ 320 self._GetApi(provider).XmlPassThroughSetAcl( 321 acl_text, storage_url, canned=canned, def_obj_acl=def_obj_acl) 322 323 def XmlPassThroughGetCors(self, storage_url, provider=None): 324 """XML compatibility function for getting CORS configuration on a bucket. 325 326 Args: 327 storage_url: StorageUrl object. 328 provider: Cloud storage provider to connect to. If not present, 329 class-wide default is used. 330 331 Raises: 332 ArgumentException for errors during input validation. 333 ServiceException for errors interacting with cloud storage providers. 334 335 Returns: 336 CORS configuration XML for the bucket specified by storage_url. 337 """ 338 return self._GetApi(provider).XmlPassThroughGetCors(storage_url) 339 340 def XmlPassThroughSetCors(self, cors_text, storage_url, provider=None): 341 """XML compatibility function for setting CORS configuration on a bucket. 342 343 Args: 344 cors_text: Raw CORS XML string. 345 storage_url: StorageUrl object. 346 provider: Cloud storage provider to connect to. If not present, 347 class-wide default is used. 348 349 Raises: 350 ArgumentException for errors during input validation. 351 ServiceException for errors interacting with cloud storage providers. 352 353 Returns: 354 None. 355 """ 356 self._GetApi(provider).XmlPassThroughSetCors(cors_text, storage_url) 357 358 def XmlPassThroughGetLifecycle(self, storage_url, provider=None): 359 """XML compatibility function for getting lifecycle config on a bucket. 360 361 Args: 362 storage_url: StorageUrl object. 363 provider: Cloud storage provider to connect to. If not present, 364 class-wide default is used. 365 366 Raises: 367 ArgumentException for errors during input validation. 368 ServiceException for errors interacting with cloud storage providers. 369 370 Returns: 371 Lifecycle configuration XML for the bucket specified by storage_url. 372 """ 373 return self._GetApi(provider).XmlPassThroughGetLifecycle(storage_url) 374 375 def XmlPassThroughSetLifecycle(self, lifecycle_text, storage_url, 376 provider=None): 377 """XML compatibility function for setting CORS configuration on a bucket. 378 379 Args: 380 lifecycle_text: Raw lifecycle configuration XML string. 381 storage_url: StorageUrl object. 382 provider: Cloud storage provider to connect to. If not present, 383 class-wide default is used. 384 385 Raises: 386 ArgumentException for errors during input validation. 387 ServiceException for errors interacting with cloud storage providers. 388 389 Returns: 390 None. 391 """ 392 self._GetApi(provider).XmlPassThroughSetLifecycle(lifecycle_text, 393 storage_url) 394 395 def XmlPassThroughGetLogging(self, storage_url, provider=None): 396 """XML compatibility function for getting logging configuration on a bucket. 397 398 Args: 399 storage_url: StorageUrl object. 400 provider: Cloud storage provider to connect to. If not present, 401 class-wide default is used. 402 403 Raises: 404 ArgumentException for errors during input validation. 405 ServiceException for errors interacting with cloud storage providers. 406 407 Returns: 408 Logging configuration XML for the bucket specified by storage_url. 409 """ 410 return self._GetApi(provider).XmlPassThroughGetLogging(storage_url) 411 412 def XmlPassThroughGetWebsite(self, storage_url, provider=None): 413 """XML compatibility function for getting website configuration on a bucket. 414 415 Args: 416 storage_url: StorageUrl object. 417 provider: Cloud storage provider to connect to. If not present, 418 class-wide default is used. 419 420 Raises: 421 ArgumentException for errors during input validation. 422 ServiceException for errors interacting with cloud storage providers. 423 424 Returns: 425 Website configuration XML for the bucket specified by storage_url. 426 """ 427 return self._GetApi(provider).XmlPassThroughGetWebsite(storage_url) 428 429
