1 # Copyright (c) 2010 Mitch Garnaat http://garnaat.org/ 2 # Copyright 2010 Google Inc. 3 # Copyright (c) 2010, Eucalyptus Systems, Inc. 4 # Copyright (c) 2011, Nexenta Systems Inc. 5 # All rights reserved. 6 # 7 # Permission is hereby granted, free of charge, to any person obtaining a 8 # copy of this software and associated documentation files (the 9 # "Software"), to deal in the Software without restriction, including 10 # without limitation the rights to use, copy, modify, merge, publish, dis- 11 # tribute, sublicense, and/or sell copies of the Software, and to permit 12 # persons to whom the Software is furnished to do so, subject to the fol- 13 # lowing conditions: 14 # 15 # The above copyright notice and this permission notice shall be included 16 # in all copies or substantial portions of the Software. 17 # 18 # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS 19 # OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- 20 # ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT 21 # SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, 22 # WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 23 # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS 24 # IN THE SOFTWARE. 25 """ 26 This class encapsulates the provider-specific header differences. 27 """ 28 29 import os 30 from boto.compat import six 31 from datetime import datetime 32 33 import boto 34 from boto import config 35 from boto.compat import expanduser 36 from boto.pyami.config import Config 37 from boto.gs.acl import ACL 38 from boto.gs.acl import CannedACLStrings as CannedGSACLStrings 39 from boto.s3.acl import CannedACLStrings as CannedS3ACLStrings 40 from boto.s3.acl import Policy 41 42 43 HEADER_PREFIX_KEY = 'header_prefix' 44 METADATA_PREFIX_KEY = 'metadata_prefix' 45 46 AWS_HEADER_PREFIX = 'x-amz-' 47 GOOG_HEADER_PREFIX = 'x-goog-' 48 49 ACL_HEADER_KEY = 'acl-header' 50 AUTH_HEADER_KEY = 'auth-header' 51 COPY_SOURCE_HEADER_KEY = 'copy-source-header' 52 COPY_SOURCE_VERSION_ID_HEADER_KEY = 'copy-source-version-id-header' 53 COPY_SOURCE_RANGE_HEADER_KEY = 'copy-source-range-header' 54 DELETE_MARKER_HEADER_KEY = 'delete-marker-header' 55 DATE_HEADER_KEY = 'date-header' 56 METADATA_DIRECTIVE_HEADER_KEY = 'metadata-directive-header' 57 RESUMABLE_UPLOAD_HEADER_KEY = 'resumable-upload-header' 58 SECURITY_TOKEN_HEADER_KEY = 'security-token-header' 59 STORAGE_CLASS_HEADER_KEY = 'storage-class' 60 MFA_HEADER_KEY = 'mfa-header' 61 SERVER_SIDE_ENCRYPTION_KEY = 'server-side-encryption-header' 62 VERSION_ID_HEADER_KEY = 'version-id-header' 63 RESTORE_HEADER_KEY = 'restore-header' 64 65 STORAGE_COPY_ERROR = 'StorageCopyError' 66 STORAGE_CREATE_ERROR = 'StorageCreateError' 67 STORAGE_DATA_ERROR = 'StorageDataError' 68 STORAGE_PERMISSIONS_ERROR = 'StoragePermissionsError' 69 STORAGE_RESPONSE_ERROR = 'StorageResponseError' 70 NO_CREDENTIALS_PROVIDED = object() 71 72 73 class ProfileNotFoundError(ValueError): 74 pass 75 76 77 class Provider(object): 78 79 CredentialMap = { 80 'aws': ('aws_access_key_id', 'aws_secret_access_key', 81 'aws_security_token', 'aws_profile'), 82 'google': ('gs_access_key_id', 'gs_secret_access_key', 83 None, None), 84 } 85 86 AclClassMap = { 87 'aws': Policy, 88 'google': ACL 89 } 90 91 CannedAclsMap = { 92 'aws': CannedS3ACLStrings, 93 'google': CannedGSACLStrings 94 } 95 96 HostKeyMap = { 97 'aws': 's3', 98 'google': 'gs' 99 } 100 101 ChunkedTransferSupport = { 102 'aws': False, 103 'google': True 104 } 105 106 MetadataServiceSupport = { 107 'aws': True, 108 'google': False 109 } 110 111 # If you update this map please make sure to put "None" for the 112 # right-hand-side for any headers that don't apply to a provider, rather 113 # than simply leaving that header out (which would cause KeyErrors). 114 HeaderInfoMap = { 115 'aws': { 116 HEADER_PREFIX_KEY: AWS_HEADER_PREFIX, 117 METADATA_PREFIX_KEY: AWS_HEADER_PREFIX + 'meta-', 118 ACL_HEADER_KEY: AWS_HEADER_PREFIX + 'acl', 119 AUTH_HEADER_KEY: 'AWS', 120 COPY_SOURCE_HEADER_KEY: AWS_HEADER_PREFIX + 'copy-source', 121 COPY_SOURCE_VERSION_ID_HEADER_KEY: AWS_HEADER_PREFIX + 122 'copy-source-version-id', 123 COPY_SOURCE_RANGE_HEADER_KEY: AWS_HEADER_PREFIX + 124 'copy-source-range', 125 DATE_HEADER_KEY: AWS_HEADER_PREFIX + 'date', 126 DELETE_MARKER_HEADER_KEY: AWS_HEADER_PREFIX + 'delete-marker', 127 METADATA_DIRECTIVE_HEADER_KEY: AWS_HEADER_PREFIX + 128 'metadata-directive', 129 RESUMABLE_UPLOAD_HEADER_KEY: None, 130 SECURITY_TOKEN_HEADER_KEY: AWS_HEADER_PREFIX + 'security-token', 131 SERVER_SIDE_ENCRYPTION_KEY: AWS_HEADER_PREFIX + 132 'server-side-encryption', 133 VERSION_ID_HEADER_KEY: AWS_HEADER_PREFIX + 'version-id', 134 STORAGE_CLASS_HEADER_KEY: AWS_HEADER_PREFIX + 'storage-class', 135 MFA_HEADER_KEY: AWS_HEADER_PREFIX + 'mfa', 136 RESTORE_HEADER_KEY: AWS_HEADER_PREFIX + 'restore', 137 }, 138 'google': { 139 HEADER_PREFIX_KEY: GOOG_HEADER_PREFIX, 140 METADATA_PREFIX_KEY: GOOG_HEADER_PREFIX + 'meta-', 141 ACL_HEADER_KEY: GOOG_HEADER_PREFIX + 'acl', 142 AUTH_HEADER_KEY: 'GOOG1', 143 COPY_SOURCE_HEADER_KEY: GOOG_HEADER_PREFIX + 'copy-source', 144 COPY_SOURCE_VERSION_ID_HEADER_KEY: GOOG_HEADER_PREFIX + 145 'copy-source-version-id', 146 COPY_SOURCE_RANGE_HEADER_KEY: None, 147 DATE_HEADER_KEY: GOOG_HEADER_PREFIX + 'date', 148 DELETE_MARKER_HEADER_KEY: GOOG_HEADER_PREFIX + 'delete-marker', 149 METADATA_DIRECTIVE_HEADER_KEY: GOOG_HEADER_PREFIX + 150 'metadata-directive', 151 RESUMABLE_UPLOAD_HEADER_KEY: GOOG_HEADER_PREFIX + 'resumable', 152 SECURITY_TOKEN_HEADER_KEY: GOOG_HEADER_PREFIX + 'security-token', 153 SERVER_SIDE_ENCRYPTION_KEY: None, 154 # Note that this version header is not to be confused with 155 # the Google Cloud Storage 'x-goog-api-version' header. 156 VERSION_ID_HEADER_KEY: GOOG_HEADER_PREFIX + 'version-id', 157 STORAGE_CLASS_HEADER_KEY: None, 158 MFA_HEADER_KEY: None, 159 RESTORE_HEADER_KEY: None, 160 } 161 } 162 163 ErrorMap = { 164 'aws': { 165 STORAGE_COPY_ERROR: boto.exception.S3CopyError, 166 STORAGE_CREATE_ERROR: boto.exception.S3CreateError, 167 STORAGE_DATA_ERROR: boto.exception.S3DataError, 168 STORAGE_PERMISSIONS_ERROR: boto.exception.S3PermissionsError, 169 STORAGE_RESPONSE_ERROR: boto.exception.S3ResponseError, 170 }, 171 'google': { 172 STORAGE_COPY_ERROR: boto.exception.GSCopyError, 173 STORAGE_CREATE_ERROR: boto.exception.GSCreateError, 174 STORAGE_DATA_ERROR: boto.exception.GSDataError, 175 STORAGE_PERMISSIONS_ERROR: boto.exception.GSPermissionsError, 176 STORAGE_RESPONSE_ERROR: boto.exception.GSResponseError, 177 } 178 } 179 180 def __init__(self, name, access_key=None, secret_key=None, 181 security_token=None, profile_name=None): 182 self.host = None 183 self.port = None 184 self.host_header = None 185 self.access_key = access_key 186 self.secret_key = secret_key 187 self.security_token = security_token 188 self.profile_name = profile_name 189 self.name = name 190 self.acl_class = self.AclClassMap[self.name] 191 self.canned_acls = self.CannedAclsMap[self.name] 192 self._credential_expiry_time = None 193 194 # Load shared credentials file if it exists 195 shared_path = os.path.join(expanduser('~'), '.' + name, 'credentials') 196 self.shared_credentials = Config(do_load=False) 197 if os.path.isfile(shared_path): 198 self.shared_credentials.load_from_path(shared_path) 199 200 self.get_credentials(access_key, secret_key, security_token, profile_name) 201 self.configure_headers() 202 self.configure_errors() 203 204 # Allow config file to override default host and port. 205 host_opt_name = '%s_host' % self.HostKeyMap[self.name] 206 if config.has_option('Credentials', host_opt_name): 207 self.host = config.get('Credentials', host_opt_name) 208 port_opt_name = '%s_port' % self.HostKeyMap[self.name] 209 if config.has_option('Credentials', port_opt_name): 210 self.port = config.getint('Credentials', port_opt_name) 211 host_header_opt_name = '%s_host_header' % self.HostKeyMap[self.name] 212 if config.has_option('Credentials', host_header_opt_name): 213 self.host_header = config.get('Credentials', host_header_opt_name) 214 215 def get_access_key(self): 216 if self._credentials_need_refresh(): 217 self._populate_keys_from_metadata_server() 218 return self._access_key 219 220 def set_access_key(self, value): 221 self._access_key = value 222 223 access_key = property(get_access_key, set_access_key) 224 225 def get_secret_key(self): 226 if self._credentials_need_refresh(): 227 self._populate_keys_from_metadata_server() 228 return self._secret_key 229 230 def set_secret_key(self, value): 231 self._secret_key = value 232 233 secret_key = property(get_secret_key, set_secret_key) 234 235 def get_security_token(self): 236 if self._credentials_need_refresh(): 237 self._populate_keys_from_metadata_server() 238 return self._security_token 239 240 def set_security_token(self, value): 241 self._security_token = value 242 243 security_token = property(get_security_token, set_security_token) 244 245 def _credentials_need_refresh(self): 246 if self._credential_expiry_time is None: 247 return False 248 else: 249 # The credentials should be refreshed if they're going to expire 250 # in less than 5 minutes. 251 delta = self._credential_expiry_time - datetime.utcnow() 252 # python2.6 does not have timedelta.total_seconds() so we have 253 # to calculate this ourselves. This is straight from the 254 # datetime docs. 255 seconds_left = ( 256 (delta.microseconds + (delta.seconds + delta.days * 24 * 3600) 257 * 10 ** 6) / 10 ** 6) 258 if seconds_left < (5 * 60): 259 boto.log.debug("Credentials need to be refreshed.") 260 return True 261 else: 262 return False 263 264 def get_credentials(self, access_key=None, secret_key=None, 265 security_token=None, profile_name=None): 266 access_key_name, secret_key_name, security_token_name, \ 267 profile_name_name = self.CredentialMap[self.name] 268 269 # Load profile from shared environment variable if it was not 270 # already passed in and the environment variable exists 271 if profile_name is None and profile_name_name is not None and \ 272 profile_name_name.upper() in os.environ: 273 profile_name = os.environ[profile_name_name.upper()] 274 275 shared = self.shared_credentials 276 277 if access_key is not None: 278 self.access_key = access_key 279 boto.log.debug("Using access key provided by client.") 280 elif access_key_name.upper() in os.environ: 281 self.access_key = os.environ[access_key_name.upper()] 282 boto.log.debug("Using access key found in environment variable.") 283 elif profile_name is not None: 284 if shared.has_option(profile_name, access_key_name): 285 self.access_key = shared.get(profile_name, access_key_name) 286 boto.log.debug("Using access key found in shared credential " 287 "file for profile %s." % profile_name) 288 elif config.has_option("profile %s" % profile_name, 289 access_key_name): 290 self.access_key = config.get("profile %s" % profile_name, 291 access_key_name) 292 boto.log.debug("Using access key found in config file: " 293 "profile %s." % profile_name) 294 else: 295 raise ProfileNotFoundError('Profile "%s" not found!' % 296 profile_name) 297 elif shared.has_option('default', access_key_name): 298 self.access_key = shared.get('default', access_key_name) 299 boto.log.debug("Using access key found in shared credential file.") 300 elif config.has_option('Credentials', access_key_name): 301 self.access_key = config.get('Credentials', access_key_name) 302 boto.log.debug("Using access key found in config file.") 303 304 if secret_key is not None: 305 self.secret_key = secret_key 306 boto.log.debug("Using secret key provided by client.") 307 elif secret_key_name.upper() in os.environ: 308 self.secret_key = os.environ[secret_key_name.upper()] 309 boto.log.debug("Using secret key found in environment variable.") 310 elif profile_name is not None: 311 if shared.has_option(profile_name, secret_key_name): 312 self.secret_key = shared.get(profile_name, secret_key_name) 313 boto.log.debug("Using secret key found in shared credential " 314 "file for profile %s." % profile_name) 315 elif config.has_option("profile %s" % profile_name, secret_key_name): 316 self.secret_key = config.get("profile %s" % profile_name, 317 secret_key_name) 318 boto.log.debug("Using secret key found in config file: " 319 "profile %s." % profile_name) 320 else: 321 raise ProfileNotFoundError('Profile "%s" not found!' % 322 profile_name) 323 elif shared.has_option('default', secret_key_name): 324 self.secret_key = shared.get('default', secret_key_name) 325 boto.log.debug("Using secret key found in shared credential file.") 326 elif config.has_option('Credentials', secret_key_name): 327 self.secret_key = config.get('Credentials', secret_key_name) 328 boto.log.debug("Using secret key found in config file.") 329 elif config.has_option('Credentials', 'keyring'): 330 keyring_name = config.get('Credentials', 'keyring') 331 try: 332 import keyring 333 except ImportError: 334 boto.log.error("The keyring module could not be imported. " 335 "For keyring support, install the keyring " 336 "module.") 337 raise 338 self.secret_key = keyring.get_password( 339 keyring_name, self.access_key) 340 boto.log.debug("Using secret key found in keyring.") 341 342 if security_token is not None: 343 self.security_token = security_token 344 boto.log.debug("Using security token provided by client.") 345 elif ((security_token_name is not None) and 346 (access_key is None) and (secret_key is None)): 347 # Only provide a token from the environment/config if the 348 # caller did not specify a key and secret. Otherwise an 349 # environment/config token could be paired with a 350 # different set of credentials provided by the caller 351 if security_token_name.upper() in os.environ: 352 self.security_token = os.environ[security_token_name.upper()] 353 boto.log.debug("Using security token found in environment" 354 " variable.") 355 elif shared.has_option(profile_name or 'default', 356 security_token_name): 357 self.security_token = shared.get(profile_name or 'default', 358 security_token_name) 359 boto.log.debug("Using security token found in shared " 360 "credential file.") 361 elif profile_name is not None: 362 if config.has_option("profile %s" % profile_name, 363 security_token_name): 364 boto.log.debug("config has option") 365 self.security_token = config.get("profile %s" % profile_name, 366 security_token_name) 367 boto.log.debug("Using security token found in config file: " 368 "profile %s." % profile_name) 369 elif config.has_option('Credentials', security_token_name): 370 self.security_token = config.get('Credentials', 371 security_token_name) 372 boto.log.debug("Using security token found in config file.") 373 374 if ((self._access_key is None or self._secret_key is None) and 375 self.MetadataServiceSupport[self.name]): 376 self._populate_keys_from_metadata_server() 377 self._secret_key = self._convert_key_to_str(self._secret_key) 378 379 def _populate_keys_from_metadata_server(self): 380 # get_instance_metadata is imported here because of a circular 381 # dependency. 382 boto.log.debug("Retrieving credentials from metadata server.") 383 from boto.utils import get_instance_metadata 384 timeout = config.getfloat('Boto', 'metadata_service_timeout', 1.0) 385 attempts = config.getint('Boto', 'metadata_service_num_attempts', 1) 386 # The num_retries arg is actually the total number of attempts made, 387 # so the config options is named *_num_attempts to make this more 388 # clear to users. 389 metadata = get_instance_metadata( 390 timeout=timeout, num_retries=attempts, 391 data='meta-data/iam/security-credentials/') 392 if metadata: 393 # I'm assuming there's only one role on the instance profile. 394 security = list(metadata.values())[0] 395 self._access_key = security['AccessKeyId'] 396 self._secret_key = self._convert_key_to_str(security['SecretAccessKey']) 397 self._security_token = security['Token'] 398 expires_at = security['Expiration'] 399 self._credential_expiry_time = datetime.strptime( 400 expires_at, "%Y-%m-%dT%H:%M:%SZ") 401 boto.log.debug("Retrieved credentials will expire in %s at: %s", 402 self._credential_expiry_time - datetime.now(), expires_at) 403 404 def _convert_key_to_str(self, key): 405 if isinstance(key, six.text_type): 406 # the secret key must be bytes and not unicode to work 407 # properly with hmac.new (see http://bugs.python.org/issue5285) 408 return str(key) 409 return key 410 411 def configure_headers(self): 412 header_info_map = self.HeaderInfoMap[self.name] 413 self.metadata_prefix = header_info_map[METADATA_PREFIX_KEY] 414 self.header_prefix = header_info_map[HEADER_PREFIX_KEY] 415 self.acl_header = header_info_map[ACL_HEADER_KEY] 416 self.auth_header = header_info_map[AUTH_HEADER_KEY] 417 self.copy_source_header = header_info_map[COPY_SOURCE_HEADER_KEY] 418 self.copy_source_version_id = header_info_map[ 419 COPY_SOURCE_VERSION_ID_HEADER_KEY] 420 self.copy_source_range_header = header_info_map[ 421 COPY_SOURCE_RANGE_HEADER_KEY] 422 self.date_header = header_info_map[DATE_HEADER_KEY] 423 self.delete_marker = header_info_map[DELETE_MARKER_HEADER_KEY] 424 self.metadata_directive_header = ( 425 header_info_map[METADATA_DIRECTIVE_HEADER_KEY]) 426 self.security_token_header = header_info_map[SECURITY_TOKEN_HEADER_KEY] 427 self.resumable_upload_header = ( 428 header_info_map[RESUMABLE_UPLOAD_HEADER_KEY]) 429 self.server_side_encryption_header = header_info_map[SERVER_SIDE_ENCRYPTION_KEY] 430 self.storage_class_header = header_info_map[STORAGE_CLASS_HEADER_KEY] 431 self.version_id = header_info_map[VERSION_ID_HEADER_KEY] 432 self.mfa_header = header_info_map[MFA_HEADER_KEY] 433 self.restore_header = header_info_map[RESTORE_HEADER_KEY] 434 435 def configure_errors(self): 436 error_map = self.ErrorMap[self.name] 437 self.storage_copy_error = error_map[STORAGE_COPY_ERROR] 438 self.storage_create_error = error_map[STORAGE_CREATE_ERROR] 439 self.storage_data_error = error_map[STORAGE_DATA_ERROR] 440 self.storage_permissions_error = error_map[STORAGE_PERMISSIONS_ERROR] 441 self.storage_response_error = error_map[STORAGE_RESPONSE_ERROR] 442 443 def get_provider_name(self): 444 return self.HostKeyMap[self.name] 445 446 def supports_chunked_transfer(self): 447 return self.ChunkedTransferSupport[self.name] 448 449 450 # Static utility method for getting default Provider. 451 def get_default(): 452 return Provider('aws') 453
