1 # -*- coding: utf-8 -*- 2 # Copyright 2014 Google Inc. All Rights Reserved. 3 # 4 # Licensed under the Apache License, Version 2.0 (the "License"); 5 # you may not use this file except in compliance with the License. 6 # You may obtain a copy of the License at 7 # 8 # http://www.apache.org/licenses/LICENSE-2.0 9 # 10 # Unless required by applicable law or agreed to in writing, software 11 # distributed under the License is distributed on an "AS IS" BASIS, 12 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 # See the License for the specific language governing permissions and 14 # limitations under the License. 15 """Utility module for translating XML API objects to/from JSON objects.""" 16 17 from __future__ import absolute_import 18 19 import datetime 20 import json 21 import re 22 import textwrap 23 import xml.etree.ElementTree 24 25 from apitools.base.py import encoding 26 import boto 27 from boto.gs.acl import ACL 28 from boto.gs.acl import ALL_AUTHENTICATED_USERS 29 from boto.gs.acl import ALL_USERS 30 from boto.gs.acl import Entries 31 from boto.gs.acl import Entry 32 from boto.gs.acl import GROUP_BY_DOMAIN 33 from boto.gs.acl import GROUP_BY_EMAIL 34 from boto.gs.acl import GROUP_BY_ID 35 from boto.gs.acl import USER_BY_EMAIL 36 from boto.gs.acl import USER_BY_ID 37 38 from gslib.cloud_api import ArgumentException 39 from gslib.cloud_api import BucketNotFoundException 40 from gslib.cloud_api import NotFoundException 41 from gslib.cloud_api import Preconditions 42 from gslib.exception import CommandException 43 from gslib.third_party.storage_apitools import storage_v1_messages as apitools_messages 44 45 # In Python 2.6, ElementTree raises ExpatError instead of ParseError. 46 # pylint: disable=g-import-not-at-top 47 try: 48 from xml.etree.ElementTree import ParseError as XmlParseError 49 except ImportError: 50 from xml.parsers.expat import ExpatError as XmlParseError 51 52 CACHE_CONTROL_REGEX = re.compile(r'^cache-control', re.I) 53 CONTENT_DISPOSITION_REGEX = re.compile(r'^content-disposition', re.I) 54 CONTENT_ENCODING_REGEX = re.compile(r'^content-encoding', re.I) 55 CONTENT_LANGUAGE_REGEX = re.compile(r'^content-language', re.I) 56 CONTENT_MD5_REGEX = re.compile(r'^content-md5', re.I) 57 CONTENT_TYPE_REGEX = re.compile(r'^content-type', re.I) 58 GOOG_API_VERSION_REGEX = re.compile(r'^x-goog-api-version', re.I) 59 GOOG_GENERATION_MATCH_REGEX = re.compile(r'^x-goog-if-generation-match', re.I) 60 GOOG_METAGENERATION_MATCH_REGEX = re.compile( 61 r'^x-goog-if-metageneration-match', re.I) 62 CUSTOM_GOOG_METADATA_REGEX = re.compile(r'^x-goog-meta-(?P<header_key>.*)', 63 re.I) 64 CUSTOM_AMZ_METADATA_REGEX = re.compile(r'^x-amz-meta-(?P<header_key>.*)', re.I) 65 CUSTOM_AMZ_HEADER_REGEX = re.compile(r'^x-amz-(?P<header_key>.*)', re.I) 66 67 # gsutil-specific GUIDs for marking special metadata for S3 compatibility. 68 S3_ACL_MARKER_GUID = '3b89a6b5-b55a-4900-8c44-0b0a2f5eab43-s3-AclMarker' 69 S3_DELETE_MARKER_GUID = 'eadeeee8-fa8c-49bb-8a7d-0362215932d8-s3-DeleteMarker' 70 S3_MARKER_GUIDS = [S3_ACL_MARKER_GUID, S3_DELETE_MARKER_GUID] 71 # This distinguishes S3 custom headers from S3 metadata on objects. 72 S3_HEADER_PREFIX = 'custom-amz-header' 73 74 DEFAULT_CONTENT_TYPE = 'application/octet-stream' 75 76 # Because CORS is just a list in apitools, we need special handling or blank 77 # CORS lists will get sent with other configuration commands such as lifecycle, 78 # which would cause CORS configuration to be unintentionally removed. 79 # Protorpc defaults list values to an empty list, and won't allow us to set the 80 # value to None like other configuration fields, so there is no way to 81 # distinguish the default value from when we actually want to remove the CORS 82 # configuration. To work around this, we create a dummy CORS entry that 83 # signifies that we should nullify the CORS configuration. 84 # A value of [] means don't modify the CORS configuration. 85 # A value of REMOVE_CORS_CONFIG means remove the CORS configuration. 86 REMOVE_CORS_CONFIG = [apitools_messages.Bucket.CorsValueListEntry( 87 maxAgeSeconds=-1, method=['REMOVE_CORS_CONFIG'])] 88 89 # Similar to CORS above, we need a sentinel value allowing us to specify 90 # when a default object ACL should be private (containing no entries). 91 # A defaultObjectAcl value of [] means don't modify the default object ACL. 92 # A value of [PRIVATE_DEFAULT_OBJ_ACL] means create an empty/private default 93 # object ACL. 94 PRIVATE_DEFAULT_OBJ_ACL = apitools_messages.ObjectAccessControl( 95 id='PRIVATE_DEFAULT_OBJ_ACL') 96 97 98 def ObjectMetadataFromHeaders(headers): 99 """Creates object metadata according to the provided headers. 100 101 gsutil -h allows specifiying various headers (originally intended 102 to be passed to boto in gsutil v3). For the JSON API to be compatible with 103 this option, we need to parse these headers into gsutil_api Object fields. 104 105 Args: 106 headers: Dict of headers passed via gsutil -h 107 108 Raises: 109 ArgumentException if an invalid header is encountered. 110 111 Returns: 112 apitools Object with relevant fields populated from headers. 113 """ 114 obj_metadata = apitools_messages.Object() 115 for header, value in headers.items(): 116 if CACHE_CONTROL_REGEX.match(header): 117 obj_metadata.cacheControl = value.strip() 118 elif CONTENT_DISPOSITION_REGEX.match(header): 119 obj_metadata.contentDisposition = value.strip() 120 elif CONTENT_ENCODING_REGEX.match(header): 121 obj_metadata.contentEncoding = value.strip() 122 elif CONTENT_MD5_REGEX.match(header): 123 obj_metadata.md5Hash = value.strip() 124 elif CONTENT_LANGUAGE_REGEX.match(header): 125 obj_metadata.contentLanguage = value.strip() 126 elif CONTENT_TYPE_REGEX.match(header): 127 if not value: 128 obj_metadata.contentType = DEFAULT_CONTENT_TYPE 129 else: 130 obj_metadata.contentType = value.strip() 131 elif GOOG_API_VERSION_REGEX.match(header): 132 # API version is only relevant for XML, ignore and rely on the XML API 133 # to add the appropriate version. 134 continue 135 elif GOOG_GENERATION_MATCH_REGEX.match(header): 136 # Preconditions are handled elsewhere, but allow these headers through. 137 continue 138 elif GOOG_METAGENERATION_MATCH_REGEX.match(header): 139 # Preconditions are handled elsewhere, but allow these headers through. 140 continue 141 else: 142 custom_goog_metadata_match = CUSTOM_GOOG_METADATA_REGEX.match(header) 143 custom_amz_metadata_match = CUSTOM_AMZ_METADATA_REGEX.match(header) 144 custom_amz_header_match = CUSTOM_AMZ_HEADER_REGEX.match(header) 145 header_key = None 146 if custom_goog_metadata_match: 147 header_key = custom_goog_metadata_match.group('header_key') 148 elif custom_amz_metadata_match: 149 header_key = custom_amz_metadata_match.group('header_key') 150 elif custom_amz_header_match: 151 # If we got here we are guaranteed by the prior statement that this is 152 # not an x-amz-meta- header. 153 header_key = (S3_HEADER_PREFIX + 154 custom_amz_header_match.group('header_key')) 155 if header_key: 156 if header_key.lower() == 'x-goog-content-language': 157 # Work around content-language being inserted into custom metadata. 158 continue 159 if not obj_metadata.metadata: 160 obj_metadata.metadata = apitools_messages.Object.MetadataValue() 161 if not obj_metadata.metadata.additionalProperties: 162 obj_metadata.metadata.additionalProperties = [] 163 obj_metadata.metadata.additionalProperties.append( 164 apitools_messages.Object.MetadataValue.AdditionalProperty( 165 key=header_key, value=value)) 166 else: 167 raise ArgumentException( 168 'Invalid header specifed: %s:%s' % (header, value)) 169 return obj_metadata 170 171 172 def HeadersFromObjectMetadata(dst_obj_metadata, provider): 173 """Creates a header dictionary based on existing object metadata. 174 175 Args: 176 dst_obj_metadata: Object metadata to create the headers from. 177 provider: Provider string ('gs' or 's3') 178 179 Returns: 180 Headers dictionary. 181 """ 182 headers = {} 183 if not dst_obj_metadata: 184 return 185 # Metadata values of '' mean suppress/remove this header. 186 if dst_obj_metadata.cacheControl is not None: 187 if not dst_obj_metadata.cacheControl: 188 headers['cache-control'] = None 189 else: 190 headers['cache-control'] = dst_obj_metadata.cacheControl.strip() 191 if dst_obj_metadata.contentDisposition: 192 if not dst_obj_metadata.contentDisposition: 193 headers['content-disposition'] = None 194 else: 195 headers['content-disposition'] = ( 196 dst_obj_metadata.contentDisposition.strip()) 197 if dst_obj_metadata.contentEncoding: 198 if not dst_obj_metadata.contentEncoding: 199 headers['content-encoding'] = None 200 else: 201 headers['content-encoding'] = dst_obj_metadata.contentEncoding.strip() 202 if dst_obj_metadata.contentLanguage: 203 if not dst_obj_metadata.contentLanguage: 204 headers['content-language'] = None 205 else: 206 headers['content-language'] = dst_obj_metadata.contentLanguage.strip() 207 if dst_obj_metadata.md5Hash: 208 if not dst_obj_metadata.md5Hash: 209 headers['Content-MD5'] = None 210 else: 211 headers['Content-MD5'] = dst_obj_metadata.md5Hash.strip() 212 if dst_obj_metadata.contentType is not None: 213 if not dst_obj_metadata.contentType: 214 headers['content-type'] = None 215 else: 216 headers['content-type'] = dst_obj_metadata.contentType.strip() 217 if (dst_obj_metadata.metadata and 218 dst_obj_metadata.metadata.additionalProperties): 219 for additional_property in dst_obj_metadata.metadata.additionalProperties: 220 # Work around content-language being inserted into custom metadata by 221 # the XML API. 222 if additional_property.key == 'content-language': 223 continue 224 # Don't translate special metadata markers. 225 if additional_property.key in S3_MARKER_GUIDS: 226 continue 227 if provider == 'gs': 228 header_name = 'x-goog-meta-' + additional_property.key 229 elif provider == 's3': 230 if additional_property.key.startswith(S3_HEADER_PREFIX): 231 header_name = ('x-amz-' + 232 additional_property.key[len(S3_HEADER_PREFIX):]) 233 else: 234 header_name = 'x-amz-meta-' + additional_property.key 235 else: 236 raise ArgumentException('Invalid provider specified: %s' % provider) 237 if (additional_property.value is not None and 238 not additional_property.value): 239 headers[header_name] = None 240 else: 241 headers[header_name] = additional_property.value 242 return headers 243 244 245 def CopyObjectMetadata(src_obj_metadata, dst_obj_metadata, override=False): 246 """Copies metadata from src_obj_metadata to dst_obj_metadata. 247 248 Args: 249 src_obj_metadata: Metadata from source object 250 dst_obj_metadata: Initialized metadata for destination object 251 override: If true, will overwrite metadata in destination object. 252 If false, only writes metadata for values that don't already 253 exist. 254 """ 255 if override or not dst_obj_metadata.cacheControl: 256 dst_obj_metadata.cacheControl = src_obj_metadata.cacheControl 257 if override or not dst_obj_metadata.contentDisposition: 258 dst_obj_metadata.contentDisposition = src_obj_metadata.contentDisposition 259 if override or not dst_obj_metadata.contentEncoding: 260 dst_obj_metadata.contentEncoding = src_obj_metadata.contentEncoding 261 if override or not dst_obj_metadata.contentLanguage: 262 dst_obj_metadata.contentLanguage = src_obj_metadata.contentLanguage 263 if override or not dst_obj_metadata.contentType: 264 dst_obj_metadata.contentType = src_obj_metadata.contentType 265 if override or not dst_obj_metadata.md5Hash: 266 dst_obj_metadata.md5Hash = src_obj_metadata.md5Hash 267 268 # TODO: Apitools should ideally treat metadata like a real dictionary instead 269 # of a list of key/value pairs (with an O(N^2) lookup). In practice the 270 # number of values is typically small enough not to matter. 271 # Work around this by creating our own dictionary. 272 if (src_obj_metadata.metadata and 273 src_obj_metadata.metadata.additionalProperties): 274 if not dst_obj_metadata.metadata: 275 dst_obj_metadata.metadata = apitools_messages.Object.MetadataValue() 276 if not dst_obj_metadata.metadata.additionalProperties: 277 dst_obj_metadata.metadata.additionalProperties = [] 278 dst_metadata_dict = {} 279 for dst_prop in dst_obj_metadata.metadata.additionalProperties: 280 dst_metadata_dict[dst_prop.key] = dst_prop.value 281 for src_prop in src_obj_metadata.metadata.additionalProperties: 282 if src_prop.key in dst_metadata_dict: 283 if override: 284 # Metadata values of '' mean suppress/remove this header. 285 if src_prop.value is not None and not src_prop.value: 286 dst_metadata_dict[src_prop.key] = None 287 else: 288 dst_metadata_dict[src_prop.key] = src_prop.value 289 else: 290 dst_metadata_dict[src_prop.key] = src_prop.value 291 # Rewrite the list with our updated dict. 292 dst_obj_metadata.metadata.additionalProperties = [] 293 for k, v in dst_metadata_dict.iteritems(): 294 dst_obj_metadata.metadata.additionalProperties.append( 295 apitools_messages.Object.MetadataValue.AdditionalProperty(key=k, 296 value=v)) 297 298 299 def PreconditionsFromHeaders(headers): 300 """Creates bucket or object preconditions acccording to the provided headers. 301 302 Args: 303 headers: Dict of headers passed via gsutil -h 304 305 Returns: 306 gsutil Cloud API Preconditions object fields populated from headers, or None 307 if no precondition headers are present. 308 """ 309 return_preconditions = Preconditions() 310 try: 311 for header, value in headers.items(): 312 if GOOG_GENERATION_MATCH_REGEX.match(header): 313 return_preconditions.gen_match = long(value) 314 if GOOG_METAGENERATION_MATCH_REGEX.match(header): 315 return_preconditions.meta_gen_match = long(value) 316 except ValueError, _: 317 raise ArgumentException('Invalid precondition header specified. ' 318 'x-goog-if-generation-match and ' 319 'x-goog-if-metageneration match must be specified ' 320 'with a positive integer value.') 321 return return_preconditions 322 323 324 def CreateNotFoundExceptionForObjectWrite( 325 dst_provider, dst_bucket_name, src_provider=None, 326 src_bucket_name=None, src_object_name=None, src_generation=None): 327 """Creates a NotFoundException for an object upload or copy. 328 329 This is necessary because 404s don't necessarily specify which resource 330 does not exist. 331 332 Args: 333 dst_provider: String abbreviation of destination provider, e.g., 'gs'. 334 dst_bucket_name: Destination bucket name for the write operation. 335 src_provider: String abbreviation of source provider, i.e. 'gs', if any. 336 src_bucket_name: Source bucket name, if any (for the copy case). 337 src_object_name: Source object name, if any (for the copy case). 338 src_generation: Source object generation, if any (for the copy case). 339 340 Returns: 341 NotFoundException with appropriate message. 342 """ 343 dst_url_string = '%s://%s' % (dst_provider, dst_bucket_name) 344 if src_bucket_name and src_object_name: 345 src_url_string = '%s://%s/%s' % (src_provider, src_bucket_name, 346 src_object_name) 347 if src_generation: 348 src_url_string += '#%s' % str(src_generation) 349 return NotFoundException( 350 'The source object %s or the destination bucket %s does not exist.' % 351 (src_url_string, dst_url_string)) 352 353 return NotFoundException( 354 'The destination bucket %s does not exist or the write to the ' 355 'destination must be restarted' % dst_url_string) 356 357 358 def CreateBucketNotFoundException(code, provider, bucket_name): 359 return BucketNotFoundException('%s://%s bucket does not exist.' % 360 (provider, bucket_name), bucket_name, 361 status=code) 362 363 364 def CreateObjectNotFoundException(code, provider, bucket_name, object_name, 365 generation=None): 366 uri_string = '%s://%s/%s' % (provider, bucket_name, object_name) 367 if generation: 368 uri_string += '#%s' % str(generation) 369 return NotFoundException('%s does not exist.' % uri_string, status=code) 370 371 372 def EncodeStringAsLong(string_to_convert): 373 """Encodes an ASCII string as a python long. 374 375 This is used for modeling S3 version_id's as apitools generation. Because 376 python longs can be arbitrarily large, this works. 377 378 Args: 379 string_to_convert: ASCII string to convert to a long. 380 381 Returns: 382 Long that represents the input string. 383 """ 384 return long(string_to_convert.encode('hex'), 16) 385 386 387 def _DecodeLongAsString(long_to_convert): 388 """Decodes an encoded python long into an ASCII string. 389 390 This is used for modeling S3 version_id's as apitools generation. 391 392 Args: 393 long_to_convert: long to convert to ASCII string. If this is already a 394 string, it is simply returned. 395 396 Returns: 397 String decoded from the input long. 398 """ 399 if isinstance(long_to_convert, basestring): 400 # Already converted. 401 return long_to_convert 402 return hex(long_to_convert)[2:-1].decode('hex') 403 404 405 def GenerationFromUrlAndString(url, generation): 406 """Decodes a generation from a StorageURL and a generation string. 407 408 This is used to represent gs and s3 versioning. 409 410 Args: 411 url: StorageUrl representing the object. 412 generation: Long or string representing the object's generation or 413 version. 414 415 Returns: 416 Valid generation string for use in URLs. 417 """ 418 if url.scheme == 's3' and generation: 419 return _DecodeLongAsString(generation) 420 return generation 421 422 423 def CheckForXmlConfigurationAndRaise(config_type_string, json_txt): 424 """Checks a JSON parse exception for provided XML configuration.""" 425 try: 426 xml.etree.ElementTree.fromstring(str(json_txt)) 427 raise ArgumentException('\n'.join(textwrap.wrap( 428 'XML {0} data provided; Google Cloud Storage {0} configuration ' 429 'now uses JSON format. To convert your {0}, set the desired XML ' 430 'ACL using \'gsutil {1} set ...\' with gsutil version 3.x. Then ' 431 'use \'gsutil {1} get ...\' with gsutil version 4 or greater to ' 432 'get the corresponding JSON {0}.'.format(config_type_string, 433 config_type_string.lower())))) 434 except XmlParseError: 435 pass 436 raise ArgumentException('JSON %s data could not be loaded ' 437 'from: %s' % (config_type_string, json_txt)) 438 439 440 class LifecycleTranslation(object): 441 """Functions for converting between various lifecycle formats. 442 443 This class handles conversation to and from Boto Cors objects, JSON text, 444 and apitools Message objects. 445 """ 446 447 @classmethod 448 def BotoLifecycleFromMessage(cls, lifecycle_message): 449 """Translates an apitools message to a boto lifecycle object.""" 450 boto_lifecycle = boto.gs.lifecycle.LifecycleConfig() 451 if lifecycle_message: 452 for rule_message in lifecycle_message.rule: 453 boto_rule = boto.gs.lifecycle.Rule() 454 if (rule_message.action and rule_message.action.type and 455 rule_message.action.type.lower() == 'delete'): 456 boto_rule.action = boto.gs.lifecycle.DELETE 457 if rule_message.condition: 458 if rule_message.condition.age: 459 boto_rule.conditions[boto.gs.lifecycle.AGE] = ( 460 str(rule_message.condition.age)) 461 if rule_message.condition.createdBefore: 462 boto_rule.conditions[boto.gs.lifecycle.CREATED_BEFORE] = ( 463 str(rule_message.condition.createdBefore)) 464 if rule_message.condition.isLive: 465 boto_rule.conditions[boto.gs.lifecycle.IS_LIVE] = ( 466 str(rule_message.condition.isLive)) 467 if rule_message.condition.numNewerVersions: 468 boto_rule.conditions[boto.gs.lifecycle.NUM_NEWER_VERSIONS] = ( 469 str(rule_message.condition.numNewerVersions)) 470 boto_lifecycle.append(boto_rule) 471 return boto_lifecycle 472 473 @classmethod 474 def BotoLifecycleToMessage(cls, boto_lifecycle): 475 """Translates a boto lifecycle object to an apitools message.""" 476 lifecycle_message = None 477 if boto_lifecycle: 478 lifecycle_message = apitools_messages.Bucket.LifecycleValue() 479 for boto_rule in boto_lifecycle: 480 lifecycle_rule = ( 481 apitools_messages.Bucket.LifecycleValue.RuleValueListEntry()) 482 lifecycle_rule.condition = (apitools_messages.Bucket.LifecycleValue. 483 RuleValueListEntry.ConditionValue()) 484 if boto_rule.action and boto_rule.action == boto.gs.lifecycle.DELETE: 485 lifecycle_rule.action = (apitools_messages.Bucket.LifecycleValue. 486 RuleValueListEntry.ActionValue( 487 type='Delete')) 488 if boto.gs.lifecycle.AGE in boto_rule.conditions: 489 lifecycle_rule.condition.age = int( 490 boto_rule.conditions[boto.gs.lifecycle.AGE]) 491 if boto.gs.lifecycle.CREATED_BEFORE in boto_rule.conditions: 492 lifecycle_rule.condition.createdBefore = ( 493 LifecycleTranslation.TranslateBotoLifecycleTimestamp( 494 boto_rule.conditions[boto.gs.lifecycle.CREATED_BEFORE])) 495 if boto.gs.lifecycle.IS_LIVE in boto_rule.conditions: 496 lifecycle_rule.condition.isLive = bool( 497 boto_rule.conditions[boto.gs.lifecycle.IS_LIVE]) 498 if boto.gs.lifecycle.NUM_NEWER_VERSIONS in boto_rule.conditions: 499 lifecycle_rule.condition.numNewerVersions = int( 500 boto_rule.conditions[boto.gs.lifecycle.NUM_NEWER_VERSIONS]) 501 lifecycle_message.rule.append(lifecycle_rule) 502 return lifecycle_message 503 504 @classmethod 505 def JsonLifecycleFromMessage(cls, lifecycle_message): 506 """Translates an apitools message to lifecycle JSON.""" 507 return str(encoding.MessageToJson(lifecycle_message)) + '\n' 508 509 @classmethod 510 def JsonLifecycleToMessage(cls, json_txt): 511 """Translates lifecycle JSON to an apitools message.""" 512 try: 513 deserialized_lifecycle = json.loads(json_txt) 514 # If lifecycle JSON is the in the following format 515 # {'lifecycle': {'rule': ... then strip out the 'lifecycle' key 516 # and reduce it to the following format 517 # {'rule': ... 518 if 'lifecycle' in deserialized_lifecycle: 519 deserialized_lifecycle = deserialized_lifecycle['lifecycle'] 520 lifecycle = encoding.DictToMessage( 521 deserialized_lifecycle, apitools_messages.Bucket.LifecycleValue) 522 return lifecycle 523 except ValueError: 524 CheckForXmlConfigurationAndRaise('lifecycle', json_txt) 525 526 @classmethod 527 def TranslateBotoLifecycleTimestamp(cls, lifecycle_datetime): 528 """Parses the timestamp from the boto lifecycle into a datetime object.""" 529 return datetime.datetime.strptime(lifecycle_datetime, '%Y-%m-%d').date() 530 531 532 class CorsTranslation(object): 533 """Functions for converting between various CORS formats. 534 535 This class handles conversation to and from Boto Cors objects, JSON text, 536 and apitools Message objects. 537 """ 538 539 @classmethod 540 def BotoCorsFromMessage(cls, cors_message): 541 """Translates an apitools message to a boto Cors object.""" 542 cors = boto.gs.cors.Cors() 543 cors.cors = [] 544 for collection_message in cors_message: 545 collection_elements = [] 546 if collection_message.maxAgeSeconds: 547 collection_elements.append((boto.gs.cors.MAXAGESEC, 548 str(collection_message.maxAgeSeconds))) 549 if collection_message.method: 550 method_elements = [] 551 for method in collection_message.method: 552 method_elements.append((boto.gs.cors.METHOD, method)) 553 collection_elements.append((boto.gs.cors.METHODS, method_elements)) 554 if collection_message.origin: 555 origin_elements = [] 556 for origin in collection_message.origin: 557 origin_elements.append((boto.gs.cors.ORIGIN, origin)) 558 collection_elements.append((boto.gs.cors.ORIGINS, origin_elements)) 559 if collection_message.responseHeader: 560 header_elements = [] 561 for header in collection_message.responseHeader: 562 header_elements.append((boto.gs.cors.HEADER, header)) 563 collection_elements.append((boto.gs.cors.HEADERS, header_elements)) 564 cors.cors.append(collection_elements) 565 return cors 566 567 @classmethod 568 def BotoCorsToMessage(cls, boto_cors): 569 """Translates a boto Cors object to an apitools message.""" 570 message_cors = [] 571 if boto_cors.cors: 572 for cors_collection in boto_cors.cors: 573 if cors_collection: 574 collection_message = apitools_messages.Bucket.CorsValueListEntry() 575 for element_tuple in cors_collection: 576 if element_tuple[0] == boto.gs.cors.MAXAGESEC: 577 collection_message.maxAgeSeconds = int(element_tuple[1]) 578 if element_tuple[0] == boto.gs.cors.METHODS: 579 for method_tuple in element_tuple[1]: 580 collection_message.method.append(method_tuple[1]) 581 if element_tuple[0] == boto.gs.cors.ORIGINS: 582 for origin_tuple in element_tuple[1]: 583 collection_message.origin.append(origin_tuple[1]) 584 if element_tuple[0] == boto.gs.cors.HEADERS: 585 for header_tuple in element_tuple[1]: 586 collection_message.responseHeader.append(header_tuple[1]) 587 message_cors.append(collection_message) 588 return message_cors 589 590 @classmethod 591 def JsonCorsToMessageEntries(cls, json_cors): 592 """Translates CORS JSON to an apitools message. 593 594 Args: 595 json_cors: JSON string representing CORS configuration. 596 597 Returns: 598 List of apitools Bucket.CorsValueListEntry. An empty list represents 599 no CORS configuration. 600 """ 601 try: 602 deserialized_cors = json.loads(json_cors) 603 cors = [] 604 for cors_entry in deserialized_cors: 605 cors.append(encoding.DictToMessage( 606 cors_entry, apitools_messages.Bucket.CorsValueListEntry)) 607 return cors 608 except ValueError: 609 CheckForXmlConfigurationAndRaise('CORS', json_cors) 610 611 @classmethod 612 def MessageEntriesToJson(cls, cors_message): 613 """Translates an apitools message to CORS JSON.""" 614 json_text = '' 615 # Because CORS is a MessageField, serialize/deserialize as JSON list. 616 json_text += '[' 617 printed_one = False 618 for cors_entry in cors_message: 619 if printed_one: 620 json_text += ',' 621 else: 622 printed_one = True 623 json_text += encoding.MessageToJson(cors_entry) 624 json_text += ']\n' 625 return json_text 626 627 628 def S3MarkerAclFromObjectMetadata(object_metadata): 629 """Retrieves GUID-marked S3 ACL from object metadata, if present. 630 631 Args: 632 object_metadata: Object metadata to check. 633 634 Returns: 635 S3 ACL text, if present, None otherwise. 636 """ 637 if (object_metadata and object_metadata.metadata and 638 object_metadata.metadata.additionalProperties): 639 for prop in object_metadata.metadata.additionalProperties: 640 if prop.key == S3_ACL_MARKER_GUID: 641 return prop.value 642 643 644 def AddS3MarkerAclToObjectMetadata(object_metadata, acl_text): 645 """Adds a GUID-marked S3 ACL to the object metadata. 646 647 Args: 648 object_metadata: Object metadata to add the acl to. 649 acl_text: S3 ACL text to add. 650 """ 651 if not object_metadata.metadata: 652 object_metadata.metadata = apitools_messages.Object.MetadataValue() 653 if not object_metadata.metadata.additionalProperties: 654 object_metadata.metadata.additionalProperties = [] 655 656 object_metadata.metadata.additionalProperties.append( 657 apitools_messages.Object.MetadataValue.AdditionalProperty( 658 key=S3_ACL_MARKER_GUID, value=acl_text)) 659 660 661 class AclTranslation(object): 662 """Functions for converting between various ACL formats. 663 664 This class handles conversion to and from Boto ACL objects, JSON text, 665 and apitools Message objects. 666 """ 667 668 JSON_TO_XML_ROLES = {'READER': 'READ', 'WRITER': 'WRITE', 669 'OWNER': 'FULL_CONTROL'} 670 XML_TO_JSON_ROLES = {'READ': 'READER', 'WRITE': 'WRITER', 671 'FULL_CONTROL': 'OWNER'} 672 673 @classmethod 674 def BotoAclFromJson(cls, acl_json): 675 acl = ACL() 676 acl.parent = None 677 acl.entries = cls.BotoEntriesFromJson(acl_json, acl) 678 return acl 679 680 @classmethod 681 # acl_message is a list of messages, either object or bucketaccesscontrol 682 def BotoAclFromMessage(cls, acl_message): 683 acl_dicts = [] 684 for message in acl_message: 685 if message == PRIVATE_DEFAULT_OBJ_ACL: 686 # Sentinel value indicating acl_dicts should be an empty list to create 687 # a private (no entries) default object ACL. 688 break 689 acl_dicts.append(encoding.MessageToDict(message)) 690 return cls.BotoAclFromJson(acl_dicts) 691 692 @classmethod 693 def BotoAclToJson(cls, acl): 694 if hasattr(acl, 'entries'): 695 return cls.BotoEntriesToJson(acl.entries) 696 return [] 697 698 @classmethod 699 def BotoObjectAclToMessage(cls, acl): 700 for entry in cls.BotoAclToJson(acl): 701 message = encoding.DictToMessage(entry, 702 apitools_messages.ObjectAccessControl) 703 message.kind = u'storage#objectAccessControl' 704 yield message 705 706 @classmethod 707 def BotoBucketAclToMessage(cls, acl): 708 for entry in cls.BotoAclToJson(acl): 709 message = encoding.DictToMessage(entry, 710 apitools_messages.BucketAccessControl) 711 message.kind = u'storage#bucketAccessControl' 712 yield message 713 714 @classmethod 715 def BotoEntriesFromJson(cls, acl_json, parent): 716 entries = Entries(parent) 717 entries.parent = parent 718 entries.entry_list = [cls.BotoEntryFromJson(entry_json) 719 for entry_json in acl_json] 720 return entries 721 722 @classmethod 723 def BotoEntriesToJson(cls, entries): 724 return [cls.BotoEntryToJson(entry) for entry in entries.entry_list] 725 726 @classmethod 727 def BotoEntryFromJson(cls, entry_json): 728 """Converts a JSON entry into a Boto ACL entry.""" 729 entity = entry_json['entity'] 730 permission = cls.JSON_TO_XML_ROLES[entry_json['role']] 731 if entity.lower() == ALL_USERS.lower(): 732 return Entry(type=ALL_USERS, permission=permission) 733 elif entity.lower() == ALL_AUTHENTICATED_USERS.lower(): 734 return Entry(type=ALL_AUTHENTICATED_USERS, permission=permission) 735 elif entity.startswith('project'): 736 raise CommandException('XML API does not support project scopes, ' 737 'cannot translate ACL.') 738 elif 'email' in entry_json: 739 if entity.startswith('user'): 740 scope_type = USER_BY_EMAIL 741 elif entity.startswith('group'): 742 scope_type = GROUP_BY_EMAIL 743 return Entry(type=scope_type, email_address=entry_json['email'], 744 permission=permission) 745 elif 'entityId' in entry_json: 746 if entity.startswith('user'): 747 scope_type = USER_BY_ID 748 elif entity.startswith('group'): 749 scope_type = GROUP_BY_ID 750 return Entry(type=scope_type, id=entry_json['entityId'], 751 permission=permission) 752 elif 'domain' in entry_json: 753 if entity.startswith('domain'): 754 scope_type = GROUP_BY_DOMAIN 755 return Entry(type=scope_type, domain=entry_json['domain'], 756 permission=permission) 757 raise CommandException('Failed to translate JSON ACL to XML.') 758 759 @classmethod 760 def BotoEntryToJson(cls, entry): 761 """Converts a Boto ACL entry to a valid JSON dictionary.""" 762 acl_entry_json = {} 763 # JSON API documentation uses camel case. 764 scope_type_lower = entry.scope.type.lower() 765 if scope_type_lower == ALL_USERS.lower(): 766 acl_entry_json['entity'] = 'allUsers' 767 elif scope_type_lower == ALL_AUTHENTICATED_USERS.lower(): 768 acl_entry_json['entity'] = 'allAuthenticatedUsers' 769 elif scope_type_lower == USER_BY_EMAIL.lower(): 770 acl_entry_json['entity'] = 'user-%s' % entry.scope.email_address 771 acl_entry_json['email'] = entry.scope.email_address 772 elif scope_type_lower == USER_BY_ID.lower(): 773 acl_entry_json['entity'] = 'user-%s' % entry.scope.id 774 acl_entry_json['entityId'] = entry.scope.id 775 elif scope_type_lower == GROUP_BY_EMAIL.lower(): 776 acl_entry_json['entity'] = 'group-%s' % entry.scope.email_address 777 acl_entry_json['email'] = entry.scope.email_address 778 elif scope_type_lower == GROUP_BY_ID.lower(): 779 acl_entry_json['entity'] = 'group-%s' % entry.scope.id 780 acl_entry_json['entityId'] = entry.scope.id 781 elif scope_type_lower == GROUP_BY_DOMAIN.lower(): 782 acl_entry_json['entity'] = 'domain-%s' % entry.scope.domain 783 acl_entry_json['domain'] = entry.scope.domain 784 else: 785 raise ArgumentException('ACL contains invalid scope type: %s' % 786 scope_type_lower) 787 788 acl_entry_json['role'] = cls.XML_TO_JSON_ROLES[entry.permission] 789 return acl_entry_json 790 791 @classmethod 792 def JsonToMessage(cls, json_data, message_type): 793 """Converts the input JSON data into list of Object/BucketAccessControls. 794 795 Args: 796 json_data: String of JSON to convert. 797 message_type: Which type of access control entries to return, 798 either ObjectAccessControl or BucketAccessControl. 799 800 Raises: 801 ArgumentException on invalid JSON data. 802 803 Returns: 804 List of ObjectAccessControl or BucketAccessControl elements. 805 """ 806 try: 807 deserialized_acl = json.loads(json_data) 808 809 acl = [] 810 for acl_entry in deserialized_acl: 811 acl.append(encoding.DictToMessage(acl_entry, message_type)) 812 return acl 813 except ValueError: 814 CheckForXmlConfigurationAndRaise('ACL', json_data) 815 816 @classmethod 817 def JsonFromMessage(cls, acl): 818 """Strips unnecessary fields from an ACL message and returns valid JSON. 819 820 Args: 821 acl: iterable ObjectAccessControl or BucketAccessControl 822 823 Returns: 824 ACL JSON string. 825 """ 826 serializable_acl = [] 827 if acl is not None: 828 for acl_entry in acl: 829 if acl_entry.kind == u'storage#objectAccessControl': 830 acl_entry.object = None 831 acl_entry.generation = None 832 acl_entry.kind = None 833 acl_entry.bucket = None 834 acl_entry.id = None 835 acl_entry.selfLink = None 836 acl_entry.etag = None 837 serializable_acl.append(encoding.MessageToDict(acl_entry)) 838 return json.dumps(serializable_acl, sort_keys=True, 839 indent=2, separators=(',', ': ')) 840
