1 # -*- coding: utf-8 -*- 2 # Copyright 2013 Google Inc. All Rights Reserved. 3 # 4 # Licensed under the Apache License, Version 2.0 (the "License"); 5 # you may not use this file except in compliance with the License. 6 # You may obtain a copy of the License at 7 # 8 # http://www.apache.org/licenses/LICENSE-2.0 9 # 10 # Unless required by applicable law or agreed to in writing, software 11 # distributed under the License is distributed on an "AS IS" BASIS, 12 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 # See the License for the specific language governing permissions and 14 # limitations under the License. 15 """Implementation of compose command for Google Cloud Storage.""" 16 17 from __future__ import absolute_import 18 19 from gslib.bucket_listing_ref import BucketListingObject 20 from gslib.command import Command 21 from gslib.command_argument import CommandArgument 22 from gslib.cs_api_map import ApiSelector 23 from gslib.exception import CommandException 24 from gslib.storage_url import ContainsWildcard 25 from gslib.storage_url import StorageUrlFromString 26 from gslib.third_party.storage_apitools import storage_v1_messages as apitools_messages 27 from gslib.translation_helper import PreconditionsFromHeaders 28 29 MAX_COMPONENT_COUNT = 1024 30 MAX_COMPOSE_ARITY = 32 31 32 _SYNOPSIS = """ 33 gsutil compose gs://bucket/obj1 gs://bucket/obj2 ... gs://bucket/composite 34 """ 35 36 _DETAILED_HELP_TEXT = (""" 37 <B>SYNOPSIS</B> 38 """ + _SYNOPSIS + """ 39 40 41 <B>DESCRIPTION</B> 42 The compose command creates a new object whose content is the concatenation 43 of a given sequence of component objects under the same bucket. gsutil uses 44 the content type of the first source object to determine the destination 45 object's content type. For more information, please see: 46 https://developers.google.com/storage/docs/composite-objects 47 48 Note also that the gsutil cp command will automatically split uploads for 49 large files into multiple component objects, upload them in parallel, and 50 compose them into a final object (which will be subject to the component 51 count limit). This will still perform all uploads from a single machine. For 52 extremely large files and/or very low per-machine bandwidth, you may want to 53 split the file and upload it from multiple machines, and later compose these 54 parts of the file manually. See the 'PARALLEL COMPOSITE UPLOADS' section under 55 'gsutil help cp' for details. 56 57 Appending simply entails uploading your new data to a temporary object, 58 composing it with the growing append-target, and deleting the temporary 59 object: 60 61 $ echo 'new data' | gsutil cp - gs://bucket/data-to-append 62 $ gsutil compose gs://bucket/append-target gs://bucket/data-to-append \\ 63 gs://bucket/append-target 64 $ gsutil rm gs://bucket/data-to-append 65 66 Note that there is a limit (currently %d) to the number of components for a 67 given composite object. This means you can append to each object at most %d 68 times. 69 """ % (MAX_COMPONENT_COUNT, MAX_COMPONENT_COUNT - 1)) 70 71 72 class ComposeCommand(Command): 73 """Implementation of gsutil compose command.""" 74 75 # Command specification. See base class for documentation. 76 command_spec = Command.CreateCommandSpec( 77 'compose', 78 command_name_aliases=['concat'], 79 usage_synopsis=_SYNOPSIS, 80 min_args=2, 81 max_args=MAX_COMPOSE_ARITY + 1, 82 supported_sub_args='', 83 # Not files, just object names without gs:// prefix. 84 file_url_ok=False, 85 provider_url_ok=False, 86 urls_start_arg=1, 87 gs_api_support=[ApiSelector.XML, ApiSelector.JSON], 88 gs_default_api=ApiSelector.JSON, 89 argparse_arguments=[ 90 CommandArgument.MakeZeroOrMoreCloudURLsArgument() 91 ] 92 ) 93 # Help specification. See help_provider.py for documentation. 94 help_spec = Command.HelpSpec( 95 help_name='compose', 96 help_name_aliases=['concat'], 97 help_type='command_help', 98 help_one_line_summary=( 99 'Concatenate a sequence of objects into a new composite object.'), 100 help_text=_DETAILED_HELP_TEXT, 101 subcommand_help_text={}, 102 ) 103 104 def CheckProvider(self, url): 105 if url.scheme != 'gs': 106 raise CommandException( 107 '"compose" called on URL with unsupported provider (%s).' % str(url)) 108 109 # Command entry point. 110 def RunCommand(self): 111 """Command entry point for the compose command.""" 112 target_url_str = self.args[-1] 113 self.args = self.args[:-1] 114 target_url = StorageUrlFromString(target_url_str) 115 self.CheckProvider(target_url) 116 if target_url.HasGeneration(): 117 raise CommandException('A version-specific URL (%s) cannot be ' 118 'the destination for gsutil compose - abort.' 119 % target_url) 120 121 dst_obj_metadata = apitools_messages.Object(name=target_url.object_name, 122 bucket=target_url.bucket_name) 123 124 components = [] 125 # Remember the first source object so we can get its content type. 126 first_src_url = None 127 for src_url_str in self.args: 128 if ContainsWildcard(src_url_str): 129 src_url_iter = self.WildcardIterator(src_url_str).IterObjects() 130 else: 131 src_url_iter = [BucketListingObject(StorageUrlFromString(src_url_str))] 132 for blr in src_url_iter: 133 src_url = blr.storage_url 134 self.CheckProvider(src_url) 135 136 if src_url.bucket_name != target_url.bucket_name: 137 raise CommandException( 138 'GCS does not support inter-bucket composing.') 139 140 if not first_src_url: 141 first_src_url = src_url 142 src_obj_metadata = ( 143 apitools_messages.ComposeRequest.SourceObjectsValueListEntry( 144 name=src_url.object_name)) 145 if src_url.HasGeneration(): 146 src_obj_metadata.generation = src_url.generation 147 components.append(src_obj_metadata) 148 # Avoid expanding too many components, and sanity check each name 149 # expansion result. 150 if len(components) > MAX_COMPOSE_ARITY: 151 raise CommandException('"compose" called with too many component ' 152 'objects. Limit is %d.' % MAX_COMPOSE_ARITY) 153 154 if len(components) < 2: 155 raise CommandException('"compose" requires at least 2 component objects.') 156 157 dst_obj_metadata.contentType = self.gsutil_api.GetObjectMetadata( 158 first_src_url.bucket_name, first_src_url.object_name, 159 provider=first_src_url.scheme, fields=['contentType']).contentType 160 161 preconditions = PreconditionsFromHeaders(self.headers or {}) 162 163 self.logger.info( 164 'Composing %s from %d component objects.', target_url, len(components)) 165 self.gsutil_api.ComposeObject(components, dst_obj_metadata, 166 preconditions=preconditions, 167 provider=target_url.scheme) 168