1 # -*- coding: utf-8 -*- 2 # Copyright 2011 Google Inc. All Rights Reserved. 3 # 4 # Licensed under the Apache License, Version 2.0 (the "License"); 5 # you may not use this file except in compliance with the License. 6 # You may obtain a copy of the License at 7 # 8 # http://www.apache.org/licenses/LICENSE-2.0 9 # 10 # Unless required by applicable law or agreed to in writing, software 11 # distributed under the License is distributed on an "AS IS" BASIS, 12 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 # See the License for the specific language governing permissions and 14 # limitations under the License. 15 """Class that runs a named gsutil command.""" 16 17 from __future__ import absolute_import 18 19 import difflib 20 import logging 21 import os 22 import pkgutil 23 import sys 24 import textwrap 25 import time 26 27 import boto 28 from boto.storage_uri import BucketStorageUri 29 import gslib 30 from gslib.cloud_api_delegator import CloudApiDelegator 31 from gslib.command import Command 32 from gslib.command import CreateGsutilLogger 33 from gslib.command import GetFailureCount 34 from gslib.command import OLD_ALIAS_MAP 35 from gslib.command import ShutDownGsutil 36 import gslib.commands 37 from gslib.cs_api_map import ApiSelector 38 from gslib.cs_api_map import GsutilApiClassMapFactory 39 from gslib.cs_api_map import GsutilApiMapFactory 40 from gslib.exception import CommandException 41 from gslib.gcs_json_api import GcsJsonApi 42 from gslib.no_op_credentials import NoOpCredentials 43 from gslib.tab_complete import MakeCompleter 44 from gslib.util import CheckMultiprocessingAvailableAndInit 45 from gslib.util import CompareVersions 46 from gslib.util import GetGsutilVersionModifiedTime 47 from gslib.util import GSUTIL_PUB_TARBALL 48 from gslib.util import IsRunningInteractively 49 from gslib.util import LAST_CHECKED_FOR_GSUTIL_UPDATE_TIMESTAMP_FILE 50 from gslib.util import LookUpGsutilVersion 51 from gslib.util import RELEASE_NOTES_URL 52 from gslib.util import SECONDS_PER_DAY 53 from gslib.util import UTF8 54 55 56 def HandleArgCoding(args): 57 """Handles coding of command-line args. 58 59 Args: 60 args: array of command-line args. 61 62 Returns: 63 array of command-line args. 64 65 Raises: 66 CommandException: if errors encountered. 67 """ 68 # Python passes arguments from the command line as byte strings. To 69 # correctly interpret them, we decode ones other than -h and -p args (which 70 # will be passed as headers, and thus per HTTP spec should not be encoded) as 71 # utf-8. The exception is x-goog-meta-* headers, which are allowed to contain 72 # non-ASCII content (and hence, should be decoded), per 73 # https://developers.google.com/storage/docs/gsutil/addlhelp/WorkingWithObjectMetadata 74 processing_header = False 75 for i in range(len(args)): 76 arg = args[i] 77 # Commands like mv can run this function twice; don't decode twice. 78 try: 79 decoded = arg if isinstance(arg, unicode) else arg.decode(UTF8) 80 except UnicodeDecodeError: 81 raise CommandException('\n'.join(textwrap.wrap( 82 'Invalid encoding for argument (%s). Arguments must be decodable as ' 83 'Unicode. NOTE: the argument printed above replaces the problematic ' 84 'characters with a hex-encoded printable representation. For more ' 85 'details (including how to convert to a gsutil-compatible encoding) ' 86 'see `gsutil help encoding`.' % repr(arg)))) 87 if processing_header: 88 if arg.lower().startswith('x-goog-meta'): 89 args[i] = decoded 90 else: 91 try: 92 # Try to encode as ASCII to check for invalid header values (which 93 # can't be sent over HTTP). 94 decoded.encode('ascii') 95 except UnicodeEncodeError: 96 # Raise the CommandException using the decoded value because 97 # _OutputAndExit function re-encodes at the end. 98 raise CommandException( 99 'Invalid non-ASCII header value (%s).\nOnly ASCII characters are ' 100 'allowed in headers other than x-goog-meta- headers' % decoded) 101 else: 102 args[i] = decoded 103 processing_header = (arg in ('-h', '-p')) 104 return args 105 106 107 class CommandRunner(object): 108 """Runs gsutil commands and does some top-level argument handling.""" 109 110 def __init__(self, bucket_storage_uri_class=BucketStorageUri, 111 gsutil_api_class_map_factory=GsutilApiClassMapFactory, 112 command_map=None): 113 """Instantiates a CommandRunner. 114 115 Args: 116 bucket_storage_uri_class: Class to instantiate for cloud StorageUris. 117 Settable for testing/mocking. 118 gsutil_api_class_map_factory: Creates map of cloud storage interfaces. 119 Settable for testing/mocking. 120 command_map: Map of command names to their implementations for 121 testing/mocking. If not set, the map is built dynamically. 122 """ 123 self.bucket_storage_uri_class = bucket_storage_uri_class 124 self.gsutil_api_class_map_factory = gsutil_api_class_map_factory 125 if command_map: 126 self.command_map = command_map 127 else: 128 self.command_map = self._LoadCommandMap() 129 130 def _LoadCommandMap(self): 131 """Returns dict mapping each command_name to implementing class.""" 132 # Import all gslib.commands submodules. 133 for _, module_name, _ in pkgutil.iter_modules(gslib.commands.__path__): 134 __import__('gslib.commands.%s' % module_name) 135 136 command_map = {} 137 # Only include Command subclasses in the dict. 138 for command in Command.__subclasses__(): 139 command_map[command.command_spec.command_name] = command 140 for command_name_aliases in command.command_spec.command_name_aliases: 141 command_map[command_name_aliases] = command 142 return command_map 143 144 def _ConfigureCommandArgumentParserArguments( 145 self, parser, arguments, gsutil_api): 146 """Configures an argument parser with the given arguments. 147 148 Args: 149 parser: argparse parser object. 150 arguments: array of CommandArgument objects. 151 gsutil_api: gsutil Cloud API instance to use. 152 Raises: 153 RuntimeError: if argument is configured with unsupported completer 154 """ 155 for command_argument in arguments: 156 action = parser.add_argument( 157 *command_argument.args, **command_argument.kwargs) 158 if command_argument.completer: 159 action.completer = MakeCompleter(command_argument.completer, gsutil_api) 160 161 def ConfigureCommandArgumentParsers(self, subparsers): 162 """Configures argparse arguments and argcomplete completers for commands. 163 164 Args: 165 subparsers: argparse object that can be used to add parsers for 166 subcommands (called just 'commands' in gsutil) 167 """ 168 169 # This should match the support map for the "ls" command. 170 support_map = { 171 'gs': [ApiSelector.XML, ApiSelector.JSON], 172 's3': [ApiSelector.XML] 173 } 174 default_map = { 175 'gs': ApiSelector.JSON, 176 's3': ApiSelector.XML 177 } 178 gsutil_api_map = GsutilApiMapFactory.GetApiMap( 179 self.gsutil_api_class_map_factory, support_map, default_map) 180 181 logger = CreateGsutilLogger('tab_complete') 182 gsutil_api = CloudApiDelegator( 183 self.bucket_storage_uri_class, gsutil_api_map, 184 logger, debug=0) 185 186 for command in set(self.command_map.values()): 187 command_parser = subparsers.add_parser( 188 command.command_spec.command_name, add_help=False) 189 if isinstance(command.command_spec.argparse_arguments, dict): 190 subcommand_parsers = command_parser.add_subparsers() 191 subcommand_argument_dict = command.command_spec.argparse_arguments 192 for subcommand, arguments in subcommand_argument_dict.iteritems(): 193 subcommand_parser = subcommand_parsers.add_parser( 194 subcommand, add_help=False) 195 self._ConfigureCommandArgumentParserArguments( 196 subcommand_parser, arguments, gsutil_api) 197 else: 198 self._ConfigureCommandArgumentParserArguments( 199 command_parser, command.command_spec.argparse_arguments, gsutil_api) 200 201 def RunNamedCommand(self, command_name, args=None, headers=None, debug=0, 202 trace_token=None, parallel_operations=False, 203 skip_update_check=False, logging_filters=None, 204 do_shutdown=True): 205 """Runs the named command. 206 207 Used by gsutil main, commands built atop other commands, and tests. 208 209 Args: 210 command_name: The name of the command being run. 211 args: Command-line args (arg0 = actual arg, not command name ala bash). 212 headers: Dictionary containing optional HTTP headers to pass to boto. 213 debug: Debug level to pass in to boto connection (range 0..3). 214 trace_token: Trace token to pass to the underlying API. 215 parallel_operations: Should command operations be executed in parallel? 216 skip_update_check: Set to True to disable checking for gsutil updates. 217 logging_filters: Optional list of logging.Filters to apply to this 218 command's logger. 219 do_shutdown: Stop all parallelism framework workers iff this is True. 220 221 Raises: 222 CommandException: if errors encountered. 223 224 Returns: 225 Return value(s) from Command that was run. 226 """ 227 command_changed_to_update = False 228 if (not skip_update_check and 229 self.MaybeCheckForAndOfferSoftwareUpdate(command_name, debug)): 230 command_name = 'update' 231 command_changed_to_update = True 232 args = ['-n'] 233 234 if not args: 235 args = [] 236 237 # Include api_version header in all commands. 238 api_version = boto.config.get_value('GSUtil', 'default_api_version', '1') 239 if not headers: 240 headers = {} 241 headers['x-goog-api-version'] = api_version 242 243 if command_name not in self.command_map: 244 close_matches = difflib.get_close_matches( 245 command_name, self.command_map.keys(), n=1) 246 if close_matches: 247 # Instead of suggesting a deprecated command alias, suggest the new 248 # name for that command. 249 translated_command_name = ( 250 OLD_ALIAS_MAP.get(close_matches[0], close_matches)[0]) 251 print >> sys.stderr, 'Did you mean this?' 252 print >> sys.stderr, '\t%s' % translated_command_name 253 elif command_name == 'update' and gslib.IS_PACKAGE_INSTALL: 254 sys.stderr.write( 255 'Update command is not supported for package installs; ' 256 'please instead update using your package manager.') 257 258 raise CommandException('Invalid command "%s".' % command_name) 259 if '--help' in args: 260 new_args = [command_name] 261 original_command_class = self.command_map[command_name] 262 subcommands = original_command_class.help_spec.subcommand_help_text.keys() 263 for arg in args: 264 if arg in subcommands: 265 new_args.append(arg) 266 break # Take the first match and throw away the rest. 267 args = new_args 268 command_name = 'help' 269 270 args = HandleArgCoding(args) 271 272 command_class = self.command_map[command_name] 273 command_inst = command_class( 274 self, args, headers, debug, trace_token, parallel_operations, 275 self.bucket_storage_uri_class, self.gsutil_api_class_map_factory, 276 logging_filters, command_alias_used=command_name) 277 return_code = command_inst.RunCommand() 278 279 if CheckMultiprocessingAvailableAndInit().is_available and do_shutdown: 280 ShutDownGsutil() 281 if GetFailureCount() > 0: 282 return_code = 1 283 if command_changed_to_update: 284 # If the command changed to update, the user's original command was 285 # not executed. 286 return_code = 1 287 print '\n'.join(textwrap.wrap( 288 'Update was successful. Exiting with code 1 as the original command ' 289 'issued prior to the update was not executed and should be re-run.')) 290 return return_code 291 292 def MaybeCheckForAndOfferSoftwareUpdate(self, command_name, debug): 293 """Checks the last time we checked for an update and offers one if needed. 294 295 Offer is made if the time since the last update check is longer 296 than the configured threshold offers the user to update gsutil. 297 298 Args: 299 command_name: The name of the command being run. 300 debug: Debug level to pass in to boto connection (range 0..3). 301 302 Returns: 303 True if the user decides to update. 304 """ 305 # Don't try to interact with user if: 306 # - gsutil is not connected to a tty (e.g., if being run from cron); 307 # - user is running gsutil -q 308 # - user is running the config command (which could otherwise attempt to 309 # check for an update for a user running behind a proxy, who has not yet 310 # configured gsutil to go through the proxy; for such users we need the 311 # first connection attempt to be made by the gsutil config command). 312 # - user is running the version command (which gets run when using 313 # gsutil -D, which would prevent users with proxy config problems from 314 # sending us gsutil -D output). 315 # - user is running the update command (which could otherwise cause an 316 # additional note that an update is available when user is already trying 317 # to perform an update); 318 # - user specified gs_host (which could be a non-production different 319 # service instance, in which case credentials won't work for checking 320 # gsutil tarball). 321 # - user is using a Cloud SDK install (which should only be updated via 322 # gcloud components update) 323 logger = logging.getLogger() 324 gs_host = boto.config.get('Credentials', 'gs_host', None) 325 if (not IsRunningInteractively() 326 or command_name in ('config', 'update', 'ver', 'version') 327 or not logger.isEnabledFor(logging.INFO) 328 or gs_host 329 or os.environ.get('CLOUDSDK_WRAPPER') == '1'): 330 return False 331 332 software_update_check_period = boto.config.getint( 333 'GSUtil', 'software_update_check_period', 30) 334 # Setting software_update_check_period to 0 means periodic software 335 # update checking is disabled. 336 if software_update_check_period == 0: 337 return False 338 339 cur_ts = int(time.time()) 340 if not os.path.isfile(LAST_CHECKED_FOR_GSUTIL_UPDATE_TIMESTAMP_FILE): 341 # Set last_checked_ts from date of VERSION file, so if the user installed 342 # an old copy of gsutil it will get noticed (and an update offered) the 343 # first time they try to run it. 344 last_checked_ts = GetGsutilVersionModifiedTime() 345 with open(LAST_CHECKED_FOR_GSUTIL_UPDATE_TIMESTAMP_FILE, 'w') as f: 346 f.write(str(last_checked_ts)) 347 else: 348 try: 349 with open(LAST_CHECKED_FOR_GSUTIL_UPDATE_TIMESTAMP_FILE, 'r') as f: 350 last_checked_ts = int(f.readline()) 351 except (TypeError, ValueError): 352 return False 353 354 if (cur_ts - last_checked_ts 355 > software_update_check_period * SECONDS_PER_DAY): 356 # Create a credential-less gsutil API to check for the public 357 # update tarball. 358 gsutil_api = GcsJsonApi(self.bucket_storage_uri_class, logger, 359 credentials=NoOpCredentials(), debug=debug) 360 361 cur_ver = LookUpGsutilVersion(gsutil_api, GSUTIL_PUB_TARBALL) 362 with open(LAST_CHECKED_FOR_GSUTIL_UPDATE_TIMESTAMP_FILE, 'w') as f: 363 f.write(str(cur_ts)) 364 (g, m) = CompareVersions(cur_ver, gslib.VERSION) 365 if m: 366 print '\n'.join(textwrap.wrap( 367 'A newer version of gsutil (%s) is available than the version you ' 368 'are running (%s). NOTE: This is a major new version, so it is ' 369 'strongly recommended that you review the release note details at ' 370 '%s before updating to this version, especially if you use gsutil ' 371 'in scripts.' % (cur_ver, gslib.VERSION, RELEASE_NOTES_URL))) 372 if gslib.IS_PACKAGE_INSTALL: 373 return False 374 print 375 answer = raw_input('Would you like to update [y/N]? ') 376 return answer and answer.lower()[0] == 'y' 377 elif g: 378 print '\n'.join(textwrap.wrap( 379 'A newer version of gsutil (%s) is available than the version you ' 380 'are running (%s). A detailed log of gsutil release changes is ' 381 'available at %s if you would like to read them before updating.' 382 % (cur_ver, gslib.VERSION, RELEASE_NOTES_URL))) 383 if gslib.IS_PACKAGE_INSTALL: 384 return False 385 print 386 answer = raw_input('Would you like to update [Y/n]? ') 387 return not answer or answer.lower()[0] != 'n' 388 return False 389
