1 # -*- coding: utf-8 -*- 2 # Copyright 2011 Google Inc. All Rights Reserved. 3 # 4 # Licensed under the Apache License, Version 2.0 (the "License"); 5 # you may not use this file except in compliance with the License. 6 # You may obtain a copy of the License at 7 # 8 # http://www.apache.org/licenses/LICENSE-2.0 9 # 10 # Unless required by applicable law or agreed to in writing, software 11 # distributed under the License is distributed on an "AS IS" BASIS, 12 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 # See the License for the specific language governing permissions and 14 # limitations under the License. 15 """Implementation of update command for updating gsutil.""" 16 17 from __future__ import absolute_import 18 19 import os 20 import shutil 21 import signal 22 import stat 23 import tarfile 24 import tempfile 25 import textwrap 26 27 import gslib 28 from gslib.command import Command 29 from gslib.cs_api_map import ApiSelector 30 from gslib.exception import CommandException 31 from gslib.sig_handling import RegisterSignalHandler 32 from gslib.util import CERTIFICATE_VALIDATION_ENABLED 33 from gslib.util import CompareVersions 34 from gslib.util import GetBotoConfigFileList 35 from gslib.util import GSUTIL_PUB_TARBALL 36 from gslib.util import IS_CYGWIN 37 from gslib.util import IS_WINDOWS 38 from gslib.util import LookUpGsutilVersion 39 from gslib.util import RELEASE_NOTES_URL 40 41 42 _SYNOPSIS = """ 43 gsutil update [-f] [-n] [url] 44 """ 45 46 _DETAILED_HELP_TEXT = (""" 47 <B>SYNOPSIS</B> 48 """ + _SYNOPSIS + """ 49 50 51 <B>DESCRIPTION</B> 52 The gsutil update command downloads the latest gsutil release, checks its 53 version, and offers to let you update to it if it differs from the version 54 you're currently running. 55 56 Once you say "Y" to the prompt of whether to install the update, the gsutil 57 update command locates where the running copy of gsutil is installed, 58 unpacks the new version into an adjacent directory, moves the previous version 59 aside, moves the new version to where the previous version was installed, 60 and removes the moved-aside old version. Because of this, users are cautioned 61 not to store data in the gsutil directory, since that data will be lost 62 when you update gsutil. (Some users change directories into the gsutil 63 directory to run the command. We advise against doing that, for this reason.) 64 Note also that the gsutil update command will refuse to run if it finds user 65 data in the gsutil directory. 66 67 By default gsutil update will retrieve the new code from 68 %s, but you can optionally specify a URL to use 69 instead. This is primarily used for distributing pre-release versions of 70 the code to a small group of early test users. 71 72 Note: gsutil periodically checks whether a more recent software update is 73 available. By default this check is performed every 30 days; you can change 74 (or disable) this check by editing the software_update_check_period variable 75 in the .boto config file. Note also that gsutil will only check for software 76 updates if stdin, stdout, and stderr are all connected to a TTY, to avoid 77 interfering with cron jobs, streaming transfers, and other cases where gsutil 78 input or output are redirected from/to files or pipes. Software update 79 periodic checks are also disabled by the gsutil -q option (see 80 'gsutil help options') 81 82 83 <B>OPTIONS</B> 84 -f Forces the update command to offer to let you update, even if you 85 have the most current copy already. This can be useful if you have 86 a corrupted local copy. 87 88 -n Causes update command to run without prompting [Y/n] whether to 89 continue if an update is available. 90 """ % GSUTIL_PUB_TARBALL) 91 92 93 class UpdateCommand(Command): 94 """Implementation of gsutil update command.""" 95 96 # Command specification. See base class for documentation. 97 command_spec = Command.CreateCommandSpec( 98 'update', 99 command_name_aliases=['refresh'], 100 usage_synopsis=_SYNOPSIS, 101 min_args=0, 102 max_args=1, 103 supported_sub_args='fn', 104 file_url_ok=True, 105 provider_url_ok=False, 106 urls_start_arg=0, 107 gs_api_support=[ApiSelector.XML, ApiSelector.JSON], 108 gs_default_api=ApiSelector.JSON, 109 ) 110 # Help specification. See help_provider.py for documentation. 111 help_spec = Command.HelpSpec( 112 help_name='update', 113 help_name_aliases=['refresh'], 114 help_type='command_help', 115 help_one_line_summary='Update to the latest gsutil release', 116 help_text=_DETAILED_HELP_TEXT, 117 subcommand_help_text={}, 118 ) 119 120 def _DisallowUpdataIfDataInGsutilDir(self): 121 """Disallows the update command if files not in the gsutil distro are found. 122 123 This prevents users from losing data if they are in the habit of running 124 gsutil from the gsutil directory and leaving data in that directory. 125 126 This will also detect someone attempting to run gsutil update from a git 127 repo, since the top-level directory will contain git files and dirs (like 128 .git) that are not distributed with gsutil. 129 130 Raises: 131 CommandException: if files other than those distributed with gsutil found. 132 """ 133 # Manifest includes recursive-includes of gslib. Directly add 134 # those to the list here so we will skip them in os.listdir() loop without 135 # having to build deeper handling of the MANIFEST file here. Also include 136 # 'third_party', which isn't present in manifest but gets added to the 137 # gsutil distro by the gsutil submodule configuration; and the MANIFEST.in 138 # and CHANGES.md files. 139 manifest_lines = ['gslib', 'third_party', 'MANIFEST.in', 'CHANGES.md'] 140 141 try: 142 with open(os.path.join(gslib.GSUTIL_DIR, 'MANIFEST.in'), 'r') as fp: 143 for line in fp: 144 if line.startswith('include '): 145 manifest_lines.append(line.split()[-1]) 146 except IOError: 147 self.logger.warn('MANIFEST.in not found in %s.\nSkipping user data ' 148 'check.\n', gslib.GSUTIL_DIR) 149 return 150 151 # Look just at top-level directory. We don't try to catch data dropped into 152 # subdirs (like gslib) because that would require deeper parsing of 153 # MANFFEST.in, and most users who drop data into gsutil dir do so at the top 154 # level directory. 155 for filename in os.listdir(gslib.GSUTIL_DIR): 156 if filename.endswith('.pyc'): 157 # Ignore compiled code. 158 continue 159 if filename not in manifest_lines: 160 raise CommandException('\n'.join(textwrap.wrap( 161 'A file (%s) that is not distributed with gsutil was found in ' 162 'the gsutil directory. The update command cannot run with user ' 163 'data in the gsutil directory.' % 164 os.path.join(gslib.GSUTIL_DIR, filename)))) 165 166 def _ExplainIfSudoNeeded(self, tf, dirs_to_remove): 167 """Explains what to do if sudo needed to update gsutil software. 168 169 Happens if gsutil was previously installed by a different user (typically if 170 someone originally installed in a shared file system location, using sudo). 171 172 Args: 173 tf: Opened TarFile. 174 dirs_to_remove: List of directories to remove. 175 176 Raises: 177 CommandException: if errors encountered. 178 """ 179 # If running under Windows or Cygwin we don't need (or have) sudo. 180 if IS_CYGWIN or IS_WINDOWS: 181 return 182 183 user_id = os.getuid() 184 if os.stat(gslib.GSUTIL_DIR).st_uid == user_id: 185 return 186 187 # Won't fail - this command runs after main startup code that insists on 188 # having a config file. 189 config_file_list = GetBotoConfigFileList() 190 config_files = ' '.join(config_file_list) 191 self._CleanUpUpdateCommand(tf, dirs_to_remove) 192 193 # Pick current protection of each boto config file for command that restores 194 # protection (rather than fixing at 600) to support use cases like how GCE 195 # installs a service account with an /etc/boto.cfg file protected to 644. 196 chmod_cmds = [] 197 for config_file in config_file_list: 198 mode = oct(stat.S_IMODE((os.stat(config_file)[stat.ST_MODE]))) 199 chmod_cmds.append('\n\tsudo chmod %s %s' % (mode, config_file)) 200 201 raise CommandException('\n'.join(textwrap.wrap( 202 'Since it was installed by a different user previously, you will need ' 203 'to update using the following commands. You will be prompted for your ' 204 'password, and the install will run as "root". If you\'re unsure what ' 205 'this means please ask your system administrator for help:')) + ( 206 '\n\tsudo chmod 0644 %s\n\tsudo env BOTO_CONFIG="%s" %s update' 207 '%s') % (config_files, config_files, self.gsutil_path, 208 ' '.join(chmod_cmds)), informational=True) 209 210 # This list is checked during gsutil update by doing a lowercased 211 # slash-left-stripped check. For example "/Dev" would match the "dev" entry. 212 unsafe_update_dirs = [ 213 'applications', 'auto', 'bin', 'boot', 'desktop', 'dev', 214 'documents and settings', 'etc', 'export', 'home', 'kernel', 'lib', 215 'lib32', 'library', 'lost+found', 'mach_kernel', 'media', 'mnt', 'net', 216 'null', 'network', 'opt', 'private', 'proc', 'program files', 'python', 217 'root', 'sbin', 'scripts', 'srv', 'sys', 'system', 'tmp', 'users', 'usr', 218 'var', 'volumes', 'win', 'win32', 'windows', 'winnt', 219 ] 220 221 def _EnsureDirsSafeForUpdate(self, dirs): 222 """Raises Exception if any of dirs is known to be unsafe for gsutil update. 223 224 This provides a fail-safe check to ensure we don't try to overwrite 225 or delete any important directories. (That shouldn't happen given the 226 way we construct tmp dirs, etc., but since the gsutil update cleanup 227 uses shutil.rmtree() it's prudent to add extra checks.) 228 229 Args: 230 dirs: List of directories to check. 231 232 Raises: 233 CommandException: If unsafe directory encountered. 234 """ 235 for d in dirs: 236 if not d: 237 d = 'null' 238 if d.lstrip(os.sep).lower() in self.unsafe_update_dirs: 239 raise CommandException('EnsureDirsSafeForUpdate: encountered unsafe ' 240 'directory (%s); aborting update' % d) 241 242 def _CleanUpUpdateCommand(self, tf, dirs_to_remove): 243 """Cleans up temp files etc. from running update command. 244 245 Args: 246 tf: Opened TarFile, or None if none currently open. 247 dirs_to_remove: List of directories to remove. 248 249 """ 250 if tf: 251 tf.close() 252 self._EnsureDirsSafeForUpdate(dirs_to_remove) 253 for directory in dirs_to_remove: 254 try: 255 shutil.rmtree(directory) 256 except OSError: 257 # Ignore errors while attempting to remove old dirs under Windows. They 258 # happen because of Windows exclusive file locking, and the update 259 # actually succeeds but just leaves the old versions around in the 260 # user's temp dir. 261 if not IS_WINDOWS: 262 raise 263 264 def RunCommand(self): 265 """Command entry point for the update command.""" 266 267 if gslib.IS_PACKAGE_INSTALL: 268 raise CommandException( 269 'The update command is only available for gsutil installed from a ' 270 'tarball. If you installed gsutil via another method, use the same ' 271 'method to update it.') 272 273 if os.environ.get('CLOUDSDK_WRAPPER') == '1': 274 raise CommandException( 275 'The update command is disabled for Cloud SDK installs. Please run ' 276 '"gcloud components update" to update it. Note: the Cloud SDK ' 277 'incorporates updates to the underlying tools approximately every 2 ' 278 'weeks, so if you are attempting to update to a recently created ' 279 'release / pre-release of gsutil it may not yet be available via ' 280 'the Cloud SDK.') 281 282 https_validate_certificates = CERTIFICATE_VALIDATION_ENABLED 283 if not https_validate_certificates: 284 raise CommandException( 285 'Your boto configuration has https_validate_certificates = False.\n' 286 'The update command cannot be run this way, for security reasons.') 287 288 self._DisallowUpdataIfDataInGsutilDir() 289 290 force_update = False 291 no_prompt = False 292 if self.sub_opts: 293 for o, unused_a in self.sub_opts: 294 if o == '-f': 295 force_update = True 296 if o == '-n': 297 no_prompt = True 298 299 dirs_to_remove = [] 300 tmp_dir = tempfile.mkdtemp() 301 dirs_to_remove.append(tmp_dir) 302 os.chdir(tmp_dir) 303 304 if not no_prompt: 305 self.logger.info('Checking for software update...') 306 if self.args: 307 update_from_url_str = self.args[0] 308 if not update_from_url_str.endswith('.tar.gz'): 309 raise CommandException( 310 'The update command only works with tar.gz files.') 311 for i, result in enumerate(self.WildcardIterator(update_from_url_str)): 312 if i > 0: 313 raise CommandException( 314 'Invalid update URL. Must name a single .tar.gz file.') 315 storage_url = result.storage_url 316 if storage_url.IsFileUrl() and not storage_url.IsDirectory(): 317 if not force_update: 318 raise CommandException( 319 ('"update" command does not support "file://" URLs without the ' 320 '-f option.')) 321 elif not (storage_url.IsCloudUrl() and storage_url.IsObject()): 322 raise CommandException( 323 'Invalid update object URL. Must name a single .tar.gz file.') 324 else: 325 update_from_url_str = GSUTIL_PUB_TARBALL 326 327 # Try to retrieve version info from tarball metadata; failing that; download 328 # the tarball and extract the VERSION file. The version lookup will fail 329 # when running the update system test, because it retrieves the tarball from 330 # a temp file rather than a cloud URL (files lack the version metadata). 331 tarball_version = LookUpGsutilVersion(self.gsutil_api, update_from_url_str) 332 if tarball_version: 333 tf = None 334 else: 335 tf = self._FetchAndOpenGsutilTarball(update_from_url_str) 336 tf.extractall() 337 with open(os.path.join('gsutil', 'VERSION'), 'r') as ver_file: 338 tarball_version = ver_file.read().strip() 339 340 if not force_update and gslib.VERSION == tarball_version: 341 self._CleanUpUpdateCommand(tf, dirs_to_remove) 342 if self.args: 343 raise CommandException('You already have %s installed.' % 344 update_from_url_str, informational=True) 345 else: 346 raise CommandException('You already have the latest gsutil release ' 347 'installed.', informational=True) 348 349 if not no_prompt: 350 (_, major) = CompareVersions(tarball_version, gslib.VERSION) 351 if major: 352 print('\n'.join(textwrap.wrap( 353 'This command will update to the "%s" version of gsutil at %s. ' 354 'NOTE: This a major new version, so it is strongly recommended ' 355 'that you review the release note details at %s before updating to ' 356 'this version, especially if you use gsutil in scripts.' 357 % (tarball_version, gslib.GSUTIL_DIR, RELEASE_NOTES_URL)))) 358 else: 359 print('This command will update to the "%s" version of\ngsutil at %s' 360 % (tarball_version, gslib.GSUTIL_DIR)) 361 self._ExplainIfSudoNeeded(tf, dirs_to_remove) 362 363 if no_prompt: 364 answer = 'y' 365 else: 366 answer = raw_input('Proceed? [y/N] ') 367 if not answer or answer.lower()[0] != 'y': 368 self._CleanUpUpdateCommand(tf, dirs_to_remove) 369 raise CommandException('Not running update.', informational=True) 370 371 if not tf: 372 tf = self._FetchAndOpenGsutilTarball(update_from_url_str) 373 374 # Ignore keyboard interrupts during the update to reduce the chance someone 375 # hitting ^C leaves gsutil in a broken state. 376 RegisterSignalHandler(signal.SIGINT, signal.SIG_IGN) 377 378 # gslib.GSUTIL_DIR lists the path where the code should end up (like 379 # /usr/local/gsutil), which is one level down from the relative path in the 380 # tarball (since the latter creates files in ./gsutil). So, we need to 381 # extract at the parent directory level. 382 gsutil_bin_parent_dir = os.path.normpath( 383 os.path.join(gslib.GSUTIL_DIR, '..')) 384 385 # Extract tarball to a temporary directory in a sibling to GSUTIL_DIR. 386 old_dir = tempfile.mkdtemp(dir=gsutil_bin_parent_dir) 387 new_dir = tempfile.mkdtemp(dir=gsutil_bin_parent_dir) 388 dirs_to_remove.append(old_dir) 389 dirs_to_remove.append(new_dir) 390 self._EnsureDirsSafeForUpdate(dirs_to_remove) 391 try: 392 tf.extractall(path=new_dir) 393 except Exception, e: 394 self._CleanUpUpdateCommand(tf, dirs_to_remove) 395 raise CommandException('Update failed: %s.' % e) 396 397 # For enterprise mode (shared/central) installation, users with 398 # different user/group than the installation user/group must be 399 # able to run gsutil so we need to do some permissions adjustments 400 # here. Since enterprise mode is not not supported for Windows 401 # users, we can skip this step when running on Windows, which 402 # avoids the problem that Windows has no find or xargs command. 403 if not IS_WINDOWS: 404 # Make all files and dirs in updated area owner-RW and world-R, and make 405 # all directories owner-RWX and world-RX. 406 for dirname, subdirs, filenames in os.walk(new_dir): 407 for filename in filenames: 408 fd = os.open(os.path.join(dirname, filename), os.O_RDONLY) 409 os.fchmod(fd, stat.S_IWRITE | stat.S_IRUSR | 410 stat.S_IRGRP | stat.S_IROTH) 411 os.close(fd) 412 for subdir in subdirs: 413 fd = os.open(os.path.join(dirname, subdir), os.O_RDONLY) 414 os.fchmod(fd, stat.S_IRWXU | stat.S_IXGRP | stat.S_IXOTH | 415 stat.S_IRGRP | stat.S_IROTH) 416 os.close(fd) 417 418 # Make main gsutil script owner-RWX and world-RX. 419 fd = os.open(os.path.join(new_dir, 'gsutil', 'gsutil'), os.O_RDONLY) 420 os.fchmod(fd, stat.S_IRWXU | stat.S_IRGRP | stat.S_IXGRP | 421 stat.S_IROTH | stat.S_IXOTH) 422 os.close(fd) 423 424 # Move old installation aside and new into place. 425 os.rename(gslib.GSUTIL_DIR, os.path.join(old_dir, 'old')) 426 os.rename(os.path.join(new_dir, 'gsutil'), gslib.GSUTIL_DIR) 427 self._CleanUpUpdateCommand(tf, dirs_to_remove) 428 RegisterSignalHandler(signal.SIGINT, signal.SIG_DFL) 429 self.logger.info('Update complete.') 430 return 0 431 432 def _FetchAndOpenGsutilTarball(self, update_from_url_str): 433 self.command_runner.RunNamedCommand( 434 'cp', [update_from_url_str, 'file://gsutil.tar.gz'], self.headers, 435 self.debug, skip_update_check=True) 436 # Note: tf is closed in _CleanUpUpdateCommand. 437 tf = tarfile.open('gsutil.tar.gz') 438 tf.errorlevel = 1 # So fatal tarball unpack errors raise exceptions. 439 return tf 440
