Home | History | Annotate | Download | only in commands 
      1 # -*- coding: utf-8 -*-
      2 # Copyright 2011 Google Inc. All Rights Reserved.
      3 #
      4 # Licensed under the Apache License, Version 2.0 (the "License");
      5 # you may not use this file except in compliance with the License.
      6 # You may obtain a copy of the License at
      7 #
      8 #     http://www.apache.org/licenses/LICENSE-2.0
      9 #
     10 # Unless required by applicable law or agreed to in writing, software
     11 # distributed under the License is distributed on an "AS IS" BASIS,
     12 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
     13 # See the License for the specific language governing permissions and
     14 # limitations under the License.
     15 """Implementation of logging configuration command for buckets."""
     16 
     17 from __future__ import absolute_import
     18 
     19 import sys
     20 
     21 from apitools.base.py import encoding
     22 
     23 from gslib.command import Command
     24 from gslib.command_argument import CommandArgument
     25 from gslib.cs_api_map import ApiSelector
     26 from gslib.exception import CommandException
     27 from gslib.help_provider import CreateHelpText
     28 from gslib.storage_url import StorageUrlFromString
     29 from gslib.third_party.storage_apitools import storage_v1_messages as apitools_messages
     30 from gslib.util import NO_MAX
     31 from gslib.util import UrlsAreForSingleProvider
     32 
     33 _SET_SYNOPSIS = """
     34   gsutil logging set on -b logging_bucket [-o log_object_prefix] url...
     35   gsutil logging set off url...
     36 """
     37 
     38 _GET_SYNOPSIS = """
     39   gsutil logging get url
     40 """
     41 
     42 _SYNOPSIS = _SET_SYNOPSIS + _GET_SYNOPSIS.lstrip('\n') + '\n'
     43 
     44 _SET_DESCRIPTION = """
     45 <B>SET</B>
     46   The set sub-command has two sub-commands:
     47 
     48 <B>ON</B>
     49   The "gsutil set on" command will enable access logging of the
     50   buckets named by the specified URLs, outputting log files in the specified
     51   logging_bucket. logging_bucket must already exist, and all URLs must name
     52   buckets (e.g., gs://bucket). The required bucket parameter specifies the
     53   bucket to which the logs are written, and the optional log_object_prefix
     54   parameter specifies the prefix for log object names. The default prefix
     55   is the bucket name. For example, the command:
     56 
     57     gsutil logging set on -b gs://my_logging_bucket -o AccessLog \\
     58         gs://my_bucket1 gs://my_bucket2
     59 
     60   will cause all read and write activity to objects in gs://mybucket1 and
     61   gs://mybucket2 to be logged to objects prefixed with the name "AccessLog",
     62   with those log objects written to the bucket gs://my_logging_bucket.
     63 
     64   Next, you need to grant cloud-storage-analytics (at] google.com write access to
     65   the log bucket, using this command:
     66 
     67     gsutil acl ch -g cloud-storage-analytics (at] google.com:W gs://my_logging_bucket
     68 
     69   Note that log data may contain sensitive information, so you should make
     70   sure to set an appropriate default bucket ACL to protect that data. (See
     71   "gsutil help defacl".)
     72 
     73 <B>OFF</B>
     74   This command will disable access logging of the buckets named by the
     75   specified URLs. All URLs must name buckets (e.g., gs://bucket).
     76 
     77   No logging data is removed from the log buckets when you disable logging,
     78   but Google Cloud Storage will stop delivering new logs once you have
     79   run this command.
     80 
     81 """
     82 
     83 _GET_DESCRIPTION = """
     84 <B>GET</B>
     85   If logging is enabled for the specified bucket url, the server responds
     86   with a JSON document that looks something like this:
     87 
     88     {
     89       "logObjectPrefix": "AccessLog",
     90       "logBucket": "my_logging_bucket"
     91     }
     92 
     93   You can download log data from your log bucket using the gsutil cp command.
     94 
     95 """
     96 
     97 _DESCRIPTION = """
     98   Google Cloud Storage offers access logs and storage data in the form of
     99   CSV files that you can download and view. Access logs provide information
    100   for all of the requests made on a specified bucket in the last 24 hours,
    101   while the storage logs provide information about the storage consumption of
    102   that bucket for the last 24 hour period. The logs and storage data files
    103   are automatically created as new objects in a bucket that you specify, in
    104   24 hour intervals.
    105 
    106   The logging command has two sub-commands:
    107 """ + _SET_DESCRIPTION + _GET_DESCRIPTION + """
    108 
    109 <B>ACCESS LOG AND STORAGE DATA FIELDS</B>
    110   For a complete list of access log fields and storage data fields, see:
    111   https://developers.google.com/storage/docs/accesslogs#reviewing
    112 """
    113 
    114 _DETAILED_HELP_TEXT = CreateHelpText(_SYNOPSIS, _DESCRIPTION)
    115 
    116 _get_help_text = CreateHelpText(_GET_SYNOPSIS, _GET_DESCRIPTION)
    117 _set_help_text = CreateHelpText(_SET_SYNOPSIS, _SET_DESCRIPTION)
    118 
    119 
    120 class LoggingCommand(Command):
    121   """Implementation of gsutil logging command."""
    122 
    123   # Command specification. See base class for documentation.
    124   command_spec = Command.CreateCommandSpec(
    125       'logging',
    126       command_name_aliases=['disablelogging', 'enablelogging', 'getlogging'],
    127       usage_synopsis=_SYNOPSIS,
    128       min_args=2,
    129       max_args=NO_MAX,
    130       supported_sub_args='b:o:',
    131       file_url_ok=False,
    132       provider_url_ok=False,
    133       urls_start_arg=0,
    134       gs_api_support=[ApiSelector.XML, ApiSelector.JSON],
    135       gs_default_api=ApiSelector.JSON,
    136       argparse_arguments=[
    137           CommandArgument('mode', choices=['on', 'off']),
    138           CommandArgument.MakeZeroOrMoreCloudBucketURLsArgument()
    139       ]
    140   )
    141   # Help specification. See help_provider.py for documentation.
    142   help_spec = Command.HelpSpec(
    143       help_name='logging',
    144       help_name_aliases=['loggingconfig', 'logs', 'log', 'getlogging',
    145                          'enablelogging', 'disablelogging'],
    146       help_type='command_help',
    147       help_one_line_summary='Configure or retrieve logging on buckets',
    148       help_text=_DETAILED_HELP_TEXT,
    149       subcommand_help_text={'get': _get_help_text, 'set': _set_help_text},
    150   )
    151 
    152   def _Get(self):
    153     """Gets logging configuration for a bucket."""
    154     bucket_url, bucket_metadata = self.GetSingleBucketUrlFromArg(
    155         self.args[0], bucket_fields=['logging'])
    156 
    157     if bucket_url.scheme == 's3':
    158       sys.stdout.write(self.gsutil_api.XmlPassThroughGetLogging(
    159           bucket_url, provider=bucket_url.scheme))
    160     else:
    161       if (bucket_metadata.logging and bucket_metadata.logging.logBucket and
    162           bucket_metadata.logging.logObjectPrefix):
    163         sys.stdout.write(str(encoding.MessageToJson(
    164             bucket_metadata.logging)) + '\n')
    165       else:
    166         sys.stdout.write('%s has no logging configuration.\n' % bucket_url)
    167     return 0
    168 
    169   def _Enable(self):
    170     """Enables logging configuration for a bucket."""
    171     # Disallow multi-provider 'logging set on' calls, because the schemas
    172     # differ.
    173     if not UrlsAreForSingleProvider(self.args):
    174       raise CommandException('"logging set on" command spanning providers not '
    175                              'allowed.')
    176     target_bucket_url = None
    177     target_prefix = None
    178     for opt, opt_arg in self.sub_opts:
    179       if opt == '-b':
    180         target_bucket_url = StorageUrlFromString(opt_arg)
    181       if opt == '-o':
    182         target_prefix = opt_arg
    183 
    184     if not target_bucket_url:
    185       raise CommandException('"logging set on" requires \'-b <log_bucket>\' '
    186                              'option')
    187     if not target_bucket_url.IsBucket():
    188       raise CommandException('-b option must specify a bucket URL.')
    189 
    190     # Iterate over URLs, expanding wildcards and setting logging on each.
    191     some_matched = False
    192     for url_str in self.args:
    193       bucket_iter = self.GetBucketUrlIterFromArg(url_str, bucket_fields=['id'])
    194       for blr in bucket_iter:
    195         url = blr.storage_url
    196         some_matched = True
    197         self.logger.info('Enabling logging on %s...', blr)
    198         logging = apitools_messages.Bucket.LoggingValue(
    199             logBucket=target_bucket_url.bucket_name,
    200             logObjectPrefix=target_prefix or url.bucket_name)
    201 
    202         bucket_metadata = apitools_messages.Bucket(logging=logging)
    203         self.gsutil_api.PatchBucket(url.bucket_name, bucket_metadata,
    204                                     provider=url.scheme, fields=['id'])
    205     if not some_matched:
    206       raise CommandException('No URLs matched')
    207     return 0
    208 
    209   def _Disable(self):
    210     """Disables logging configuration for a bucket."""
    211     # Iterate over URLs, expanding wildcards, and disabling logging on each.
    212     some_matched = False
    213     for url_str in self.args:
    214       bucket_iter = self.GetBucketUrlIterFromArg(url_str, bucket_fields=['id'])
    215       for blr in bucket_iter:
    216         url = blr.storage_url
    217         some_matched = True
    218         self.logger.info('Disabling logging on %s...', blr)
    219         logging = apitools_messages.Bucket.LoggingValue()
    220 
    221         bucket_metadata = apitools_messages.Bucket(logging=logging)
    222         self.gsutil_api.PatchBucket(url.bucket_name, bucket_metadata,
    223                                     provider=url.scheme, fields=['id'])
    224     if not some_matched:
    225       raise CommandException('No URLs matched')
    226     return 0
    227 
    228   def RunCommand(self):
    229     """Command entry point for the logging command."""
    230     # Parse the subcommand and alias for the new logging command.
    231     action_subcommand = self.args.pop(0)
    232     if action_subcommand == 'get':
    233       func = self._Get
    234     elif action_subcommand == 'set':
    235       state_subcommand = self.args.pop(0)
    236       if not self.args:
    237         self.RaiseWrongNumberOfArgumentsException()
    238       if state_subcommand == 'on':
    239         func = self._Enable
    240       elif state_subcommand == 'off':
    241         func = self._Disable
    242       else:
    243         raise CommandException((
    244             'Invalid subcommand "%s" for the "%s %s" command.\n'
    245             'See "gsutil help logging".') % (
    246                 state_subcommand, self.command_name, action_subcommand))
    247     else:
    248       raise CommandException(('Invalid subcommand "%s" for the %s command.\n'
    249                               'See "gsutil help logging".') %
    250                              (action_subcommand, self.command_name))
    251     self.ParseSubOpts(check_args=True)
    252     func()
    253     return 0
    254