Home | History | Annotate | Download | only in internal 
      1 # Protocol Buffers - Google's data interchange format
      2 # Copyright 2008 Google Inc.  All rights reserved.
      3 # https://developers.google.com/protocol-buffers/
      4 #
      5 # Redistribution and use in source and binary forms, with or without
      6 # modification, are permitted provided that the following conditions are
      7 # met:
      8 #
      9 #     * Redistributions of source code must retain the above copyright
     10 # notice, this list of conditions and the following disclaimer.
     11 #     * Redistributions in binary form must reproduce the above
     12 # copyright notice, this list of conditions and the following disclaimer
     13 # in the documentation and/or other materials provided with the
     14 # distribution.
     15 #     * Neither the name of Google Inc. nor the names of its
     16 # contributors may be used to endorse or promote products derived from
     17 # this software without specific prior written permission.
     18 #
     19 # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
     20 # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
     21 # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
     22 # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
     23 # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
     24 # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
     25 # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
     26 # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
     27 # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
     28 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
     29 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
     30 
     31 """Constants and static functions to support protocol buffer wire format."""
     32 
     33 __author__ = 'robinson (at] google.com (Will Robinson)'
     34 
     35 import struct
     36 from google.protobuf import descriptor
     37 from google.protobuf import message
     38 
     39 
     40 TAG_TYPE_BITS = 3  # Number of bits used to hold type info in a proto tag.
     41 TAG_TYPE_MASK = (1 << TAG_TYPE_BITS) - 1  # 0x7
     42 
     43 # These numbers identify the wire type of a protocol buffer value.
     44 # We use the least-significant TAG_TYPE_BITS bits of the varint-encoded
     45 # tag-and-type to store one of these WIRETYPE_* constants.
     46 # These values must match WireType enum in google/protobuf/wire_format.h.
     47 WIRETYPE_VARINT = 0
     48 WIRETYPE_FIXED64 = 1
     49 WIRETYPE_LENGTH_DELIMITED = 2
     50 WIRETYPE_START_GROUP = 3
     51 WIRETYPE_END_GROUP = 4
     52 WIRETYPE_FIXED32 = 5
     53 _WIRETYPE_MAX = 5
     54 
     55 
     56 # Bounds for various integer types.
     57 INT32_MAX = int((1 << 31) - 1)
     58 INT32_MIN = int(-(1 << 31))
     59 UINT32_MAX = (1 << 32) - 1
     60 
     61 INT64_MAX = (1 << 63) - 1
     62 INT64_MIN = -(1 << 63)
     63 UINT64_MAX = (1 << 64) - 1
     64 
     65 # "struct" format strings that will encode/decode the specified formats.
     66 FORMAT_UINT32_LITTLE_ENDIAN = '<I'
     67 FORMAT_UINT64_LITTLE_ENDIAN = '<Q'
     68 FORMAT_FLOAT_LITTLE_ENDIAN = '<f'
     69 FORMAT_DOUBLE_LITTLE_ENDIAN = '<d'
     70 
     71 
     72 # We'll have to provide alternate implementations of AppendLittleEndian*() on
     73 # any architectures where these checks fail.
     74 if struct.calcsize(FORMAT_UINT32_LITTLE_ENDIAN) != 4:
     75   raise AssertionError('Format "I" is not a 32-bit number.')
     76 if struct.calcsize(FORMAT_UINT64_LITTLE_ENDIAN) != 8:
     77   raise AssertionError('Format "Q" is not a 64-bit number.')
     78 
     79 
     80 def PackTag(field_number, wire_type):
     81   """Returns an unsigned 32-bit integer that encodes the field number and
     82   wire type information in standard protocol message wire format.
     83 
     84   Args:
     85     field_number: Expected to be an integer in the range [1, 1 << 29)
     86     wire_type: One of the WIRETYPE_* constants.
     87   """
     88   if not 0 <= wire_type <= _WIRETYPE_MAX:
     89     raise message.EncodeError('Unknown wire type: %d' % wire_type)
     90   return (field_number << TAG_TYPE_BITS) | wire_type
     91 
     92 
     93 def UnpackTag(tag):
     94   """The inverse of PackTag().  Given an unsigned 32-bit number,
     95   returns a (field_number, wire_type) tuple.
     96   """
     97   return (tag >> TAG_TYPE_BITS), (tag & TAG_TYPE_MASK)
     98 
     99 
    100 def ZigZagEncode(value):
    101   """ZigZag Transform:  Encodes signed integers so that they can be
    102   effectively used with varint encoding.  See wire_format.h for
    103   more details.
    104   """
    105   if value >= 0:
    106     return value << 1
    107   return (value << 1) ^ (~0)
    108 
    109 
    110 def ZigZagDecode(value):
    111   """Inverse of ZigZagEncode()."""
    112   if not value & 0x1:
    113     return value >> 1
    114   return (value >> 1) ^ (~0)
    115 
    116 
    117 
    118 # The *ByteSize() functions below return the number of bytes required to
    119 # serialize "field number + type" information and then serialize the value.
    120 
    121 
    122 def Int32ByteSize(field_number, int32):
    123   return Int64ByteSize(field_number, int32)
    124 
    125 
    126 def Int32ByteSizeNoTag(int32):
    127   return _VarUInt64ByteSizeNoTag(0xffffffffffffffff & int32)
    128 
    129 
    130 def Int64ByteSize(field_number, int64):
    131   # Have to convert to uint before calling UInt64ByteSize().
    132   return UInt64ByteSize(field_number, 0xffffffffffffffff & int64)
    133 
    134 
    135 def UInt32ByteSize(field_number, uint32):
    136   return UInt64ByteSize(field_number, uint32)
    137 
    138 
    139 def UInt64ByteSize(field_number, uint64):
    140   return TagByteSize(field_number) + _VarUInt64ByteSizeNoTag(uint64)
    141 
    142 
    143 def SInt32ByteSize(field_number, int32):
    144   return UInt32ByteSize(field_number, ZigZagEncode(int32))
    145 
    146 
    147 def SInt64ByteSize(field_number, int64):
    148   return UInt64ByteSize(field_number, ZigZagEncode(int64))
    149 
    150 
    151 def Fixed32ByteSize(field_number, fixed32):
    152   return TagByteSize(field_number) + 4
    153 
    154 
    155 def Fixed64ByteSize(field_number, fixed64):
    156   return TagByteSize(field_number) + 8
    157 
    158 
    159 def SFixed32ByteSize(field_number, sfixed32):
    160   return TagByteSize(field_number) + 4
    161 
    162 
    163 def SFixed64ByteSize(field_number, sfixed64):
    164   return TagByteSize(field_number) + 8
    165 
    166 
    167 def FloatByteSize(field_number, flt):
    168   return TagByteSize(field_number) + 4
    169 
    170 
    171 def DoubleByteSize(field_number, double):
    172   return TagByteSize(field_number) + 8
    173 
    174 
    175 def BoolByteSize(field_number, b):
    176   return TagByteSize(field_number) + 1
    177 
    178 
    179 def EnumByteSize(field_number, enum):
    180   return UInt32ByteSize(field_number, enum)
    181 
    182 
    183 def StringByteSize(field_number, string):
    184   return BytesByteSize(field_number, string.encode('utf-8'))
    185 
    186 
    187 def BytesByteSize(field_number, b):
    188   return (TagByteSize(field_number)
    189           + _VarUInt64ByteSizeNoTag(len(b))
    190           + len(b))
    191 
    192 
    193 def GroupByteSize(field_number, message):
    194   return (2 * TagByteSize(field_number)  # START and END group.
    195           + message.ByteSize())
    196 
    197 
    198 def MessageByteSize(field_number, message):
    199   return (TagByteSize(field_number)
    200           + _VarUInt64ByteSizeNoTag(message.ByteSize())
    201           + message.ByteSize())
    202 
    203 
    204 def MessageSetItemByteSize(field_number, msg):
    205   # First compute the sizes of the tags.
    206   # There are 2 tags for the beginning and ending of the repeated group, that
    207   # is field number 1, one with field number 2 (type_id) and one with field
    208   # number 3 (message).
    209   total_size = (2 * TagByteSize(1) + TagByteSize(2) + TagByteSize(3))
    210 
    211   # Add the number of bytes for type_id.
    212   total_size += _VarUInt64ByteSizeNoTag(field_number)
    213 
    214   message_size = msg.ByteSize()
    215 
    216   # The number of bytes for encoding the length of the message.
    217   total_size += _VarUInt64ByteSizeNoTag(message_size)
    218 
    219   # The size of the message.
    220   total_size += message_size
    221   return total_size
    222 
    223 
    224 def TagByteSize(field_number):
    225   """Returns the bytes required to serialize a tag with this field number."""
    226   # Just pass in type 0, since the type won't affect the tag+type size.
    227   return _VarUInt64ByteSizeNoTag(PackTag(field_number, 0))
    228 
    229 
    230 # Private helper function for the *ByteSize() functions above.
    231 
    232 def _VarUInt64ByteSizeNoTag(uint64):
    233   """Returns the number of bytes required to serialize a single varint
    234   using boundary value comparisons. (unrolled loop optimization -WPierce)
    235   uint64 must be unsigned.
    236   """
    237   if uint64 <= 0x7f: return 1
    238   if uint64 <= 0x3fff: return 2
    239   if uint64 <= 0x1fffff: return 3
    240   if uint64 <= 0xfffffff: return 4
    241   if uint64 <= 0x7ffffffff: return 5
    242   if uint64 <= 0x3ffffffffff: return 6
    243   if uint64 <= 0x1ffffffffffff: return 7
    244   if uint64 <= 0xffffffffffffff: return 8
    245   if uint64 <= 0x7fffffffffffffff: return 9
    246   if uint64 > UINT64_MAX:
    247     raise message.EncodeError('Value out of range: %d' % uint64)
    248   return 10
    249 
    250 
    251 NON_PACKABLE_TYPES = (
    252   descriptor.FieldDescriptor.TYPE_STRING,
    253   descriptor.FieldDescriptor.TYPE_GROUP,
    254   descriptor.FieldDescriptor.TYPE_MESSAGE,
    255   descriptor.FieldDescriptor.TYPE_BYTES
    256 )
    257 
    258 
    259 def IsTypePackable(field_type):
    260   """Return true iff packable = true is valid for fields of this type.
    261 
    262   Args:
    263     field_type: a FieldDescriptor::Type value.
    264 
    265   Returns:
    266     True iff fields of this type are packable.
    267   """
    268   return field_type not in NON_PACKABLE_TYPES
    269