wire_format.py 6.9 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245
  1. # Protocol Buffers - Google's data interchange format
  2. # Copyright 2008 Google Inc. All rights reserved.
  3. #
  4. # Use of this source code is governed by a BSD-style
  5. # license that can be found in the LICENSE file or at
  6. # https://developers.google.com/open-source/licenses/bsd
  7. """Constants and static functions to support protocol buffer wire format."""
  8. __author__ = 'robinson@google.com (Will Robinson)'
  9. import struct
  10. from google.protobuf import descriptor
  11. from google.protobuf import message
  12. TAG_TYPE_BITS = 3 # Number of bits used to hold type info in a proto tag.
  13. TAG_TYPE_MASK = (1 << TAG_TYPE_BITS) - 1 # 0x7
  14. # These numbers identify the wire type of a protocol buffer value.
  15. # We use the least-significant TAG_TYPE_BITS bits of the varint-encoded
  16. # tag-and-type to store one of these WIRETYPE_* constants.
  17. # These values must match WireType enum in //google/protobuf/wire_format.h.
  18. WIRETYPE_VARINT = 0
  19. WIRETYPE_FIXED64 = 1
  20. WIRETYPE_LENGTH_DELIMITED = 2
  21. WIRETYPE_START_GROUP = 3
  22. WIRETYPE_END_GROUP = 4
  23. WIRETYPE_FIXED32 = 5
  24. _WIRETYPE_MAX = 5
  25. # Bounds for various integer types.
  26. INT32_MAX = int((1 << 31) - 1)
  27. INT32_MIN = int(-(1 << 31))
  28. UINT32_MAX = (1 << 32) - 1
  29. INT64_MAX = (1 << 63) - 1
  30. INT64_MIN = -(1 << 63)
  31. UINT64_MAX = (1 << 64) - 1
  32. # "struct" format strings that will encode/decode the specified formats.
  33. FORMAT_UINT32_LITTLE_ENDIAN = '<I'
  34. FORMAT_UINT64_LITTLE_ENDIAN = '<Q'
  35. FORMAT_FLOAT_LITTLE_ENDIAN = '<f'
  36. FORMAT_DOUBLE_LITTLE_ENDIAN = '<d'
  37. # We'll have to provide alternate implementations of AppendLittleEndian*() on
  38. # any architectures where these checks fail.
  39. if struct.calcsize(FORMAT_UINT32_LITTLE_ENDIAN) != 4:
  40. raise AssertionError('Format "I" is not a 32-bit number.')
  41. if struct.calcsize(FORMAT_UINT64_LITTLE_ENDIAN) != 8:
  42. raise AssertionError('Format "Q" is not a 64-bit number.')
  43. def PackTag(field_number, wire_type):
  44. """Returns an unsigned 32-bit integer that encodes the field number and
  45. wire type information in standard protocol message wire format.
  46. Args:
  47. field_number: Expected to be an integer in the range [1, 1 << 29)
  48. wire_type: One of the WIRETYPE_* constants.
  49. """
  50. if not 0 <= wire_type <= _WIRETYPE_MAX:
  51. raise message.EncodeError('Unknown wire type: %d' % wire_type)
  52. return (field_number << TAG_TYPE_BITS) | wire_type
  53. def UnpackTag(tag):
  54. """The inverse of PackTag(). Given an unsigned 32-bit number,
  55. returns a (field_number, wire_type) tuple.
  56. """
  57. return (tag >> TAG_TYPE_BITS), (tag & TAG_TYPE_MASK)
  58. def ZigZagEncode(value):
  59. """ZigZag Transform: Encodes signed integers so that they can be
  60. effectively used with varint encoding. See wire_format.h for
  61. more details.
  62. """
  63. if value >= 0:
  64. return value << 1
  65. return (value << 1) ^ (~0)
  66. def ZigZagDecode(value):
  67. """Inverse of ZigZagEncode()."""
  68. if not value & 0x1:
  69. return value >> 1
  70. return (value >> 1) ^ (~0)
  71. # The *ByteSize() functions below return the number of bytes required to
  72. # serialize "field number + type" information and then serialize the value.
  73. def Int32ByteSize(field_number, int32):
  74. return Int64ByteSize(field_number, int32)
  75. def Int32ByteSizeNoTag(int32):
  76. return _VarUInt64ByteSizeNoTag(0xffffffffffffffff & int32)
  77. def Int64ByteSize(field_number, int64):
  78. # Have to convert to uint before calling UInt64ByteSize().
  79. return UInt64ByteSize(field_number, 0xffffffffffffffff & int64)
  80. def UInt32ByteSize(field_number, uint32):
  81. return UInt64ByteSize(field_number, uint32)
  82. def UInt64ByteSize(field_number, uint64):
  83. return TagByteSize(field_number) + _VarUInt64ByteSizeNoTag(uint64)
  84. def SInt32ByteSize(field_number, int32):
  85. return UInt32ByteSize(field_number, ZigZagEncode(int32))
  86. def SInt64ByteSize(field_number, int64):
  87. return UInt64ByteSize(field_number, ZigZagEncode(int64))
  88. def Fixed32ByteSize(field_number, fixed32):
  89. return TagByteSize(field_number) + 4
  90. def Fixed64ByteSize(field_number, fixed64):
  91. return TagByteSize(field_number) + 8
  92. def SFixed32ByteSize(field_number, sfixed32):
  93. return TagByteSize(field_number) + 4
  94. def SFixed64ByteSize(field_number, sfixed64):
  95. return TagByteSize(field_number) + 8
  96. def FloatByteSize(field_number, flt):
  97. return TagByteSize(field_number) + 4
  98. def DoubleByteSize(field_number, double):
  99. return TagByteSize(field_number) + 8
  100. def BoolByteSize(field_number, b):
  101. return TagByteSize(field_number) + 1
  102. def EnumByteSize(field_number, enum):
  103. return UInt32ByteSize(field_number, enum)
  104. def StringByteSize(field_number, string):
  105. return BytesByteSize(field_number, string.encode('utf-8'))
  106. def BytesByteSize(field_number, b):
  107. return (TagByteSize(field_number)
  108. + _VarUInt64ByteSizeNoTag(len(b))
  109. + len(b))
  110. def GroupByteSize(field_number, message):
  111. return (2 * TagByteSize(field_number) # START and END group.
  112. + message.ByteSize())
  113. def MessageByteSize(field_number, message):
  114. return (TagByteSize(field_number)
  115. + _VarUInt64ByteSizeNoTag(message.ByteSize())
  116. + message.ByteSize())
  117. def MessageSetItemByteSize(field_number, msg):
  118. # First compute the sizes of the tags.
  119. # There are 2 tags for the beginning and ending of the repeated group, that
  120. # is field number 1, one with field number 2 (type_id) and one with field
  121. # number 3 (message).
  122. total_size = (2 * TagByteSize(1) + TagByteSize(2) + TagByteSize(3))
  123. # Add the number of bytes for type_id.
  124. total_size += _VarUInt64ByteSizeNoTag(field_number)
  125. message_size = msg.ByteSize()
  126. # The number of bytes for encoding the length of the message.
  127. total_size += _VarUInt64ByteSizeNoTag(message_size)
  128. # The size of the message.
  129. total_size += message_size
  130. return total_size
  131. def TagByteSize(field_number):
  132. """Returns the bytes required to serialize a tag with this field number."""
  133. # Just pass in type 0, since the type won't affect the tag+type size.
  134. return _VarUInt64ByteSizeNoTag(PackTag(field_number, 0))
  135. # Private helper function for the *ByteSize() functions above.
  136. def _VarUInt64ByteSizeNoTag(uint64):
  137. """Returns the number of bytes required to serialize a single varint
  138. using boundary value comparisons. (unrolled loop optimization -WPierce)
  139. uint64 must be unsigned.
  140. """
  141. if uint64 <= 0x7f: return 1
  142. if uint64 <= 0x3fff: return 2
  143. if uint64 <= 0x1fffff: return 3
  144. if uint64 <= 0xfffffff: return 4
  145. if uint64 <= 0x7ffffffff: return 5
  146. if uint64 <= 0x3ffffffffff: return 6
  147. if uint64 <= 0x1ffffffffffff: return 7
  148. if uint64 <= 0xffffffffffffff: return 8
  149. if uint64 <= 0x7fffffffffffffff: return 9
  150. if uint64 > UINT64_MAX:
  151. raise message.EncodeError('Value out of range: %d' % uint64)
  152. return 10
  153. NON_PACKABLE_TYPES = (
  154. descriptor.FieldDescriptor.TYPE_STRING,
  155. descriptor.FieldDescriptor.TYPE_GROUP,
  156. descriptor.FieldDescriptor.TYPE_MESSAGE,
  157. descriptor.FieldDescriptor.TYPE_BYTES
  158. )
  159. def IsTypePackable(field_type):
  160. """Return true iff packable = true is valid for fields of this type.
  161. Args:
  162. field_type: a FieldDescriptor::Type value.
  163. Returns:
  164. True iff fields of this type are packable.
  165. """
  166. return field_type not in NON_PACKABLE_TYPES