py2 use conversion functions to convert bytes and strings

charleslaw · charleslaw · commit 927ac6376ffc · 2013-03-18T09:08:31.000-07:00
diff --git a/python/google/protobuf/internal/decoder.py b/python/google/protobuf/internal/decoder.py
@@ -83,6 +83,7 @@
 import struct
 from google.protobuf.internal import encoder
 from google.protobuf.internal import wire_format
+from google.protobuf.internal.utils import bytestr_to_string, byte_ord
 from google.protobuf import message
 
 
@@ -108,12 +109,11 @@ def _VarintDecoder(mask):
   decoder returns a (value, new_pos) pair.
   """
 
-  local_ord = ord
   def DecodeVarint(buffer, pos):
     result = 0
     shift = 0
     while 1:
-      b = local_ord(buffer[pos])
+      b = byte_ord(buffer[pos])
       result |= ((b & 0x7f) << shift)
       pos += 1
       if not (b & 0x80):
@@ -128,12 +128,11 @@ def DecodeVarint(buffer, pos):
 def _SignedVarintDecoder(mask):
   """Like _VarintDecoder() but decodes signed values."""
 
-  local_ord = ord
   def DecodeVarint(buffer, pos):
     result = 0
     shift = 0
     while 1:
-      b = local_ord(buffer[pos])
+      b = byte_ord(buffer[pos])
       result |= ((b & 0x7f) << shift)
       pos += 1
       if not (b & 0x80):
@@ -169,7 +168,7 @@ def ReadTag(buffer, pos):
   """
 
   start = pos
-  while ord(buffer[pos]) & 0x80:
+  while byte_ord(buffer[pos]) & 0x80:
     pos += 1
   pos += 1
   return (buffer[start:pos], pos)
@@ -378,7 +377,6 @@ def StringDecoder(field_number, is_repeated, is_packed, key, new_default):
   """Returns a decoder for a string field."""
 
   local_DecodeVarint = _DecodeVarint
-  local_unicode = unicode
 
   assert not is_packed
   if is_repeated:
@@ -394,7 +392,7 @@ def DecodeRepeatedField(buffer, pos, end, message, field_dict):
         new_pos = pos + size
         if new_pos > end:
           raise _DecodeError('Truncated string.')
-        value.append(local_unicode(buffer[pos:new_pos], 'utf-8'))
+        value.append(bytestr_to_string(buffer[pos:new_pos]))
         # Predict that the next tag is another copy of the same repeated field.
         pos = new_pos + tag_len
         if buffer[new_pos:pos] != tag_bytes or new_pos == end:
@@ -407,7 +405,7 @@ def DecodeField(buffer, pos, end, message, field_dict):
       new_pos = pos + size
       if new_pos > end:
         raise _DecodeError('Truncated string.')
-      field_dict[key] = local_unicode(buffer[pos:new_pos], 'utf-8')
+      field_dict[key] = bytestr_to_string(buffer[pos:new_pos])
       return new_pos
     return DecodeField
 
@@ -626,7 +624,7 @@ def DecodeItem(buffer, pos, end, message, field_dict):
 def _SkipVarint(buffer, pos, end):
   """Skip a varint value.  Returns the new position."""
 
-  while ord(buffer[pos]) & 0x80:
+  while byte_ord(buffer[pos]) & 0x80:
     pos += 1
   pos += 1
   if pos > end:
@@ -693,7 +691,6 @@ def _FieldSkipper():
       ]
 
   wiretype_mask = wire_format.TAG_TYPE_MASK
-  local_ord = ord
 
   def SkipField(buffer, pos, end, tag_bytes):
     """Skips a field with the specified tag.
@@ -706,7 +703,7 @@ def SkipField(buffer, pos, end, tag_bytes):
     """
 
     # The wire type is always in the first byte since varints are little-endian.
-    wire_type = local_ord(tag_bytes[0]) & wiretype_mask
+    wire_type = byte_ord(tag_bytes[0]) & wiretype_mask
     return WIRETYPE_TO_SKIPPER[wire_type](buffer, pos, end)
 
   return SkipField
diff --git a/python/google/protobuf/internal/encoder.py b/python/google/protobuf/internal/encoder.py
@@ -68,6 +68,7 @@
 
 import struct
 from google.protobuf.internal import wire_format
+from google.protobuf.internal.utils import string_to_bytestr
 
 
 # This will overflow and thus become IEEE-754 "infinity".  We would use
@@ -659,14 +660,14 @@ def StringEncoder(field_number, is_repeated, is_packed):
   if is_repeated:
     def EncodeRepeatedField(write, value):
       for element in value:
-        encoded = element.encode('utf-8')
+        encoded = string_to_bytestr(element)
         write(tag)
         local_EncodeVarint(write, local_len(encoded))
         write(encoded)
     return EncodeRepeatedField
   else:
     def EncodeField(write, value):
-      encoded = value.encode('utf-8')
+      encoded = string_to_bytestr(value)
       write(tag)
       local_EncodeVarint(write, local_len(encoded))
       return write(encoded)
diff --git a/python/google/protobuf/internal/utils.py b/python/google/protobuf/internal/utils.py
@@ -6,3 +6,19 @@
     def cmp(a, b):
         return (a > b) - (a < b)
 
+
+
+def bytestr_to_string(bytestr):
+    return unicode(bytestr, 'utf-8')
+
+def string_to_bytestr(string):
+    #return b''.join([chr(b) for b in string.encode('utf-8')])
+    return string.encode('utf-8')
+
+
+def byte_ord(bytes):
+    return ord(bytes)
+
+def char_byte(char):
+    return char
+
diff --git a/python/google/protobuf/internal/wire_format.py b/python/google/protobuf/internal/wire_format.py
@@ -35,6 +35,7 @@
 import struct
 from google.protobuf import descriptor
 from google.protobuf import message
+from google.protobuf.internal.utils import string_to_bytestr
 
 
 TAG_TYPE_BITS = 3  # Number of bits used to hold type info in a proto tag.
@@ -181,7 +182,7 @@ def EnumByteSize(field_number, enum):
 
 
 def StringByteSize(field_number, string):
-  return BytesByteSize(field_number, string.encode('utf-8'))
+  return BytesByteSize(field_number, string_to_bytestr(string))
 
 
 def BytesByteSize(field_number, b):
diff --git a/python/google/protobuf/text_format.py b/python/google/protobuf/text_format.py
@@ -37,6 +37,7 @@
 
 from collections import deque
 from google.protobuf.internal import type_checkers
+from google.protobuf.internal.utils import char_byte, bytestr_to_string
 from google.protobuf import descriptor
 
 __all__ = [ 'MessageToString', 'PrintMessage', 'PrintField',
@@ -403,7 +404,7 @@ def LookingAtInteger(self):
     if not self.token:
       return False
     c = self.token[0]
-    return (c >= '0' and c <= '9') or c == '-' or c == '+'
+    return (c >= char_byte('0') and c <= char_byte('9')) or c == char_byte('-') or c == char_byte('+')
 
   def ConsumeIdentifier(self):
     """Consumes protocol message field identifier.
@@ -538,9 +539,9 @@ def ConsumeString(self):
     Raises:
       ParseError: If a string value couldn't be consumed.
     """
-    bytes = self.ConsumeByteString()
+    bytestr = self.ConsumeByteString()
     try:
-      return unicode(bytes, 'utf-8')
+      return bytestr_to_string(bytestr)
     except UnicodeDecodeError, e:
       raise self._StringParseError(e)
 
@@ -554,7 +555,7 @@ def ConsumeByteString(self):
       ParseError: If a byte array value couldn't be consumed.
     """
     list = [self._ConsumeSingleByteString()]
-    while len(self.token) > 0 and self.token[0] in ('\'', '"'):
+    while len(self.token) > 0 and self.token[0] in (char_byte('\''), char_byte('"')):
       list.append(self._ConsumeSingleByteString())
     return "".join(list)
 
@@ -566,7 +567,7 @@ def _ConsumeSingleByteString(self):
     method only consumes one token.
     """
     text = self.token
-    if len(text) < 1 or text[0] not in ('\'', '"'):
+    if len(text) < 1 or text[0] not in (char_byte('\''), char_byte('"')):
       raise self._ParseError('Exptected string.')
 
     if len(text) < 2 or text[-1] != text[0]:
