#7475: add (un)transform method to bytes/bytearray and str, add back codecs that can be used with them from Python 2.

Author: birkenfeld

Doc/library/codecs.rst

@@ -1165,6 +1165,46 @@ particular, the following variants typically exist:
 |                    |         | operand                   |
 +--------------------+---------+---------------------------+
 
+The following codecs provide bytes-to-bytes mappings.  They can be used with
+:meth:`bytes.transform` and :meth:`bytes.untransform`.
+
++--------------------+---------------------------+---------------------------+
+| Codec              | Aliases                   | Purpose                   |
++====================+===========================+===========================+
+| base64_codec       | base64, base-64           | Convert operand to MIME   |
+|                    |                           | base64                    |
++--------------------+---------------------------+---------------------------+
+| bz2_codec          | bz2                       | Compress the operand      |
+|                    |                           | using bz2                 |
++--------------------+---------------------------+---------------------------+
+| hex_codec          | hex                       | Convert operand to        |
+|                    |                           | hexadecimal               |
+|                    |                           | representation, with two  |
+|                    |                           | digits per byte           |
++--------------------+---------------------------+---------------------------+
+| quopri_codec       | quopri, quoted-printable, | Convert operand to MIME   |
+|                    | quotedprintable           | quoted printable          |
++--------------------+---------------------------+---------------------------+
+| uu_codec           | uu                        | Convert the operand using |
+|                    |                           | uuencode                  |
++--------------------+---------------------------+---------------------------+
+| zlib_codec         | zip, zlib                 | Compress the operand      |
+|                    |                           | using gzip                |
++--------------------+---------------------------+---------------------------+
+
+The following codecs provide string-to-string mappings.  They can be used with
+:meth:`str.transform` and :meth:`str.untransform`.
+
++--------------------+---------------------------+---------------------------+
+| Codec              | Aliases                   | Purpose                   |
++====================+===========================+===========================+
+| rot_13             | rot13                     | Returns the Caesar-cypher |
+|                    |                           | encryption of the operand |
++--------------------+---------------------------+---------------------------+
+
+.. versionadded:: 3.2
+   bytes-to-bytes and string-to-string codecs.
+
 
 :mod:`encodings.idna` --- Internationalized Domain Names in Applications
 ------------------------------------------------------------------------

Doc/library/stdtypes.rst

@@ -1352,6 +1352,19 @@ functions based on regular expressions.
         "They're Bill's Friends."
 
 
+.. method:: str.transform(encoding, errors='strict')
+
+   Return an encoded version of the string.  In contrast to :meth:`encode`, this
+   method works with codecs that provide string-to-string mappings, and not
+   string-to-bytes mappings.  :meth:`transform` therefore returns a string
+   object.
+
+   The codecs that can be used with this method are listed in
+   :ref:`standard-encodings`.
+
+   .. versionadded:: 3.2
+
+
 .. method:: str.translate(map)
 
    Return a copy of the *s* where all characters have been mapped through the
@@ -1369,6 +1382,14 @@ functions based on regular expressions.
       example).
 
 
+.. method:: str.untransform(encoding, errors='strict')
+
+   Return a decoded version of the string.  This provides the reverse operation
+   of :meth:`transform`.
+
+   .. versionadded:: 3.2
+
+
 .. method:: str.upper()
 
    Return a copy of the string converted to uppercase.
@@ -1800,6 +1821,20 @@ The bytes and bytearray types have an additional class method:
 The maketrans and translate methods differ in semantics from the versions
 available on strings:
 
+.. method:: bytes.transform(encoding, errors='strict')
+            bytearray.transform(encoding, errors='strict')
+
+   Return an encoded version of the bytes object.  In contrast to
+   :meth:`encode`, this method works with codecs that provide bytes-to-bytes
+   mappings, and not string-to-bytes mappings.  :meth:`transform` therefore
+   returns a bytes or bytearray object.
+
+   The codecs that can be used with this method are listed in
+   :ref:`standard-encodings`.
+
+   .. versionadded:: 3.2
+
+
 .. method:: bytes.translate(table[, delete])
             bytearray.translate(table[, delete])
 
@@ -1817,6 +1852,15 @@ available on strings:
       b'rd ths shrt txt'
 
 
+.. method:: bytes.untransform(encoding, errors='strict')
+            bytearray.untransform(encoding, errors='strict')
+
+   Return an decoded version of the bytes object.  This provides the reverse
+   operation of :meth:`transform`.
+
+   .. versionadded:: 3.2
+
+
 .. staticmethod:: bytes.maketrans(from, to)
                   bytearray.maketrans(from, to)
 

Lib/codecs.py

@@ -396,6 +396,8 @@ def __exit__(self, type, value, tb):
 
 class StreamReader(Codec):
 
+    charbuffertype = str
+
     def __init__(self, stream, errors='strict'):
 
         """ Creates a StreamReader instance.
@@ -417,9 +419,8 @@ def __init__(self, stream, errors='strict'):
         self.stream = stream
         self.errors = errors
         self.bytebuffer = b""
-        # For str->str decoding this will stay a str
-        # For str->unicode decoding the first read will promote it to unicode
-        self.charbuffer = ""
+        self._empty_charbuffer = self.charbuffertype()
+        self.charbuffer = self._empty_charbuffer
         self.linebuffer = None
 
     def decode(self, input, errors='strict'):
@@ -455,7 +456,7 @@ def read(self, size=-1, chars=-1, firstline=False):
         """
         # If we have lines cached, first merge them back into characters
         if self.linebuffer:
-            self.charbuffer = "".join(self.linebuffer)
+            self.charbuffer = self._empty_charbuffer.join(self.linebuffer)
             self.linebuffer = None
 
         # read until we get the required number of characters (if available)
@@ -498,7 +499,7 @@ def read(self, size=-1, chars=-1, firstline=False):
         if chars < 0:
             # Return everything we've got
             result = self.charbuffer
-            self.charbuffer = ""
+            self.charbuffer = self._empty_charbuffer
         else:
             # Return the first chars characters
             result = self.charbuffer[:chars]
@@ -529,15 +530,16 @@ def readline(self, size=None, keepends=True):
             return line
 
         readsize = size or 72
-        line = ""
+        line = self._empty_charbuffer
         # If size is given, we call read() only once
         while True:
             data = self.read(readsize, firstline=True)
             if data:
                 # If we're at a "\r" read one extra character (which might
                 # be a "\n") to get a proper line ending. If the stream is
                 # temporarily exhausted we return the wrong line ending.
-                if data.endswith("\r"):
+                if (isinstance(data, str) and data.endswith("\r")) or \
+                   (isinstance(data, bytes) and data.endswith(b"\r")):
                     data += self.read(size=1, chars=1)
 
             line += data
@@ -563,7 +565,8 @@ def readline(self, size=None, keepends=True):
                 line0withoutend = lines[0].splitlines(False)[0]
                 if line0withend != line0withoutend: # We really have a line end
                     # Put the rest back together and keep it until the next call
-                    self.charbuffer = "".join(lines[1:]) + self.charbuffer
+                    self.charbuffer = self._empty_charbuffer.join(lines[1:]) + \
+                                      self.charbuffer
                     if keepends:
                         line = line0withend
                     else:
@@ -574,7 +577,7 @@ def readline(self, size=None, keepends=True):
                 if line and not keepends:
                     line = line.splitlines(False)[0]
                 break
-            if readsize<8000:
+            if readsize < 8000:
                 readsize *= 2
         return line
 
@@ -603,7 +606,7 @@ def reset(self):
 
         """
         self.bytebuffer = b""
-        self.charbuffer = ""
+        self.charbuffer = self._empty_charbuffer
         self.linebuffer = None
 
     def seek(self, offset, whence=0):

Lib/encodings/aliases.py

@@ -33,9 +33,9 @@
     'us'                 : 'ascii',
     'us_ascii'           : 'ascii',
 
-    ## base64_codec codec
-    #'base64'             : 'base64_codec',
-    #'base_64'            : 'base64_codec',
+    # base64_codec codec
+    'base64'             : 'base64_codec',
+    'base_64'            : 'base64_codec',
 
     # big5 codec
     'big5_tw'            : 'big5',
@@ -45,8 +45,8 @@
     'big5_hkscs'         : 'big5hkscs',
     'hkscs'              : 'big5hkscs',
 
-    ## bz2_codec codec
-    #'bz2'                : 'bz2_codec',
+    # bz2_codec codec
+    'bz2'                : 'bz2_codec',
 
     # cp037 codec
     '037'                : 'cp037',
@@ -248,8 +248,8 @@
     'cp936'              : 'gbk',
     'ms936'              : 'gbk',
 
-    ## hex_codec codec
-    #'hex'                : 'hex_codec',
+    # hex_codec codec
+    'hex'                : 'hex_codec',
 
     # hp_roman8 codec
     'roman8'             : 'hp_roman8',
@@ -450,13 +450,13 @@
     'cp154'              : 'ptcp154',
     'cyrillic_asian'     : 'ptcp154',
 
-    ## quopri_codec codec
-    #'quopri'             : 'quopri_codec',
-    #'quoted_printable'   : 'quopri_codec',
-    #'quotedprintable'    : 'quopri_codec',
+    # quopri_codec codec
+    'quopri'             : 'quopri_codec',
+    'quoted_printable'   : 'quopri_codec',
+    'quotedprintable'    : 'quopri_codec',
 
-    ## rot_13 codec
-    #'rot13'              : 'rot_13',
+    # rot_13 codec
+    'rot13'              : 'rot_13',
 
     # shift_jis codec
     'csshiftjis'         : 'shift_jis',
@@ -518,12 +518,12 @@
     'utf8_ucs2'          : 'utf_8',
     'utf8_ucs4'          : 'utf_8',
 
-    ## uu_codec codec
-    #'uu'                 : 'uu_codec',
+    # uu_codec codec
+    'uu'                 : 'uu_codec',
 
-    ## zlib_codec codec
-    #'zip'                : 'zlib_codec',
-    #'zlib'               : 'zlib_codec',
+    # zlib_codec codec
+    'zip'                : 'zlib_codec',
+    'zlib'               : 'zlib_codec',
 
     # temporary mac CJK aliases, will be replaced by proper codecs in 3.1
     'x_mac_japanese'      : 'shift_jis',

Lib/encodings/base64_codec.py

@@ -0,0 +1,55 @@
+"""Python 'base64_codec' Codec - base64 content transfer encoding.
+
+This codec de/encodes from bytes to bytes and is therefore usable with
+bytes.transform() and bytes.untransform().
+
+Written by Marc-Andre Lemburg (mal@lemburg.com).
+"""
+
+import codecs
+import base64
+
+### Codec APIs
+
+def base64_encode(input, errors='strict'):
+    assert errors == 'strict'
+    return (base64.encodestring(input), len(input))
+
+def base64_decode(input, errors='strict'):
+    assert errors == 'strict'
+    return (base64.decodestring(input), len(input))
+
+class Codec(codecs.Codec):
+    def encode(self, input, errors='strict'):
+        return base64_encode(input, errors)
+    def decode(self, input, errors='strict'):
+        return base64_decode(input, errors)
+
+class IncrementalEncoder(codecs.IncrementalEncoder):
+    def encode(self, input, final=False):
+        assert self.errors == 'strict'
+        return base64.encodestring(input)
+
+class IncrementalDecoder(codecs.IncrementalDecoder):
+    def decode(self, input, final=False):
+        assert self.errors == 'strict'
+        return base64.decodestring(input)
+
+class StreamWriter(Codec, codecs.StreamWriter):
+    charbuffertype = bytes
+
+class StreamReader(Codec, codecs.StreamReader):
+    charbuffertype = bytes
+
+### encodings module API
+
+def getregentry():
+    return codecs.CodecInfo(
+        name='base64',
+        encode=base64_encode,
+        decode=base64_decode,
+        incrementalencoder=IncrementalEncoder,
+        incrementaldecoder=IncrementalDecoder,
+        streamwriter=StreamWriter,
+        streamreader=StreamReader,
+    )