strconv: New package + expose .quote() there

Move quote implementation from gcompat to strconv. Make quote work on both
unicode|bytes string input and produce the same output type. Preserve qq
(still remaining in gcompat) to always produce str, since that is used
in prints.

README.rst

@@ -134,6 +134,9 @@ without escaping printable UTF-8 characters::
 `qq` accepts both `str` and `bytes` (`unicode` and `str` on Python2)
 and also any other type that can be converted to `str`.
 
+Package `golang.strconv` provides direct access to conversion routines, for
+example `strconv.quote`.
+
 
 Benchmarking and testing
 ------------------------

golang/gcompat.py

@@ -19,109 +19,26 @@
 # See https://www.nexedi.com/licensing for rationale and options.
 """Package gcompat provides Go-compatibility layer for Python"""
 
-import six, unicodedata
-from six.moves import range as xrange
+from golang import strconv
+import six
 
 # qq is substitute for %q, which is missing in python.
 #
 # (python's automatic escape uses smartquotes quoting with either ' or ").
+#
+# like %s, %q automatically converts its argument to string.
 def qq(obj):
-    # go: like %s, %q automatically converts to string
-    decode_utf8 = False
-    if isinstance(obj, bytes):                      # py2: str      py3: bytes
-        if six.PY3:
-            decode_utf8 = True
-    elif not isinstance(obj, six.text_type):        # py2: unicode  py3: str
+    # make sure obj is text | bytes
+    # py2: unicode | str
+    # py3: str     | bytes
+    if not isinstance(obj, (six.text_type, six.binary_type)):
         obj = str(obj)
 
-    if isinstance(obj, six.text_type):              # py2: unicode  py3: str    -> bytes
-        obj = obj.encode('UTF-8')
-        decode_utf8 = True
-
-    qobj = _quote(obj)
+    qobj = strconv.quote(obj)
 
-    if decode_utf8:
+    # `printf('%s', qq(obj))` should work. For this make sure qobj is always a
+    # str - not bytes under py3 (if it was bytes it will print e.g. as b'...')
+    if six.PY3 and isinstance(qobj, bytes):
         qobj = qobj.decode('UTF-8')
-    return qobj
-
-
-# _quote quotes bytes string into valid "..." bytes string always quoted with ".
-def _quote(s):
-    outv = []
-    emit = outv.append
-    i = 0
-    while i < len(s):
-        c = s[i:i+1]
-        # fast path - ASCII only
-        if ord(c) < 0x80:
-            if c in b'\\"':
-                emit(b'\\'+c)
-
-            # printable ASCII
-            elif b' ' <= c <= b'\x7e':
-                emit(c)
-
-            # non-printable ASCII
-            elif c == b'\t':
-                emit(br'\t')
-            elif c == b'\n':
-                emit(br'\n')
-            elif c == b'\r':
-                emit(br'\r')
-
-            # everything else is non-printable
-            else:
-                emit(br'\x%02x' % ord(c))
-
-            i += 1
-
-        # slow path - full UTF-8 decoding + unicodedata
-        else:
-            r, size = _utf8_decode_rune(s[i:])
-            isize = i + size
-
-            # decode error - just emit raw byte as escaped
-            if r == _rune_error:
-                emit(br'\x%02x' % ord(c))
-
-            # printable utf-8 characters go as is
-            elif unicodedata.category(r)[0] in _printable_cat0:
-                emit(s[i:isize])
 
-            # everything else goes in numeric byte escapes
-            else:
-                for j in xrange(i, isize):
-                    emit(br'\x%02x' % ord(s[j:j+1]))
-
-            i = isize
-
-    return b'"' + b''.join(outv) + b'"'
-
-
-_printable_cat0 = frozenset(['L', 'N', 'P', 'S'])   # letters, numbers, punctuation, symbols
-
-_rune_error = u'\uFFFD' # unicode replacement character
-
-# _utf8_decode_rune decodes next UTF8-character from byte string s.
-#
-# _utf8_decode_rune(s) -> (r, size)
-def _utf8_decode_rune(s):
-    if len(s) == 0:
-        return '', 0
-
-    l = min(len(s), 4)  # max size of an UTF-8 encoded character
-    while l > 0:
-        try:
-            r = s[:l].decode('utf-8', 'strict')
-        except UnicodeDecodeError:
-            l -= 1
-            continue
-
-        if len(r) == 1:
-            return r, l
-
-        l -= 1
-        continue
-
-    # invalid UTF-8
-    return _rune_error, 1
+    return qobj

golang/gcompat_test.py

@@ -18,41 +18,4 @@
 # See COPYING file for full licensing terms.
 # See https://www.nexedi.com/licensing for rationale and options.
 
-from golang.gcompat import qq
-from six import int2byte as bchr
-from six.moves import range as xrange
-
-def byterange(start, stop):
-    b = b""
-    for i in xrange(start, stop):
-        b += bchr(i)
-
-    return b
-
-def test_qq():
-    testv = (
-        # in                want without leading/trailing "
-        ('',                r""),
-
-        (byterange(0,32),   r'\x00\x01\x02\x03\x04\x05\x06\x07\x08\t\n\x0b\x0c\r\x0e\x0f\x10\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f'),
-
-        ('\'',              r"'"),
-        ('"',               r"\""),
-        ('ab c\ndef',       r"ab c\ndef"),
-        ('a\'c\ndef',       r"a'c\ndef"),
-        ('a\"c\ndef',       r"a\"c\ndef"),
-        (u'a\"c\ndef',      u"a\\\"c\\ndef"),
-        (b'a\"c\ndef',      r'a\"c\ndef'),
-        ('привет\nмир',     r"привет\nмир"),
-        (u'привет\nмир',    u"привет\\nмир"),
-
-        # invalid utf-8
-        (b"\xd0a",          r"\xd0a"),
-
-        # non-printable utf-8
-        (u"\u007f\u0080\u0081\u0082\u0083\u0084\u0085\u0086\u0087", u"\\x7f\\xc2\\x80\\xc2\\x81\\xc2\\x82\\xc2\\x83\\xc2\\x84\\xc2\\x85\\xc2\\x86\\xc2\\x87"),
-    )
-
-    for tin, twant in testv:
-        twant = '"' + twant + '"'   # add lead/trail "
-        assert qq(tin) == twant
+# qq is tested as part of strconv.

golang/strconv.py

+# -*- coding: utf-8 -*-
+# Copyright (C) 2018  Nexedi SA and Contributors.
+#                     Kirill Smelkov <kirr@nexedi.com>
+#
+# This program is free software: you can Use, Study, Modify and Redistribute
+# it under the terms of the GNU General Public License version 3, or (at your
+# option) any later version, as published by the Free Software Foundation.
+#
+# You can also Link and Combine this program with other software covered by
+# the terms of any of the Free Software licenses or any of the Open Source
+# Initiative approved licenses and Convey the resulting work. Corresponding
+# source of such a combination shall include the source code for all other
+# software used.
+#
+# This program is distributed WITHOUT ANY WARRANTY; without even the implied
+# warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
+#
+# See COPYING file for full licensing terms.
+# See https://www.nexedi.com/licensing for rationale and options.
+"""Package strconv provides Go-compatible string conversions"""
+
+import six, unicodedata, codecs
+from six.moves import range as xrange
+
+
+# _bstr converts str/unicode/bytes s to UTF-8 encoded bytestring.
+#
+# TypeError is raised if type(s) is not one of the above.
+def _bstr(s):   # -> sbytes, wasunicode
+    wasunicode = False
+    if isinstance(s, bytes):                    # py2: str      py3: bytes
+        pass
+    elif isinstance(s, six.text_type):          # py2: unicode  py3: str
+        wasunicode = True
+    else:
+        raise TypeError("_bstr: invalid type %s", type(s))
+
+    if wasunicode:                              # py2: unicode  py3: str    -> bytes
+        s = s.encode('UTF-8')
+
+    return s, wasunicode
+
+# quote quotes unicode|bytes string into valid "..." unicode|bytes string always quoted with ".
+def quote(s):
+    s, wasunicode = _bstr(s)
+    qs = _quote(s)
+    if wasunicode:
+        qs = qs.decode('UTF-8')
+    return qs
+
+def _quote(s):
+    assert isinstance(s, bytes)
+
+    outv = []
+    emit = outv.append
+    i = 0
+    while i < len(s):
+        c = s[i:i+1]
+        # fast path - ASCII only
+        if ord(c) < 0x80:
+            if c in b'\\"':
+                emit(b'\\'+c)
+
+            # printable ASCII
+            elif b' ' <= c <= b'\x7e':
+                emit(c)
+
+            # non-printable ASCII
+            elif c == b'\t':
+                emit(br'\t')
+            elif c == b'\n':
+                emit(br'\n')
+            elif c == b'\r':
+                emit(br'\r')
+
+            # everything else is non-printable
+            else:
+                emit(br'\x%02x' % ord(c))
+
+            i += 1
+
+        # slow path - full UTF-8 decoding + unicodedata
+        else:
+            r, size = _utf8_decode_rune(s[i:])
+            isize = i + size
+
+            # decode error - just emit raw byte as escaped
+            if r == _rune_error:
+                emit(br'\x%02x' % ord(c))
+
+            # printable utf-8 characters go as is
+            elif unicodedata.category(r)[0] in _printable_cat0:
+                emit(s[i:isize])
+
+            # everything else goes in numeric byte escapes
+            else:
+                for j in xrange(i, isize):
+                    emit(br'\x%02x' % ord(s[j:j+1]))
+
+            i = isize
+
+    return b'"' + b''.join(outv) + b'"'
+
+
+_printable_cat0 = frozenset(['L', 'N', 'P', 'S'])   # letters, numbers, punctuation, symbols
+
+_rune_error = u'\uFFFD' # unicode replacement character
+
+# _utf8_decode_rune decodes next UTF8-character from byte string s.
+#
+# _utf8_decode_rune(s) -> (r, size)
+def _utf8_decode_rune(s):
+    assert isinstance(s, bytes)
+
+    if len(s) == 0:
+        return '', 0
+
+    l = min(len(s), 4)  # max size of an UTF-8 encoded character
+    while l > 0:
+        try:
+            r = s[:l].decode('utf-8', 'strict')
+        except UnicodeDecodeError:
+            l -= 1
+            continue
+
+        if len(r) == 1:
+            return r, l
+
+        l -= 1
+        continue
+
+    # invalid UTF-8
+    return _rune_error, 1

golang/strconv_test.py

+# -*- coding: utf-8 -*-
+# Copyright (C) 2018  Nexedi SA and Contributors.
+#                     Kirill Smelkov <kirr@nexedi.com>
+#
+# This program is free software: you can Use, Study, Modify and Redistribute
+# it under the terms of the GNU General Public License version 3, or (at your
+# option) any later version, as published by the Free Software Foundation.
+#
+# You can also Link and Combine this program with other software covered by
+# the terms of any of the Free Software licenses or any of the Open Source
+# Initiative approved licenses and Convey the resulting work. Corresponding
+# source of such a combination shall include the source code for all other
+# software used.
+#
+# This program is distributed WITHOUT ANY WARRANTY; without even the implied
+# warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
+#
+# See COPYING file for full licensing terms.
+# See https://www.nexedi.com/licensing for rationale and options.
+
+from golang.strconv import quote
+from golang.gcompat import qq
+
+from six import int2byte as bchr, PY3
+from six.moves import range as xrange
+
+def byterange(start, stop):
+    b = b""
+    for i in xrange(start, stop):
+        b += bchr(i)
+
+    return b
+
+def asstr(s):
+    if PY3 and isinstance(s, bytes):
+        s = s.decode('utf-8')
+    return s
+
+def test_quote():
+    testv = (
+        # in                quoted without leading/trailing "
+        ('',                r""),
+
+        (byterange(0,32),   br'\x00\x01\x02\x03\x04\x05\x06\x07\x08\t\n\x0b\x0c\r\x0e\x0f\x10\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f'),
+
+        ('\'',              r"'"),
+        ('"',               r"\""),
+        ('ab c\ndef',       r"ab c\ndef"),
+        ('a\'c\ndef',       r"a'c\ndef"),
+        ('a\"c\ndef',       r"a\"c\ndef"),
+        (u'a\"c\ndef',      u"a\\\"c\\ndef"),
+        (b'a\"c\ndef',      br'a\"c\ndef'),
+        ('привет\nмир',     r"привет\nмир"),
+        (u'привет\nмир',    u"привет\\nмир"),
+
+        # invalid utf-8
+        (b"\xd0a",          br"\xd0a"),
+
+        # non-printable utf-8
+        (u"\u007f\u0080\u0081\u0082\u0083\u0084\u0085\u0086\u0087", u"\\x7f\\xc2\\x80\\xc2\\x81\\xc2\\x82\\xc2\\x83\\xc2\\x84\\xc2\\x85\\xc2\\x86\\xc2\\x87"),
+    )
+
+    for tin, tquoted in testv:
+        # quote(in) == quoted
+        q = b'"' if isinstance(tquoted, bytes) else '"'
+        tquoted = q + tquoted + q   # add lead/trail "
+
+        assert quote(tin) == tquoted
+
+        # qq always gives str
+        assert qq(tin) == asstr(tquoted)
+
+        # also check how it works on complementary unicode/bytes input type
+        if isinstance(tin, bytes):
+            try:
+                tin = tin.decode('utf-8')
+            except UnicodeDecodeError:
+                # some inputs are not valid UTF-8
+                continue
+            tquoted = tquoted.decode('utf-8')
+        else:
+            # tin was unicode
+            tin = tin.encode('utf-8')
+            tquoted = tquoted.encode('utf-8')
+
+        assert quote(tin) == tquoted
+
+        # qq always gives str
+        assert qq(tin) == asstr(tquoted)