Allow setting a default encoding to ease str/unicode <-> c string conversion.

Cython/Compiler/ExprNodes.py

@@ -59,19 +59,42 @@ not_a_constant = NotConstant()
 constant_value_not_set = object()
 
 # error messages when coercing from key[0] to key[1]
-find_coercion_error = {
+coercion_error_dict = {
     # string related errors
     (Builtin.unicode_type, Builtin.bytes_type) : "Cannot convert Unicode string to 'bytes' implicitly, encoding required.",
     (Builtin.unicode_type, Builtin.str_type)   : "Cannot convert Unicode string to 'str' implicitly. This is not portable and requires explicit encoding.",
     (Builtin.unicode_type, PyrexTypes.c_char_ptr_type) : "Unicode objects do not support coercion to C types.",
+    (Builtin.unicode_type, PyrexTypes.c_uchar_ptr_type) : "Unicode objects do not support coercion to C types.",
     (Builtin.bytes_type, Builtin.unicode_type) : "Cannot convert 'bytes' object to unicode implicitly, decoding required",
     (Builtin.bytes_type, Builtin.str_type) : "Cannot convert 'bytes' object to str implicitly. This is not portable to Py3.",
     (Builtin.str_type, Builtin.unicode_type) : "str objects do not support coercion to unicode, use a unicode string literal instead (u'')",
     (Builtin.str_type, Builtin.bytes_type) : "Cannot convert 'str' to 'bytes' implicitly. This is not portable.",
     (Builtin.str_type, PyrexTypes.c_char_ptr_type) : "'str' objects do not support coercion to C types (use 'bytes'?).",
+    (Builtin.str_type, PyrexTypes.c_uchar_ptr_type) : "'str' objects do not support coercion to C types (use 'bytes'?).",
     (PyrexTypes.c_char_ptr_type, Builtin.unicode_type) : "Cannot convert 'char*' to unicode implicitly, decoding required",
     (PyrexTypes.c_uchar_ptr_type, Builtin.unicode_type) : "Cannot convert 'char*' to unicode implicitly, decoding required",
-    }.get
+}
+def find_coercion_error(type_tuple, default, env):
+    err = coercion_error_dict.get(type_tuple)
+    if err is None:
+        return default
+    elif ((PyrexTypes.c_char_ptr_type in type_tuple or PyrexTypes.c_uchar_ptr_type in type_tuple)
+            and env.directives['c_string_encoding']):
+        if type_tuple[1].is_pyobject:
+            return default
+        elif env.directives['c_string_encoding'] == 'ascii':
+            return default
+        else:
+            return "'%s' objects do not support coercion to C types with non-ascii default encoding" % type_tuple[0].name
+    else:
+        return err
+
+def default_str_type(env):
+    return {
+        'bytes': bytes_type,
+        'str': str_type,
+        'unicode': unicode_type
+    }.get(env.directives['c_string_type'])
 
 
 class ExprNode(Node):
@@ -616,7 +639,7 @@ class ExprNode(Node):
         src = self
         src_type = self.type
 
-        if self.check_for_coercion_error(dst_type):
+        if self.check_for_coercion_error(dst_type, env):
             return self
 
         if dst_type.is_reference and not src_type.is_reference:
@@ -681,7 +704,7 @@ class ExprNode(Node):
                 if dst_type is bytes_type and src.type.is_int:
                     src = CoerceIntToBytesNode(src, env)
                 else:
-                    src = CoerceToPyTypeNode(src, env)
+                    src = CoerceToPyTypeNode(src, env, type=dst_type)
             if not src.type.subtype_of(dst_type):
                 if not isinstance(src, NoneNode):
                     src = PyTypeTestNode(src, dst_type, env)
@@ -702,10 +725,10 @@ class ExprNode(Node):
     def fail_assignment(self, dst_type):
         error(self.pos, "Cannot assign type '%s' to '%s'" % (self.type, dst_type))
 
-    def check_for_coercion_error(self, dst_type, fail=False, default=None):
+    def check_for_coercion_error(self, dst_type, env, fail=False, default=None):
         if fail and not default:
             default = "Cannot assign type '%(FROM)s' to '%(TO)s'"
-        message = find_coercion_error((self.type, dst_type), default)
+        message = find_coercion_error((self.type, dst_type), default, env)
         if message is not None:
             error(self.pos, message % {'FROM': self.type, 'TO': dst_type})
             return True
@@ -1101,7 +1124,7 @@ class BytesNode(ConstNode):
             if dst_type in (py_object_type, Builtin.bytes_type):
                 node.type = Builtin.bytes_type
             else:
-                self.check_for_coercion_error(dst_type, fail=True)
+                self.check_for_coercion_error(dst_type, env, fail=True)
                 return node
         elif dst_type == PyrexTypes.c_char_ptr_type:
             node.type = dst_type
@@ -1156,7 +1179,7 @@ class UnicodeNode(PyConstNode):
                 return BytesNode(self.pos, value=self.bytes_value).coerce_to(dst_type, env)
             error(self.pos, "Unicode literals do not support coercion to C types other than Py_UNICODE or Py_UCS4.")
         elif dst_type is not py_object_type:
-            if not self.check_for_coercion_error(dst_type):
+            if not self.check_for_coercion_error(dst_type, env):
                 self.fail_assignment(dst_type)
         return self
 
@@ -1213,7 +1236,7 @@ class StringNode(PyConstNode):
 #                return BytesNode(self.pos, value=self.value)
             if not dst_type.is_pyobject:
                 return BytesNode(self.pos, value=self.value).coerce_to(dst_type, env)
-            self.check_for_coercion_error(dst_type, fail=True)
+            self.check_for_coercion_error(dst_type, env, fail=True)
         return self
 
     def can_coerce_to_char_literal(self):
@@ -3456,7 +3479,7 @@ class SliceIndexNode(ExprNode):
             self.stop = self.stop.analyse_types(env)
         base_type = self.base.type
         if base_type.is_string or base_type.is_cpp_string:
-            self.type = bytes_type
+            self.type = default_str_type(env)
         elif base_type.is_ptr:
             self.type = base_type
         elif base_type.is_array:
@@ -3483,6 +3506,16 @@ class SliceIndexNode(ExprNode):
     nogil_check = Node.gil_error
     gil_message = "Slicing Python object"
 
+    def coerce_to(self, dst_type, env):
+        if ((self.base.type.is_string or self.base.type.is_cpp_string)
+                and dst_type in (bytes_type, str_type, unicode_type)):
+            if dst_type is not bytes_type and not env.directives['c_string_encoding']:
+                error(self.pos,
+                    "default encoding required for conversion from '%s' to '%s'" % 
+                    (self.base.type, dst_type))
+            self.type = dst_type
+        return super(SliceIndexNode, self).coerce_to(dst_type, env)
+
     def generate_result_code(self, code):
         if not self.type.is_pyobject:
             error(self.pos,
@@ -3499,15 +3532,17 @@ class SliceIndexNode(ExprNode):
                 base_result = '((const char*)%s)' % base_result
             if self.stop is None:
                 code.putln(
-                    "%s = PyBytes_FromString(%s + %s); %s" % (
+                    "%s = __Pyx_Py%s_FromString(%s + %s); %s" % (
                         result,
+                        self.type.name.title(),
                         base_result,
                         start_code,
                         code.error_goto_if_null(result, self.pos)))
             else:
                 code.putln(
-                    "%s = PyBytes_FromStringAndSize(%s + %s, %s - %s); %s" % (
-                        self.result(),
+                    "%s = __Pyx_Py%s_FromStringAndSize(%s + %s, %s - %s); %s" % (
+                        result,
+                        self.type.name.title(),
                         base_result,
                         start_code,
                         stop_code,
@@ -7689,7 +7724,7 @@ class TypecastNode(ExprNode):
                 self.operand = CoerceIntToBytesNode(self.operand, env)
             elif self.operand.type.can_coerce_to_pyobject(env):
                 self.result_ctype = py_object_type
-                self.operand = self.operand.coerce_to_pyobject(env)
+                self.operand = self.operand.coerce_to(base_type, env)
             else:
                 if self.operand.type.is_ptr:
                     if not (self.operand.type.base_type.is_void or self.operand.type.base_type.is_struct):
@@ -7709,6 +7744,9 @@ class TypecastNode(ExprNode):
         elif from_py and to_py:
             if self.typecheck and self.type.is_extension_type:
                 self.operand = PyTypeTestNode(self.operand, self.type, env, notnone=True)
+            elif isinstance(self.operand, SliceIndexNode):
+                # This cast can influence the created type of string slices.
+                self.operand = self.operand.coerce_to(self.type, env)
         elif self.type.is_complex and self.operand.type.is_complex:
             self.operand = self.operand.coerce_to_simple(env)
         elif self.operand.type.is_fused:
@@ -9054,12 +9092,12 @@ class CmpNode(object):
                 new_common_type = type1
             elif type1.is_pyobject or type2.is_pyobject:
                 if type2.is_numeric or type2.is_string:
-                    if operand2.check_for_coercion_error(type1):
+                    if operand2.check_for_coercion_error(type1, env):
                         new_common_type = error_type
                     else:
                         new_common_type = py_object_type
                 elif type1.is_numeric or type1.is_string:
-                    if operand1.check_for_coercion_error(type2):
+                    if operand1.check_for_coercion_error(type2, env):
                         new_common_type = error_type
                     else:
                         new_common_type = py_object_type
@@ -9835,11 +9873,17 @@ class CoerceToPyTypeNode(CoercionNode):
         if type is py_object_type:
             # be specific about some known types
             if arg.type.is_string or arg.type.is_cpp_string:
-                self.type = bytes_type
+                self.type = default_str_type(env)
             elif arg.type.is_unicode_char:
                 self.type = unicode_type
             elif arg.type.is_complex:
                 self.type = Builtin.complex_type
+        elif arg.type.is_string or arg.type.is_cpp_string:
+            if type is not bytes_type and not env.directives['c_string_encoding']:
+                error(arg.pos,
+                    "default encoding required for conversion from '%s' to '%s'" % 
+                    (arg.type, type))
+            self.type = type
         else:
             # FIXME: check that the target type and the resulting type are compatible
             pass
@@ -9876,11 +9920,15 @@ class CoerceToPyTypeNode(CoercionNode):
         return self
 
     def generate_result_code(self, code):
-        if self.arg.type.is_memoryviewslice:
-            funccall = self.arg.type.get_to_py_function(self.env, self.arg)
-        else:
-            funccall = "%s(%s)" % (self.arg.type.to_py_function,
-                                   self.arg.result())
+        arg_type = self.arg.type
+        if arg_type.is_memoryviewslice:
+            funccall = arg_type.get_to_py_function(self.env, self.arg)
+        else:
+            func = arg_type.to_py_function
+            if ((arg_type.is_string or arg_type.is_cpp_string)
+                    and self.type in (bytes_type, str_type, unicode_type)):
+                func = func.replace("Object", self.type.name.title())
+            funccall = "%s(%s)" % (func, self.arg.result())
 
         code.putln('%s = %s; %s' % (
             self.result(),

Cython/Compiler/ModuleNode.py

@@ -556,7 +556,17 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode):
             code.putln("#endif")
             code.putln("")
         code.put(UtilityCode.load_as_string("UtilityFunctionPredeclarations", "ModuleSetupCode.c")[0])
+
+        c_string_type = env.directives['c_string_type']
+        c_string_encoding = env.directives['c_string_encoding']
+        if c_string_type != 'bytes' and not c_string_encoding:
+            error(self.pos, "a default encoding must be provided if c_string_type != bytes")
+        code.putln('#define __PYX_DEFAULT_STRING_ENCODING_IS_ASCII %s' % int(c_string_encoding == 'ascii'))
+        code.putln('#define __PYX_DEFAULT_STRING_ENCODING "%s"' % c_string_encoding)
+        code.putln('#define __Pyx_PyObject_FromString __Pyx_Py%s_FromString' % c_string_type.title())
+        code.putln('#define __Pyx_PyObject_FromStringAndSize __Pyx_Py%s_FromStringAndSize' % c_string_type.title())
         code.put(UtilityCode.load_as_string("TypeConversions", "TypeConversion.c")[0])
+        
         code.put(Nodes.branch_prediction_macros)
         code.putln('')
         code.putln('static PyObject *%s;' % env.module_cname)
@@ -1888,6 +1898,10 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode):
         code.putln("/*--- Initialize various global constants etc. ---*/")
         code.putln(code.error_goto_if_neg("__Pyx_InitGlobals()", self.pos))
 
+        code.putln("#ifdef __PYX_DEFAULT_STRING_ENCODING_IS_ASCII")
+        code.putln("if (__Pyx_init_sys_getdefaultencoding_not_ascii() < 0) %s" % code.error_goto(self.pos))
+        code.putln("#endif")
+
         __main__name = code.globalstate.get_py_string_const(
             EncodedString("__main__"), identifier=True)
         code.putln("if (%s%s) {" % (Naming.module_is_main, self.full_module_name.replace('.', '__')))

Cython/Compiler/Options.py

@@ -95,6 +95,8 @@ directive_defaults = {
     'language_level': 2,
     'fast_getattr': False, # Undocumented until we come up with a better way to handle this everywhere.
     'py2_import': False, # For backward compatibility of Cython's source code in Py3 source mode
+    'c_string_type': 'bytes',
+    'c_string_encoding': '',
 
     # set __file__ and/or __path__ to known source/target path at import time (instead of not having them available)
     'set_initial_path' : None,  # SOURCEFILE or "/full/path/to/module"
@@ -160,6 +162,9 @@ directive_scopes = { # defaults to available everywhere
     'set_initial_path' : ('module',),
     'test_assert_path_exists' : ('function', 'class', 'cclass'),
     'test_fail_if_path_exists' : ('function', 'class', 'cclass'),
+    # Avoid scope-specific to/from_py_functions.
+    'c_string_type': ('module',),
+    'c_string_encoding': ('module',),
 }
 
 def parse_directive_value(name, value, relaxed_bool=False):

Cython/Compiler/PyrexTypes.py

@@ -2179,13 +2179,13 @@ class CPointerBaseType(CType):
 
         if self.is_string and not base_type.is_error:
             if base_type.signed:
-                self.to_py_function = "PyBytes_FromString"
+                self.to_py_function = "__Pyx_PyObject_FromString"
                 if self.is_ptr:
-                    self.from_py_function = "PyBytes_AsString"
+                    self.from_py_function = "__Pyx_PyObject_AsString"
             else:
-                self.to_py_function = "__Pyx_PyBytes_FromUString"
+                self.to_py_function = "__Pyx_PyObject_FromUString"
                 if self.is_ptr:
-                    self.from_py_function = "__Pyx_PyBytes_AsUString"
+                    self.from_py_function = "__Pyx_PyObject_AsUString"
             self.exception_value = "NULL"
 
     def py_type_name(self):

Cython/Utility/CppConvert.pyx

@@ -7,10 +7,13 @@ cdef extern from *:
     cdef cppclass string "std::string":
         string()
         string(char* c_str, size_t size)
+    cdef char* __Pyx_PyObject_AsStringAndSize(object, Py_ssize_t*)
 
 @cname("{{cname}}")
 cdef string {{cname}}(object o) except *:
-    return string(<char*>o, len(o))
+    cdef Py_ssize_t length
+    cdef char* data = __Pyx_PyObject_AsStringAndSize(o, &length)
+    return string(data, length)
 
 
 #################### string.to_py ####################
@@ -21,10 +24,11 @@ cdef extern from *:
     cdef cppclass string "const std::string":
         char* data()
         size_t size()
+    cdef object __Pyx_PyObject_FromStringAndSize(char*, size_t)
 
 @cname("{{cname}}")
 cdef object {{cname}}(string& s):
-    return s.data()[:s.size()]
+    return __Pyx_PyObject_FromStringAndSize(s.data(), s.size())
 
 
 #################### vector.from_py ####################

Cython/Utility/TypeConversion.c

@@ -2,8 +2,27 @@
 
 /* Type Conversion Predeclarations */
 
-#define __Pyx_PyBytes_FromUString(s) PyBytes_FromString((char*)s)
-#define __Pyx_PyBytes_AsUString(s)   ((unsigned char*) PyBytes_AsString(s))
+static CYTHON_INLINE char* __Pyx_PyObject_AsString(PyObject*);
+static CYTHON_INLINE char* __Pyx_PyObject_AsStringAndSize(PyObject*, Py_ssize_t* length);
+
+#define __Pyx_PyBytes_FromString        PyBytes_FromString
+#define __Pyx_PyBytes_FromStringAndSize PyBytes_FromStringAndSize
+static CYTHON_INLINE PyObject* __Pyx_PyUnicode_FromString(char*);
+#define __Pyx_PyUnicode_FromStringAndSize(c_str, size) PyUnicode_Decode(c_str, size, __PYX_DEFAULT_STRING_ENCODING, NULL)
+
+#if PY_VERSION_HEX < 0x03000000
+    #define __Pyx_PyStr_FromString        __Pyx_PyBytes_FromString
+    #define __Pyx_PyStr_FromStringAndSize __Pyx_PyBytes_FromStringAndSize
+#else
+    #define __Pyx_PyStr_FromString        __Pyx_PyUnicode_FromString
+    #define __Pyx_PyStr_FromStringAndSize __Pyx_PyUnicode_FromStringAndSize
+#endif
+
+#define __Pyx_PyObject_AsUString(s)    ((unsigned char*) __Pyx_PyObject_AsString(s))
+#define __Pyx_PyObject_FromUString(s)  __Pyx_PyObject_FromString((char*)s)
+#define __Pyx_PyBytes_FromUString(s)   __Pyx_PyBytes_FromString((char*)s)
+#define __Pyx_PyStr_FromUString(s)     __Pyx_PyStr_FromString((char*)s)
+#define __Pyx_PyUnicode_FromUString(s) __Pyx_PyUnicode_FromString((char*)s)
 
 #define __Pyx_Owned_Py_None(b) (Py_INCREF(Py_None), Py_None)
 #define __Pyx_PyBool_FromLong(b) ((b) ? (Py_INCREF(Py_True), Py_True) : (Py_INCREF(Py_False), Py_False))
@@ -21,10 +40,129 @@ static CYTHON_INLINE size_t __Pyx_PyInt_AsSize_t(PyObject*);
 #endif
 #define __pyx_PyFloat_AsFloat(x) ((float) __pyx_PyFloat_AsDouble(x))
 
+#if PY_VERSION_HEX < 0x03000000 && __PYX_DEFAULT_STRING_ENCODING_IS_ASCII
+static int __Pyx_sys_getdefaultencoding_not_ascii;
+static int __Pyx_init_sys_getdefaultencoding_not_ascii() {
+    PyObject* sys = NULL;
+    PyObject* default_encoding = NULL;
+    PyObject* codecs = NULL;
+    PyObject* normalized_encoding = NULL;
+    PyObject* normalized_encoding_name = NULL;
+    sys = PyImport_ImportModule("sys");
+    if (sys == NULL) goto bad;
+    default_encoding = PyObject_CallMethod(sys, (char*) (const char*) "getdefaultencoding", NULL);
+    if (default_encoding == NULL) goto bad;
+    if (strcmp(PyBytes_AsString(default_encoding), "ascii") == 0) {
+        __Pyx_sys_getdefaultencoding_not_ascii = 0;
+    } else {
+        char* normalized_encoding_c;
+        codecs = PyImport_ImportModule("codecs");
+        printf("codecs %p\n", codecs);
+        if (codecs == NULL) goto bad;
+        normalized_encoding = PyObject_CallMethod(codecs, (char*) (const char*) "lookup", (char*) (const char*) "O", default_encoding);
+        printf("normalized_encoding %p\n", normalized_encoding);
+        if (normalized_encoding == NULL) goto bad;
+        normalized_encoding_name = PyObject_GetAttrString(normalized_encoding, (char*) (const char*) "name");
+        printf("normalized_encoding_name %p\n", normalized_encoding_name);
+        if (normalized_encoding_name == NULL) goto bad;
+        normalized_encoding_c = PyBytes_AsString(normalized_encoding_name);
+        printf("normalized_encoding_c %s\n", normalized_encoding_c);
+        if (normalized_encoding_c == NULL) goto bad;
+        __Pyx_sys_getdefaultencoding_not_ascii = strcmp(normalized_encoding_c, "ascii");
+        if (!__Pyx_sys_getdefaultencoding_not_ascii) {
+            int ascii_compatible =
+                (strncmp(normalized_encoding_c, "iso8859-", 8) == 0 ||
+                 strcmp(normalized_encoding_c, "macroman") == 0 ||
+                 strcmp(normalized_encoding_c, "utf-8") == 0);
+            // I've never heard of a system where this happens, but it might...
+            if (!ascii_compatible) {
+                PyErr_Format(
+                    PyExc_ValueError,
+                    "This module compiled with c_string_encoding=ascii, but default encoding '%s' is not a superset of ascii.",
+                    normalized_encoding_c);
+                goto bad;
+            }
+        }
+    }
+    printf("__Pyx_sys_getdefaultencoding_not_ascii %d\n", __Pyx_sys_getdefaultencoding_not_ascii);
+    Py_XDECREF(sys);
+    Py_XDECREF(default_encoding);
+    Py_XDECREF(codecs);
+    Py_XDECREF(normalized_encoding);
+    Py_XDECREF(normalized_encoding_name);
+    return 0;
+bad:
+    Py_XDECREF(sys);
+    Py_XDECREF(default_encoding);
+    Py_XDECREF(codecs);
+    Py_XDECREF(normalized_encoding);
+    Py_XDECREF(normalized_encoding_name);
+    return -1;
+}
+#else
+#define __Pyx_init_sys_getdefaultencoding_not_ascii() 0
+#endif
+
+
 /////////////// TypeConversions ///////////////
 
 /* Type Conversion Functions */
 
+static CYTHON_INLINE PyObject* __Pyx_PyUnicode_FromString(char* c_str) {
+    return __Pyx_PyUnicode_FromStringAndSize(c_str, strlen(c_str));
+}
+
+static CYTHON_INLINE char* __Pyx_PyObject_AsString(PyObject* o) {
+    Py_ssize_t ignore;
+    return __Pyx_PyObject_AsStringAndSize(o, &ignore);
+}
+
+static CYTHON_INLINE char* __Pyx_PyObject_AsStringAndSize(PyObject* o, Py_ssize_t *length) {
+#if __PYX_DEFAULT_STRING_ENCODING_IS_ASCII
+    if (
+#if PY_VERSION_HEX < 0x03000000
+            __Pyx_sys_getdefaultencoding_not_ascii && 
+#endif
+            PyUnicode_Check(o)) {
+#if PY_VERSION_HEX < 0x03030000
+        // borrowed, cached reference
+        PyObject* defenc = _PyUnicode_AsDefaultEncodedString(o, NULL);
+        char* maybe_ascii = PyBytes_AS_STRING(defenc);
+        char* end = maybe_ascii + PyBytes_GET_SIZE(defenc);
+        for (char* c = maybe_ascii; c < end; c++) {
+            if ((unsigned char) (*c) >= 128) {
+                // raise the error
+                PyUnicode_AsASCIIString(o);
+                return NULL;
+            }
+        }
+        *length = PyBytes_GET_SIZE(defenc);
+        return maybe_ascii;
+#else /* PY_VERSION_HEX < 0x03030000 */
+        PyUnicode_READY(o);
+        if (PyUnicode_IS_ASCIII(o)) {
+            // cached for the lifetime of the object
+            *length = PyUnicode_GET_DATA_SIZE(o);
+            return PyUnicode_AsUTF8(o);
+        } else {
+            // raise the error
+            PyUnicode_AsASCIIString(o);
+            return NULL;
+        }
+#endif /* PY_VERSION_HEX < 0x03030000 */
+    } else
+#endif /* __PYX_DEFAULT_STRING_ENCODING_IS_ASCII */
+    {
+        char* result;
+        int r = PyBytes_AsStringAndSize(o, &result, length);
+        if (r < 0) {
+            return NULL;
+        } else {
+            return result;
+        }
+    }
+}
+
 /* Note: __Pyx_PyObject_IsTrue is written to minimize branching. */
 static CYTHON_INLINE int __Pyx_PyObject_IsTrue(PyObject* x) {
    int is_true = x == Py_True;