Relax class private names for c types (GH-3546)

Allow C functions/variables with names starting with "__" to be found
from within classes, even though their names would normally be
inaccessible due to the Python "class private names" mechanism.
https://github.com/cython/cython/issues/3544

Warn about unmangled Python names that are being found and used, since they do not adhere to Python lookup rules.

Removed `mangle_special_name` in favour of `mangle_class_private_name`,
which previously applied slightly different rules for creating vs looking up variables.

Closes #3548.

Cython/Compiler/Symtab.py

@@ -725,6 +725,7 @@ class Scope(object):
         return entry
 
     def declare_builtin(self, name, pos):
+        name = self.mangle_class_private_name(name)
         return self.outer_scope.declare_builtin(name, pos)
 
     def _declare_pyfunction(self, name, pos, visibility='extern', entry=None):
@@ -921,20 +922,43 @@ class Scope(object):
     def lookup(self, name):
         # Look up name in this scope or an enclosing one.
         # Return None if not found.
-        name = self.mangle_class_private_name(name)
-        return (self.lookup_here(name)
-            or (self.outer_scope and self.outer_scope.lookup(name))
-            or None)
+
+        mangled_name = self.mangle_class_private_name(name)
+        entry = (self.lookup_here(name)  # lookup here also does mangling
+                or (self.outer_scope and self.outer_scope.lookup(mangled_name))
+                or None)
+        if entry:
+            return entry
+
+        # look up the original name in the outer scope
+        # Not strictly Python behaviour but see https://github.com/cython/cython/issues/3544
+        entry = (self.outer_scope and self.outer_scope.lookup(name)) or None
+        if entry and entry.is_pyglobal:
+            self._emit_class_private_warning(entry.pos, name)
+        return entry
 
     def lookup_here(self, name):
         # Look up in this scope only, return None if not found.
-        name = self.mangle_class_private_name(name)
+
+        entry = self.entries.get(self.mangle_class_private_name(name), None)
+        if entry:
+            return entry
+        # Also check the unmangled name in the current scope
+        # (even if mangling should give us something else).
+        # This is to support things like global __foo which makes a declaration for __foo
+        return self.entries.get(name, None)
+
+    def lookup_here_unmangled(self, name):
         return self.entries.get(name, None)
 
     def lookup_target(self, name):
         # Look up name in this scope only. Declare as Python
         # variable if not found.
         entry = self.lookup_here(name)
+        if not entry:
+            entry = self.lookup_here_unmangled(name)
+            if entry and entry.is_pyglobal:
+                self._emit_class_private_warning(entry.pos, name)
         if not entry:
             entry = self.declare_var(name, py_object_type, None)
         return entry
@@ -986,6 +1010,11 @@ class Scope(object):
         operands = [FakeOperand(pos, type=type) for type in types]
         return self.lookup_operator(operator, operands)
 
+    def _emit_class_private_warning(self, pos, name):
+        warning(pos, "Global name %s matched from within class scope "
+                            "in contradiction to to Python 'class private name' rules. "
+                            "This may change in a future release." % name, 1)
+
     def use_utility_code(self, new_code):
         self.global_scope().use_utility_code(new_code)
 
@@ -2006,9 +2035,6 @@ class ClassScope(Scope):
         # a few utilitycode names need to specifically be ignored
         if name and name.lower().startswith("__pyx_"):
             return name
-        return self.mangle_special_name(name)
-
-    def mangle_special_name(self, name):
         if name and name.startswith('__') and not name.endswith('__'):
             name = EncodedString('_%s%s' % (self.class_name.lstrip('_'), name))
         return name
@@ -2049,7 +2075,7 @@ class PyClassScope(ClassScope):
     def declare_var(self, name, type, pos,
                     cname = None, visibility = 'private',
                     api = 0, in_pxd = 0, is_cdef = 0):
-        name = self.mangle_special_name(name)
+        name = self.mangle_class_private_name(name)
         if type is unspecified_type:
             type = py_object_type
         # Add an entry for a class attribute.
@@ -2179,7 +2205,7 @@ class CClassScope(ClassScope):
     def declare_var(self, name, type, pos,
                     cname = None, visibility = 'private',
                     api = 0, in_pxd = 0, is_cdef = 0):
-        name = self.mangle_special_name(name)
+        name = self.mangle_class_private_name(name)
         if is_cdef:
             # Add an entry for an attribute.
             if self.defined:

docs/src/userguide/migrating_to_cy30.rst

@@ -121,3 +121,39 @@ without actually calling it, e.g.
 
     # Explicitly disable the automatic initialisation of NumPy's C-API.
     <void>import_array
+
+Class-private name mangling
+===========================
+
+Cython has been updated to follow the `Python rules for class-private names
+<https://docs.python.org/3/tutorial/classes.html#private-variables>`_
+more closely. Essentially any name that starts with and doesn't end with 
+``__`` within a class is mangled with the class name. Most user code
+should be unaffected -- unlike in Python unmangled global names will
+still be matched to ensure it is possible to access C names
+beginning with ``__``::
+
+     cdef extern void __foo()
+     
+     class C: # or "cdef class"
+        def call_foo(self):
+            return __foo() # still calls the global name
+            
+What will no-longer work is overriding methods starting with ``__`` in
+a ``cdef class``::
+
+    cdef class Base:
+        cdef __bar(self):
+            return 1
+
+        def call_bar(self):
+            return self.__bar()
+
+    cdef class Derived(Base):
+        cdef __bar(self):
+            return 2
+
+Here ``Base.__bar`` is mangled to ``_Base__bar`` and ``Derived.__bar``
+to ``_Derived__bar``. Therefore ``call_bar`` will always call 
+``_Base__bar``. This matches established Python behaviour and applies
+for ``def``, ``cdef`` and ``cpdef`` methods and attributes.

tests/run/methodmangling_T5.py

@@ -4,6 +4,7 @@
 # A small number of extra tests checking:
 # 1) this works correctly with pure-Python-mode decorators - methodmangling_pure.py.
 # 2) this works correctly with cdef classes - methodmangling_cdef.pyx
+# 3) with "error_on_unknown_names" - methodmangling_unknown_names.py
 
 class CyTest(object):
     """

tests/run/methodmangling_cdef.pyx

@@ -3,6 +3,11 @@
 def call_cdt_private_cdef(CDefTest o):
     return o._CDefTest__private_cdef()
 
+cdef __c_func():
+    return "cdef function"
+
+cdef __c_var = "Shouldn't see this"
+
 cdef class CDefTest:
     """
     >>> cd = CDefTest()
@@ -48,6 +53,32 @@ cdef class CDefTest:
             return o._CDefTest__x, o.__x, o.__private()
         return get(self)
 
+    def get_c_func(self):
+        """
+        Should still be able to access C function with __names
+        >>> CDefTest().get_c_func()
+        'cdef function'
+        """
+        return __c_func()
+
+    def get_c_func2(self):
+        """
+        Should find mangled name before C __name
+        >>> CDefTest().get_c_func2()
+        'lambda'
+        """
+        _CDefTest__c_func = lambda: "lambda"
+        return __c_func()
+
+    def get_c_var(self):
+        """
+        >>> CDefTest().get_c_var()
+        'c var'
+        """
+        global __c_var
+        __c_var = "c var"
+        return __c_var
+
 def call_inpdx_private_cdef(InPxd o):
     return o._InPxd__private_cdef()
 

tests/run/methodmangling_unknown_names.py

+# mode: run
+# tag: allow_unknown_names, pure2.0, pure3.0
+
+class Test(object):
+    def run(self):
+        """
+        >>> Test().run()
+        NameError1
+        NameError2
+        found mangled
+        """
+        try:
+            print(__something)
+        except NameError:
+            print("NameError1") # correct - shouldn't exist
+        globals()['__something'] = 'found unmangled'
+        try:
+            print(__something)
+        except NameError:
+            print("NameError2") # correct - shouldn't exist
+        globals()['_Test__something'] = 'found mangled'
+        try:
+            print(__something) # should print this
+        except NameError:
+            print("NameError3")