Implement cdef dataclasses (GH-3400)

New decorator/function "@cython.dataclasses.dataclass" and "cython.dataclasses.field()" to mark dataclasses and their fields.

Tries to match the interface provided by a regular dataclass as much as possible.
This means taking the types from the dataclasses module if available (so they match exactly) or a fallback Python version that just implements the core parts (executed with "PyRun_String()" in the C source).

Use of placeholders in generated "__init__" code means the code in the C file isn't hugely readable. Probably not a huge issue, but don't really see a way round that.

As part of this I've also also implemented a Cython version of "typing.ClassVar". Although really designed for use with dataclasses it behaves sensibly when used in types in a normal cdef class. This is worth documenting more thoroughly.

Closes https://github.com/cython/cython/issues/2903

Cython/Compiler/Builtin.py

@@ -4,7 +4,8 @@
 
 from __future__ import absolute_import
 
-from .Symtab import BuiltinScope, StructOrUnionScope
+from .StringEncoding import EncodedString
+from .Symtab import BuiltinScope, StructOrUnionScope, ModuleScope
 from .Code import UtilityCode
 from .TypeSlots import Signature
 from . import PyrexTypes
@@ -451,3 +452,56 @@ def init_builtins():
 
 
 init_builtins()
+
+##############################
+# Support for a few standard library modules that Cython understands (currently typing and dataclasses)
+##############################
+_known_module_scopes = {}
+
+def get_known_standard_library_module_scope(module_name):
+    mod = _known_module_scopes.get(module_name)
+    if mod:
+        return mod
+
+    if module_name == "typing":
+        mod = ModuleScope(module_name, None, None)
+        for name, tp in [
+                ('Dict', dict_type),
+                ('List', list_type),
+                ('Tuple', tuple_type),
+                ('Set', set_type),
+                ('FrozenSet', frozenset_type),
+                ]:
+            name = EncodedString(name)
+            if name == "Tuple":
+                indexed_type = PyrexTypes.PythonTupleTypeConstructor(EncodedString("typing."+name), tp)
+            else:
+                indexed_type = PyrexTypes.PythonTypeConstructor(EncodedString("typing."+name), tp)
+            entry = mod.declare_type(name, indexed_type, pos = None)
+
+        for name in ['ClassVar', 'Optional']:
+            indexed_type = PyrexTypes.SpecialPythonTypeConstructor(EncodedString("typing."+name))
+            entry = mod.declare_type(name, indexed_type, pos = None)
+        _known_module_scopes[module_name] = mod
+    elif module_name == "dataclasses":
+        mod = ModuleScope(module_name, None, None)
+        indexed_type = PyrexTypes.SpecialPythonTypeConstructor(EncodedString("dataclasses.InitVar"))
+        entry = mod.declare_type(EncodedString("InitVar"), indexed_type, pos = None)
+        _known_module_scopes[module_name] = mod
+    return mod
+
+
+def get_known_standard_library_entry(qualified_name):
+    name_parts = qualified_name.split(".")
+    module_name = EncodedString(name_parts[0])
+    rest = name_parts[1:]
+
+    if len(rest) > 1:  # for now, we don't know how to deal with any nested modules
+        return None
+
+    mod = get_known_standard_library_module_scope(module_name)
+
+    # eventually handle more sophisticated multiple lookups if needed
+    if mod and rest:
+        return mod.lookup_here(rest[0])
+    return None

Cython/Compiler/CythonScope.py

@@ -6,6 +6,7 @@ from .UtilityCode import CythonUtilityCode
 from .Errors import error
 from .Scanning import StringSourceDescriptor
 from . import MemoryView
+from .StringEncoding import EncodedString
 
 
 class CythonScope(ModuleScope):
@@ -135,9 +136,16 @@ class CythonScope(ModuleScope):
         for ext_type in ext_types:
             ext_type.is_cython_builtin_type = 1
 
-
         # self.entries["array"] = view_utility_scope.entries.pop("array")
 
+        # dataclasses scope
+        dc_str = EncodedString(u'dataclasses')
+        dataclassesscope = ModuleScope(dc_str, self, context=None)
+        self.declare_module(dc_str, dataclassesscope, pos=None).as_module = dataclassesscope
+        dataclassesscope.is_cython_builtin = True
+        dataclassesscope.pxd_file_loaded = True
+        # doesn't actually have any contents
+
 
 def create_cython_scope(context):
     # One could in fact probably make it a singleton,

Cython/Compiler/ExprNodes.py

@@ -1183,6 +1183,15 @@ class ExprNode(Node):
                 kwargs[attr_name] = value
         return cls(node.pos, **kwargs)
 
+    def get_known_standard_library_import(self):
+        """
+        Gets the module.path that this node was imported from.
+
+        Many nodes do not have one, or it is ambiguous, in which case
+        this function returns a false value.
+        """
+        return None
+
 
 class AtomicExprNode(ExprNode):
     #  Abstract base class for expression nodes which have
@@ -2038,13 +2047,25 @@ class NameNode(AtomicExprNode):
                           "'%s' cannot be specialized since its type is not a fused argument to this function" %
                           self.name)
                     atype = error_type
+            visibility = 'private'
+            if 'dataclasses.dataclass' in env.directives:
+                # handle "frozen" directive - full inspection of the dataclass directives happens
+                # in Dataclass.py
+                frozen_directive = None
+                dataclass_directive = env.directives['dataclasses.dataclass']
+                if dataclass_directive:
+                    dataclass_directive_kwds = dataclass_directive[1]
+                    frozen_directive = dataclass_directive_kwds.get('frozen', None)
+                is_frozen = frozen_directive and frozen_directive.is_literal and frozen_directive.value
+                if atype.is_pyobject or atype.can_coerce_to_pyobject(env):
+                    visibility = 'readonly' if is_frozen else 'public'
+                    # If the object can't be coerced that's fine - we just don't create a property
             if as_target and env.is_c_class_scope and not (atype.is_pyobject or atype.is_error):
-                # TODO: this will need revising slightly if either cdef dataclasses or
-                # annotated cdef attributes are implemented
+                # TODO: this will need revising slightly if annotated cdef attributes are implemented
                 atype = py_object_type
                 warning(annotation.pos, "Annotation ignored since class-level attributes must be Python objects. "
                         "Were you trying to set up an instance attribute?", 2)
-            entry = self.entry = env.declare_var(name, atype, self.pos, is_cdef=not as_target)
+            entry = self.entry = env.declare_var(name, atype, self.pos, is_cdef=not as_target, visibility=visibility)
         # Even if the entry already exists, make sure we're supplying an annotation if we can.
         if annotation and not entry.annotation:
             entry.annotation = annotation
@@ -2057,6 +2078,10 @@ class NameNode(AtomicExprNode):
             entry = env.lookup(self.name)
         if entry and entry.as_module:
             return entry.as_module
+        if entry and entry.known_standard_library_import:
+            scope = Builtin.get_known_standard_library_module_scope(entry.known_standard_library_import)
+            if scope and scope.is_module_scope:
+                return scope
         return None
 
     def analyse_as_type(self, env):
@@ -2071,6 +2096,10 @@ class NameNode(AtomicExprNode):
             entry = env.lookup(self.name)
         if entry and entry.is_type:
             return entry.type
+        elif entry and entry.known_standard_library_import:
+            entry = Builtin.get_known_standard_library_entry(entry.known_standard_library_import)
+            if entry and entry.is_type:
+                return entry.type
         else:
             return None
 
@@ -2098,9 +2127,14 @@ class NameNode(AtomicExprNode):
                 self.entry = env.lookup_assignment_expression_target(self.name)
             else:
                 self.entry = env.lookup_here(self.name)
+        if self.entry:
+            self.entry.known_standard_library_import = ""  # already exists somewhere and so is now ambiguous
         if not self.entry and self.annotation is not None:
             # name : type = ...
-            self.declare_from_annotation(env, as_target=True)
+            is_dataclass = 'dataclasses.dataclass' in env.directives
+            # In a dataclass, an assignment should not prevent a name from becoming an instance attribute.
+            # Hence, "as_target = not is_dataclass".
+            self.declare_from_annotation(env, as_target=not is_dataclass)
         if not self.entry:
             if env.directives['warn.undeclared']:
                 warning(self.pos, "implicit declaration of '%s'" % self.name, 1)
@@ -2609,6 +2643,11 @@ class NameNode(AtomicExprNode):
                 style, text = 'c_call', 'c function (%s)'
             code.annotate(pos, AnnotationItem(style, text % self.type, size=len(self.name)))
 
+    def get_known_standard_library_import(self):
+        if self.entry:
+            return self.entry.known_standard_library_import
+        return None
+
 class BackquoteNode(ExprNode):
     #  `expr`
     #
@@ -2718,6 +2757,9 @@ class ImportNode(ExprNode):
             code.error_goto_if_null(self.result(), self.pos)))
         self.generate_gotref(code)
 
+    def get_known_standard_library_import(self):
+        return self.module_name.value
+
 
 class IteratorNode(ExprNode):
     #  Used as part of for statement implementation.
@@ -3630,9 +3672,9 @@ class IndexNode(_IndexingBaseNode):
 
     def analyse_as_type(self, env):
         base_type = self.base.analyse_as_type(env)
-        if base_type and not base_type.is_pyobject:
-            if base_type.is_cpp_class:
-                if isinstance(self.index, TupleNode):
+        if base_type and (not base_type.is_pyobject or base_type.python_type_constructor_name):
+            if base_type.is_cpp_class or base_type.python_type_constructor_name:
+                if self.index.is_sequence_constructor:
                     template_values = self.index.args
                 else:
                     template_values = [self.index]
@@ -7489,6 +7531,12 @@ class AttributeNode(ExprNode):
             style, text = 'c_attr', 'c attribute (%s)'
         code.annotate(self.pos, AnnotationItem(style, text % self.type, size=len(self.attribute)))
 
+    def get_known_standard_library_import(self):
+        module_name = self.obj.get_known_standard_library_import()
+        if module_name:
+            return StringEncoding.EncodedString("%s.%s" % (module_name, self.attribute))
+        return None
+
 
 #-------------------------------------------------------------------
 #
@@ -9029,6 +9077,11 @@ class DictNode(ExprNode):
         for item in self.key_value_pairs:
             item.annotate(code)
 
+    def as_python_dict(self):
+        # returns a dict with constant keys and Node values
+        # (only works on DictNodes where the keys are ConstNodes or PyConstNode)
+        return dict([(key.value, value) for key, value in self.key_value_pairs])
+
 
 class DictItemNode(ExprNode):
     # Represents a single item in a DictNode
@@ -9859,6 +9912,9 @@ class LambdaNode(InnerFunctionNode):
     name = StringEncoding.EncodedString('<lambda>')
 
     def analyse_declarations(self, env):
+        if hasattr(self, "lambda_name"):
+            # this if-statement makes it safe to run twice
+            return
         self.lambda_name = self.def_node.lambda_name = env.next_id('lambda')
         self.def_node.no_assignment_synthesis = True
         self.def_node.pymethdef_required = True
@@ -9888,6 +9944,9 @@ class GeneratorExpressionNode(LambdaNode):
     binding = False
 
     def analyse_declarations(self, env):
+        if hasattr(self, "genexpr_name"):
+            # this if-statement makes it safe to run twice
+            return
         self.genexpr_name = env.next_id('genexpr')
         super(GeneratorExpressionNode, self).analyse_declarations(env)
         # No pymethdef required

Cython/Compiler/Naming.py

@@ -135,6 +135,10 @@ type_dict_guard_temp = pyrex_prefix + "typedict_guard"
 cython_runtime_cname   = pyrex_prefix + "cython_runtime"
 cyfunction_type_cname = pyrex_prefix + "CyFunctionType"
 fusedfunction_type_cname = pyrex_prefix + "FusedFunctionType"
+# the name "dflt" was picked by analogy with the CPython dataclass module which stores
+# the default values in variables named f"_dflt_{field.name}" in a hidden scope that's
+# passed to the __init__ function. (The name is unimportant to the exact workings though)
+dataclass_field_default_cname = pyrex_prefix + "dataclass_dflt"
 
 global_code_object_cache_find = pyrex_prefix + 'find_code_object'
 global_code_object_cache_insert = pyrex_prefix + 'insert_code_object'

Cython/Compiler/Nodes.py

@@ -585,7 +585,9 @@ class CArrayDeclaratorNode(CDeclaratorNode):
     child_attrs = ["base", "dimension"]
 
     def analyse(self, base_type, env, nonempty=0, visibility=None, in_pxd=False):
-        if (base_type.is_cpp_class and base_type.is_template_type()) or base_type.is_cfunction:
+        if ((base_type.is_cpp_class and base_type.is_template_type()) or
+                base_type.is_cfunction or
+                base_type.python_type_constructor_name):
             from .ExprNodes import TupleNode
             if isinstance(self.dimension, TupleNode):
                 args = self.dimension.args
@@ -597,7 +599,7 @@ class CArrayDeclaratorNode(CDeclaratorNode):
                 error(args[ix].pos, "Template parameter not a type")
                 base_type = error_type
             else:
-                base_type = base_type.specialize_here(self.pos, values)
+                base_type = base_type.specialize_here(self.pos, env, values)
             return self.base.analyse(base_type, env, nonempty=nonempty, visibility=visibility, in_pxd=in_pxd)
         if self.dimension:
             self.dimension = self.dimension.analyse_const_expression(env)
@@ -963,6 +965,11 @@ class CArgDeclNode(Node):
         base_type, arg_type = annotation.analyse_type_annotation(env, assigned_value=self.default)
         if base_type is not None:
             self.base_type = base_type
+        if arg_type and arg_type.python_type_constructor_name == "typing.Optional":
+            self.or_none = True
+            arg_type = arg_type.resolve()
+        if arg_type and arg_type.is_pyobject and not self.or_none:
+            self.not_none = True
         return arg_type
 
     def calculate_default_value_code(self, code):
@@ -1064,7 +1071,13 @@ class CSimpleBaseTypeNode(CBaseTypeNode):
                     else:
                         scope = None
                         break
-
+                if scope is None and len(self.module_path) == 1:
+                    # (may be possible to handle longer module paths?)
+                    # TODO: probably not the best place to declare it?
+                    from .Builtin import get_known_standard_library_module_scope
+                    found_entry = env.lookup(self.module_path[0])
+                    if found_entry and found_entry.known_standard_library_import:
+                        scope = get_known_standard_library_module_scope(found_entry.known_standard_library_import)
                 if scope is None:
                     # Maybe it's a cimport.
                     scope = env.find_imported_module(self.module_path, self.pos)
@@ -1189,20 +1202,23 @@ class TemplatedTypeNode(CBaseTypeNode):
             base_type = self.base_type_node.analyse(env)
         if base_type.is_error: return base_type
 
-        if base_type.is_cpp_class and base_type.is_template_type():
+        if ((base_type.is_cpp_class and base_type.is_template_type()) or
+                base_type.python_type_constructor_name):
             # Templated class
             if self.keyword_args and self.keyword_args.key_value_pairs:
-                error(self.pos, "c++ templates cannot take keyword arguments")
+                tp = "c++ templates" if base_type.is_cpp_class else "indexed types"
+                error(self.pos, "%s cannot take keyword arguments" % tp)
                 self.type = PyrexTypes.error_type
             else:
                 template_types = []
                 for template_node in self.positional_args:
                     type = template_node.analyse_as_type(env)
-                    if type is None:
+                    if type is None and base_type.is_cpp_class:
                         error(template_node.pos, "unknown type in template argument")
                         type = error_type
+                    # for indexed_pytype we can be a bit more flexible and pass None
                     template_types.append(type)
-                self.type = base_type.specialize_here(self.pos, template_types)
+                self.type = base_type.specialize_here(self.pos, env, template_types)
 
         elif base_type.is_pyobject:
             # Buffer
@@ -5066,6 +5082,7 @@ class CClassDefNode(ClassDefNode):
     check_size = None
     decorators = None
     shadow = False
+    is_dataclass = False
 
     @property
     def punycode_class_name(self):
@@ -5115,6 +5132,8 @@ class CClassDefNode(ClassDefNode):
 
         if env.in_cinclude and not self.objstruct_name:
             error(self.pos, "Object struct name specification required for C class defined in 'extern from' block")
+        if "dataclasses.dataclass" in env.directives:
+            self.is_dataclass = True
         if self.decorators:
             error(self.pos, "Decorators not allowed on cdef classes (used on type '%s')" % self.class_name)
         self.base_type = None
@@ -5846,6 +5865,13 @@ class SingleAssignmentNode(AssignmentNode):
                 self.lhs.analyse_assignment_expression_target_declaration(env)
             else:
                 self.lhs.analyse_target_declaration(env)
+                # if an entry doesn't exist that just implies that lhs isn't made up purely
+                # of AttributeNodes and NameNodes - it isn't useful as a known path to
+                # a standard library module
+                if (self.lhs.is_attribute or self.lhs.is_name) and self.lhs.entry and not self.lhs.entry.known_standard_library_import:
+                    stdlib_import_name = self.rhs.get_known_standard_library_import()
+                    if stdlib_import_name:
+                        self.lhs.entry.known_standard_library_import = stdlib_import_name
 
     def analyse_types(self, env, use_temp=0):
         from . import ExprNodes
@@ -8548,7 +8574,8 @@ class CImportStatNode(StatNode):
                 env.declare_module(top_name, top_module_scope, self.pos)
         else:
             name = self.as_name or self.module_name
-            env.declare_module(name, module_scope, self.pos)
+            entry = env.declare_module(name, module_scope, self.pos)
+            entry.known_standard_library_import = self.module_name
         if self.module_name in utility_code_for_cimports:
             env.use_utility_code(utility_code_for_cimports[self.module_name]())
 
@@ -8662,6 +8689,14 @@ class FromImportStatNode(StatNode):
                 self.import_star = 1
             else:
                 target.analyse_target_declaration(env)
+                if target.entry:
+                    if target.get_known_standard_library_import() is None:
+                        target.entry.known_standard_library_import = EncodedString(
+                            "%s.%s" % (self.module.module_name.value, name))
+                else:
+                    # it isn't unambiguous
+                    target.entry.known_standard_library_import = ""
+
 
     def analyse_expressions(self, env):
         from . import ExprNodes

Cython/Compiler/Optimize.py

@@ -195,16 +195,6 @@ class IterationTransform(Visitor.EnvTransform):
             annotation = iterable.entry.annotation.expr
             if annotation.is_subscript:
                 annotation = annotation.base  # container base type
-            # FIXME: generalise annotation evaluation => maybe provide a "qualified name" also for imported names?
-            if annotation.is_name:
-                if annotation.entry and annotation.entry.qualified_name == 'typing.Dict':
-                    annotation_type = Builtin.dict_type
-                elif annotation.name == 'Dict':
-                    annotation_type = Builtin.dict_type
-                if annotation.entry and annotation.entry.qualified_name in ('typing.Set', 'typing.FrozenSet'):
-                    annotation_type = Builtin.set_type
-                elif annotation.name in ('Set', 'FrozenSet'):
-                    annotation_type = Builtin.set_type
 
         if Builtin.dict_type in (iterable.type, annotation_type):
             # like iterating over dict.keys()

Cython/Compiler/Options.py

@@ -302,6 +302,12 @@ def normalise_encoding_name(option_name, encoding):
             return name
     return encoding
 
+# use as a sential value to defer analysis of the arguments
+# instead of analysing them in InterpretCompilerDirectives. The dataclass directives are quite
+# complicated and it's easier to deal with them at the point the dataclass is created
+class DEFER_ANALYSIS_OF_ARGUMENTS:
+    pass
+DEFER_ANALYSIS_OF_ARGUMENTS = DEFER_ANALYSIS_OF_ARGUMENTS()
 
 # Override types possibilities above, if needed
 directive_types = {
@@ -328,6 +334,8 @@ directive_types = {
     'c_string_encoding': normalise_encoding_name,
     'trashcan': bool,
     'total_ordering': bool,
+    'dataclasses.dataclass': DEFER_ANALYSIS_OF_ARGUMENTS,
+    'dataclasses.field': DEFER_ANALYSIS_OF_ARGUMENTS,
 }
 
 for key, val in _directive_defaults.items():
@@ -372,6 +380,7 @@ directive_scopes = {  # defaults to available everywhere
     'iterable_coroutine': ('module', 'function'),
     'trashcan' : ('cclass',),
     'total_ordering': ('cclass', ),
+    'dataclasses.dataclass' : ('class', 'cclass',),
     'cpp_locals': ('module', 'function', 'cclass'),  # I don't think they make sense in a with_statement
 }
 

Cython/Compiler/ParseTreeTransforms.py

@@ -954,7 +954,6 @@ class InterpretCompilerDirectives(CythonTransform):
             for pos, name, as_name, kind in node.imported_names:
                 full_name = submodule + name
                 qualified_name = u"cython." + full_name
-
                 if self.is_parallel_directive(qualified_name, node.pos):
                     # from cython cimport parallel, or
                     # from cython.parallel cimport parallel, prange, ...
@@ -964,6 +963,10 @@ class InterpretCompilerDirectives(CythonTransform):
                     if kind is not None:
                         self.context.nonfatal_error(PostParseError(pos,
                             "Compiler directive imports must be plain imports"))
+                elif full_name in ['dataclasses', 'typing']:
+                    self.directive_names[as_name or name] = full_name
+                    # unlike many directives, still treat it as a regular module
+                    newimp.append((pos, name, as_name, kind))
                 else:
                     newimp.append((pos, name, as_name, kind))
 
@@ -1105,7 +1108,7 @@ class InterpretCompilerDirectives(CythonTransform):
                 if directivetype is bool:
                     arg = ExprNodes.BoolNode(node.pos, value=True)
                     return [self.try_to_parse_directive(optname, [arg], None, node.pos)]
-                elif directivetype is None:
+                elif directivetype is None or directivetype is Options.DEFER_ANALYSIS_OF_ARGUMENTS:
                     return [(optname, None)]
                 else:
                     raise PostParseError(
@@ -1160,7 +1163,7 @@ class InterpretCompilerDirectives(CythonTransform):
             if len(args) != 0:
                 raise PostParseError(pos,
                     'The %s directive takes no prepositional arguments' % optname)
-            return optname, dict([(key.value, value) for key, value in kwds.key_value_pairs])
+            return optname, kwds.as_python_dict()
         elif directivetype is list:
             if kwds and len(kwds.key_value_pairs) != 0:
                 raise PostParseError(pos,
@@ -1172,6 +1175,9 @@ class InterpretCompilerDirectives(CythonTransform):
                 raise PostParseError(pos,
                     'The %s directive takes one compile-time string argument' % optname)
             return (optname, directivetype(optname, str(args[0].value)))
+        elif directivetype is Options.DEFER_ANALYSIS_OF_ARGUMENTS:
+            # signal to pass things on without processing
+            return (optname, (args, kwds.as_python_dict()))
         else:
             assert False
 
@@ -1239,7 +1245,8 @@ class InterpretCompilerDirectives(CythonTransform):
                         name, value = directive
                         if self.directives.get(name, object()) != value:
                             directives.append(directive)
-                        if directive[0] == 'staticmethod':
+                        if (directive[0] == 'staticmethod' or
+                                (directive[0] == 'dataclasses.dataclass' and scope_name == 'class')):
                             both.append(dec)
                     # Adapt scope type based on decorators that change it.
                     if directive[0] == 'cclass' and scope_name == 'class':
@@ -1248,6 +1255,12 @@ class InterpretCompilerDirectives(CythonTransform):
                 realdecs.append(dec)
         if realdecs and (scope_name == 'cclass' or
                          isinstance(node, (Nodes.CClassDefNode, Nodes.CVarDefNode))):
+            for realdec in realdecs:
+                realdec = realdec.decorator
+                if ((realdec.is_name and realdec.name == "dataclass") or
+                        (realdec.is_attribute and realdec.attribute == "dataclass")):
+                    error(realdec.pos,
+                          "Use '@cython.dataclasses.dataclass' on cdef classes to create a dataclass")
             # Note - arbitrary C function decorators are caught later in DecoratorTransform
             raise PostParseError(realdecs[0].pos, "Cdef functions/classes cannot take arbitrary decorators.")
         node.decorators = realdecs[::-1] + both[::-1]
@@ -1906,6 +1919,9 @@ if VALUE is not None:
 
     def visit_CClassDefNode(self, node):
         node = self.visit_ClassDefNode(node)
+        if node.scope and 'dataclasses.dataclass' in node.scope.directives:
+            from .Dataclass import handle_cclass_dataclass
+            handle_cclass_dataclass(node, node.scope.directives['dataclasses.dataclass'], self)
         if node.scope and node.scope.implemented and node.body:
             stats = []
             for entry in node.scope.var_entries:

Cython/Compiler/PyrexTypes.py

@@ -194,6 +194,7 @@ class PyrexType(BaseType):
     #  is_string             boolean     Is a C char * type
     #  is_pyunicode_ptr      boolean     Is a C PyUNICODE * type
     #  is_cpp_string         boolean     Is a C++ std::string type
+    #  python_type_constructor_name     string or None     non-None if it is a Python type constructor that can be indexed/"templated"
     #  is_unicode_char       boolean     Is either Py_UCS4 or Py_UNICODE
     #  is_returncode         boolean     Is used only to signal exceptions
     #  is_error              boolean     Is the dummy error type
@@ -257,6 +258,7 @@ class PyrexType(BaseType):
     is_struct_or_union = 0
     is_cpp_class = 0
     is_optional_cpp_class = 0
+    python_type_constructor_name = None
     is_cpp_string = 0
     is_struct = 0
     is_enum = 0
@@ -1507,12 +1509,14 @@ class PyExtensionType(PyObjectType):
     #  early_init       boolean          Whether to initialize early (as opposed to during module execution).
     #  defered_declarations [thunk]      Used to declare class hierarchies in order
     #  check_size       'warn', 'error', 'ignore'    What to do if tp_basicsize does not match
+    #  dataclass_fields  OrderedDict nor None   Used for inheriting from dataclasses
 
     is_extension_type = 1
     has_attributes = 1
     early_init = 1
 
     objtypedef_cname = None
+    dataclass_fields = None
 
     def __init__(self, name, typedef_flag, base_type, is_external=0, check_size=None):
         self.name = name
@@ -3872,7 +3876,7 @@ class CppClassType(CType):
                 T.get_fused_types(result, seen)
         return result
 
-    def specialize_here(self, pos, template_values=None):
+    def specialize_here(self, pos, env, template_values=None):
         if not self.is_template_type():
             error(pos, "'%s' type is not a template" % self)
             return error_type
@@ -4400,6 +4404,102 @@ class ErrorType(PyrexType):
         return "dummy"
 
 
+class PythonTypeConstructor(PyObjectType):
+    """Used to help Cython interpret indexed types from the typing module (or similar)
+    """
+
+    def __init__(self, name, base_type=None):
+        self.python_type_constructor_name = name
+        self.base_type = base_type
+
+    def specialize_here(self, pos, env, template_values=None):
+        if self.base_type:
+            # for a lot of the typing classes it doesn't really matter what the template is
+            # (i.e. typing.Dict[int] is really just a dict)
+            return self.base_type
+        return self
+
+    def __repr__(self):
+        if self.base_type:
+            return "%s[%r]" % (self.name, self.base_type)
+        else:
+            return self.name
+
+    def is_template_type(self):
+        return True
+
+
+class PythonTupleTypeConstructor(PythonTypeConstructor):
+    def specialize_here(self, pos, env, template_values=None):
+        if (template_values and None not in template_values and
+                not any(v.is_pyobject for v in template_values)):
+            entry = env.declare_tuple_type(pos, template_values)
+            if entry:
+                return entry.type
+        return super(PythonTupleTypeConstructor, self).specialize_here(pos, env, template_values)
+
+
+class SpecialPythonTypeConstructor(PythonTypeConstructor):
+    """
+    For things like ClassVar, Optional, etc, which have extra features on top of being
+    a "templated" type.
+    """
+
+    def __init__(self, name, template_type=None):
+        super(SpecialPythonTypeConstructor, self).__init__(name, None)
+        if (name == "typing.ClassVar" and template_type
+                and not template_type.is_pyobject):
+            # because classvars end up essentially used as globals they have
+            # to be PyObjects. Try to find the nearest suitable type (although
+            # practically I doubt this matters).
+            py_type_name = template_type.py_type_name()
+            if py_type_name:
+                from .Builtin import builtin_scope
+                template_type = (builtin_scope.lookup_type(py_type_name)
+                                        or py_object_type)
+            else:
+                template_type = py_object_types
+        self.template_type = template_type
+
+    def __repr__(self):
+        if self.template_type:
+            return "%s[%r]" % (self.name, self.template_type)
+        else:
+            return self.name
+
+    def is_template_type(self):
+        return self.template_type is None
+
+    def resolve(self):
+        if self.template_type:
+            return self.template_type.resolve()
+        else:
+            return self
+
+    def specialize_here(self, pos, env, template_values=None):
+        if len(template_values) != 1:
+            error(pos, "'%s' takes exactly one template argument." % self.name)
+        # return a copy of the template type with python_type_constructor_name as an attribute
+        # so it can be identified, and a resolve function that gets back to
+        # the original type (since types are usually tested with "is")
+        new_type = template_values[0]
+        if self.python_type_constructor_name == "typing.ClassVar":
+            # classvar must remain a py_object_type
+            new_type = py_object_type
+        if (self.python_type_constructor_name == "typing.Optional" and
+                not new_type.is_pyobject):
+            # optional must be a py_object, but can be a specialized py_object
+            new_type = py_object_type
+        return SpecialPythonTypeConstructor(
+            self.python_type_constructor_name,
+            template_type = template_values[0])
+
+    def __getattr__(self, name):
+        if self.template_type:
+            return getattr(self.template_type, name)
+        return super(SpecialPythonTypeConstructor, self).__getattr__(name)
+
+
 rank_to_type_name = (
     "char",          # 0
     "short",         # 1

Cython/Compiler/Symtab.py

@@ -159,6 +159,9 @@ class Entry(object):
     #                              is a specialization
     # is_cgetter       boolean    Is a c-level getter function
     # is_cpp_optional  boolean    Entry should be declared as std::optional (cpp_locals directive)
+    # known_standard_library_import     Either None (default), an empty string (definitely can't be determined)
+    #                             or a string of "modulename.something.attribute"
+    #                             Used for identifying imports from typing/dataclasses etc
 
     # TODO: utility_code and utility_code_definition serves the same purpose...
 
@@ -166,6 +169,7 @@ class Entry(object):
     borrowed = 0
     init = ""
     annotation = None
+    pep563_annotation = None
     visibility = 'private'
     is_builtin = 0
     is_cglobal = 0
@@ -231,6 +235,7 @@ class Entry(object):
     outer_entry = None
     is_cgetter = False
     is_cpp_optional = False
+    known_standard_library_import = None
 
     def __init__(self, name, cname, type, pos = None, init = None):
         self.name = name
@@ -998,13 +1003,27 @@ class Scope(object):
             entry = self.declare_var(name, py_object_type, None)
         return entry
 
-    def lookup_type(self, name):
-        entry = self.lookup(name)
+    def _type_or_specialized_type_from_entry(self, entry):
         if entry and entry.is_type:
             if entry.type.is_fused and self.fused_to_specific:
                 return entry.type.specialize(self.fused_to_specific)
             return entry.type
-        return None
+
+    def lookup_type(self, name):
+        entry = self.lookup(name)
+        # The logic here is:
+        #  1. if entry is a type then return it (and maybe specialize it)
+        #  2. if the entry comes from a known standard library import then follow that
+        #  3. repeat step 1 with the (possibly) updated entry
+
+        tp = self._type_or_specialized_type_from_entry(entry)
+        if tp:
+            return tp
+        # allow us to find types from the "typing" module and similar
+        if entry and entry.known_standard_library_import:
+            from .Builtin import get_known_standard_library_entry
+            entry = get_known_standard_library_entry(entry.known_standard_library_import)
+        return self._type_or_specialized_type_from_entry(entry)
 
     def lookup_operator(self, operator, operands):
         if operands[0].type.is_cpp_class:
@@ -2284,6 +2303,15 @@ class CClassScope(ClassScope):
                     cname = None, visibility = 'private',
                     api = 0, in_pxd = 0, is_cdef = 0):
         name = self.mangle_class_private_name(name)
+
+        if type.python_type_constructor_name == "typing.ClassVar":
+            is_cdef = 0
+            type = type.resolve()
+
+        if (type.python_type_constructor_name == "dataclasses.InitVar" and
+                'dataclasses.dataclass' not in self.directives):
+            error(pos, "Use of cython.dataclasses.InitVar does not make sense outside a dataclass")
+
         if is_cdef:
             # Add an entry for an attribute.
             if self.defined:
@@ -2530,6 +2558,7 @@ class CClassScope(ClassScope):
                 base_entry.name, adapt(base_entry.cname),
                 base_entry.type, None, 'private')
             entry.is_variable = 1
+            entry.annotation = base_entry.annotation
             self.inherited_var_entries.append(entry)
 
         # If the class defined in a pxd, specific entries have not been added.

Cython/Shadow.py

@@ -525,6 +525,30 @@ class CythonDotParallel(object):
     # def threadsavailable(self):
         # return 1
 
+class CythonDotImportedFromElsewhere(object):
+    """
+    cython.dataclasses just shadows the standard library modules of the same name
+    """
+    def __init__(self, module):
+        self.__path__ = []
+        self.__file__ = None
+        self.__name__ = module
+        self.__package__ = module
+
+    def __getattr__(self, attr):
+        # we typically only expect this to be called once
+        from importlib import import_module
+        import sys
+        try:
+            mod = import_module(self.__name__)
+        except ImportError:
+            # but if they don't exist (Python is not sufficiently up-to-date) then
+            # you can't use them
+            raise AttributeError("%s: the standard library module %s is not available" %
+                                 (attr, self.__name__))
+        sys.modules['cython.%s' % self.__name__] = mod
+        return getattr(mod, attr)
+
 
 class CythonCImports(object):
     """
@@ -547,4 +571,7 @@ sys.modules['cython.parallel'] = CythonDotParallel()
 sys.modules['cython.cimports'] = CythonCImports('cython.cimports')
 sys.modules['cython.cimports.libc'] = CythonCImports('cython.cimports.libc')
 sys.modules['cython.cimports.libc.math'] = math
+# In pure Python mode @cython.dataclasses.dataclass and dataclass field should just
+# shadow the standard library ones (if they are available)
+dataclasses = sys.modules['cython.dataclasses'] = CythonDotImportedFromElsewhere('dataclasses')
 del math, sys

Cython/Utility/CommonStructures.c

+/////////////// FetchSharedCythonModule.proto ///////
+
+static PyObject *__Pyx_FetchSharedCythonABIModule(void);
+
+/////////////// FetchSharedCythonModule ////////////
+
+static PyObject *__Pyx_FetchSharedCythonABIModule(void) {
+    PyObject *abi_module = PyImport_AddModule((char*) __PYX_ABI_MODULE_NAME);
+    if (unlikely(!abi_module)) return NULL;
+    Py_INCREF(abi_module);
+    return abi_module;
+}
+
 /////////////// FetchCommonType.proto ///////////////
 
 #if !CYTHON_USE_TYPE_SPECS
@@ -8,15 +21,9 @@ static PyTypeObject* __Pyx_FetchCommonTypeFromSpec(PyObject *module, PyType_Spec
 
 /////////////// FetchCommonType ///////////////
 //@requires:ExtensionTypes.c::FixUpExtensionType
+//@requires: FetchSharedCythonModule
 //@requires:StringTools.c::IncludeStringH
 
-static PyObject *__Pyx_FetchSharedCythonABIModule(void) {
-    PyObject *abi_module = PyImport_AddModule((char*) __PYX_ABI_MODULE_NAME);
-    if (!abi_module) return NULL;
-    Py_INCREF(abi_module);
-    return abi_module;
-}
-
 static int __Pyx_VerifyCachedType(PyObject *cached_type,
                                const char *name,
                                Py_ssize_t basicsize,

Cython/Utility/Dataclasses.c

+///////////////////// ModuleLoader.proto //////////////////////////
+
+static PyObject* __Pyx_LoadInternalModule(const char* name, const char* fallback_code); /* proto */
+
+//////////////////// ModuleLoader ///////////////////////
+//@requires: CommonStructures.c::FetchSharedCythonModule
+
+static PyObject* __Pyx_LoadInternalModule(const char* name, const char* fallback_code) {
+    // We want to be able to use the contents of the standard library dataclasses module where available.
+    // If those objects aren't available (due to Python version) then a simple fallback is substituted
+    // instead, which largely just fails with a not-implemented error.
+    //
+    // The fallbacks are placed in the "shared abi module" as a convenient internal place to
+    // store them
+
+    PyObject *shared_abi_module = 0, *module = 0;
+
+    shared_abi_module = __Pyx_FetchSharedCythonABIModule();
+    if (!shared_abi_module) return NULL;
+
+    if (PyObject_HasAttrString(shared_abi_module, name)) {
+        PyObject* result = PyObject_GetAttrString(shared_abi_module, name);
+        Py_DECREF(shared_abi_module);
+        return result;
+    }
+
+    // the best and simplest case is simply to defer to the standard library (if available)
+    module = PyImport_ImportModule(name);
+    if (!module) {
+        PyObject *localDict, *runValue, *builtins, *modulename;
+        if (!PyErr_ExceptionMatches(PyExc_ImportError)) goto bad;
+        PyErr_Clear();  // this is reasonably likely (especially on older versions of Python)
+#if PY_MAJOR_VERSION < 3
+        modulename = PyBytes_FromFormat("_cython_" CYTHON_ABI ".%s", name);
+#else
+        modulename = PyUnicode_FromFormat("_cython_" CYTHON_ABI ".%s", name);
+#endif
+        if (!modulename) goto bad;
+#if PY_MAJOR_VERSION >= 3 && CYTHON_COMPILING_IN_CPYTHON
+        module = PyImport_AddModuleObject(modulename); // borrowed
+#else
+        module = PyImport_AddModule(PyBytes_AsString(modulename)); // borrowed
+#endif
+        Py_DECREF(modulename);
+        if (!module) goto bad;
+        Py_INCREF(module);
+        if (PyObject_SetAttrString(shared_abi_module, name, module) < 0) goto bad;
+        localDict = PyModule_GetDict(module); // borrowed
+        if (!localDict) goto bad;
+        builtins = PyEval_GetBuiltins(); // borrowed
+        if (!builtins) goto bad;
+        if (PyDict_SetItemString(localDict, "__builtins__", builtins) <0) goto bad;
+
+        runValue = PyRun_String(fallback_code, Py_file_input, localDict, localDict);
+        if (!runValue) goto bad;
+        Py_DECREF(runValue);
+    }
+    goto shared_cleanup;
+
+    bad:
+        Py_CLEAR(module);
+    shared_cleanup:
+        Py_XDECREF(shared_abi_module);
+    return module;
+}
+
+///////////////////// SpecificModuleLoader.proto //////////////////////
+//@substitute: tempita
+
+static PyObject* __Pyx_Load_{{cname}}_Module(void); /* proto */
+
+
+//////////////////// SpecificModuleLoader ///////////////////////
+//@requires: ModuleLoader
+
+static PyObject* __Pyx_Load_{{cname}}_Module(void) {
+    return __Pyx_LoadInternalModule("{{cname}}", {{py_code}});
+}

Cython/Utility/Dataclasses.py

+################### Dataclasses_fallback ###############################
+
+# This is the fallback dataclass code if the stdlib module isn't available.
+# It defines enough of the support types to be used with cdef classes
+# and to fail if used on regular types.
+
+# (Intended to be included as py code - not compiled)
+
+from collections import namedtuple
+try:
+    from types import MappingProxyType
+except ImportError:
+    # mutable fallback if unavailable
+    MappingProxyType = lambda x: x
+
+class _MISSING_TYPE(object):
+    pass
+MISSING = _MISSING_TYPE()
+
+_DataclassParams = namedtuple('_DataclassParams',
+    ["init", "repr", "eq", "order", "unsafe_hash", "frozen"])
+class Field(object):
+    __slots__ = ('name',
+                 'type',
+                 'default',
+                 'default_factory',
+                 'repr',
+                 'hash',
+                 'init',
+                 'compare',
+                 'metadata',
+                 '_field_type',  # Private: not to be used by user code.
+                 )
+
+    def __init__(self, default, default_factory, init, repr, hash, compare,
+                 metadata):
+        self.name = None
+        self.type = None
+        self.default = default
+        self.default_factory = default_factory
+        self.init = init
+        self.repr = repr
+        self.hash = hash
+        self.compare = compare
+        # Be aware that if MappingProxyType is unavailable (i.e. py2?) then we
+        # don't enforce non-mutability that the real module does
+        self.metadata = (MappingProxyType({})
+                         if metadata is None else
+                         MappingProxyType(metadata))
+        self._field_type = None
+
+    def __repr__(self):
+        return ('Field('
+                'name={0!r},'
+                'type={1!r},'
+                'default={2!r},'
+                'default_factory={3!r},'
+                'init={4!r},'
+                'repr={5!r},'
+                'hash={6!r},'
+                'compare={7!r},'
+                'metadata={8!r},'
+                ')'.format(self.name, self.type, self.default,
+                           self.default_factory, self.init,
+                           self.repr, self.hash, self.compare,
+                           self.metadata))
+
+# A sentinel object for default values to signal that a default
+# factory will be used.  This is given a nice repr() which will appear
+# in the function signature of dataclasses' constructors.
+class _HAS_DEFAULT_FACTORY_CLASS:
+    def __repr__(self):
+        return '<factory>'
+_HAS_DEFAULT_FACTORY = _HAS_DEFAULT_FACTORY_CLASS()
+
+def dataclass(*args, **kwds):
+    raise NotImplementedError("Standard library 'dataclasses' module"
+        "is unavailable, likely due to the version of Python you're using.")
+
+# Markers for the various kinds of fields and pseudo-fields.
+class _FIELD_BASE:
+    def __init__(self, name):
+        self.name = name
+    def __repr__(self):
+        return self.name
+_FIELD = _FIELD_BASE('_FIELD')
+_FIELD_CLASSVAR = _FIELD_BASE('_FIELD_CLASSVAR')
+_FIELD_INITVAR = _FIELD_BASE('_FIELD_INITVAR')
+
+def field(*ignore, **kwds):
+    default = kwds.pop("default", MISSING)
+    default_factory = kwds.pop("default_factory", MISSING)
+    init = kwds.pop("init", True)
+    repr = kwds.pop("repr", True)
+    hash = kwds.pop("hash", None)
+    compare = kwds.pop("compare", True)
+    metadata = kwds.pop("metadata", None)
+
+    if kwds:
+        raise ValueError("field received unexpected keyword arguments: %s"
+                         % list(kwds.keys()))
+    if default is not MISSING and default_factory is not MISSING:
+        raise ValueError('cannot specify both default and default_factory')
+    if ignore:
+        raise ValueError("'field' does not take any positional arguments")
+    return Field(default, default_factory, init, repr, hash, compare, metadata)

docs/examples/userguide/extension_types/dataclass.pyx

+cimport cython
+try:
+    import typing
+    import dataclasses
+except ImportError:
+    pass  # The modules don't actually have to exists for Cython to use them as annotations
+
+@cython.dataclasses.dataclass
+cdef class MyDataclass:
+    # fields can be declared using annotations
+    a: cython.int = 0
+    b: double = cython.dataclasses.field(default_factory = lambda: 10, repr=False)
+
+    # fields can also be declared using `cdef`:
+    cdef str c
+    c = "hello"  # assignment of default value on a separate line
+
+    # typing.InitVar and typing.ClassVar also work
+    d: dataclasses.InitVar[double] = 5
+    e: typing.ClassVar[list] = []

docs/src/tutorial/pure.rst

@@ -335,15 +335,27 @@ declare types of variables in a Python 3.6 compatible way as follows:
 
 There is currently no way to express the visibility of object attributes.
 
-Cython does not support the full range of annotations described by PEP-484.
-For example it does not currently understand features from the ``typing`` module
-such  as ``Optional[]`` or typed containers such as ``List[str]``. This is partly
-because some of these type hints are not relevant for the compilation to
+``typing`` Module
+^^^^^^^^^^^^^^^^^
+
+Support for the full range of annotations described by PEP-484 is not yet
+complete. Cython 3 currently understands the following features from the
+``typing`` module:
+
+* ``Optional[tp]``, which is interpreted as ``tp or None``;
+* typed containers such as ``List[str]``, which is interpreted as ``list``. The
+  hint that the elements are of type ``str`` is currently ignored;
+* ``Tuple[...]``, which is converted into a Cython C-tuple where possible
+  and a regular Python ``tuple`` otherwise.
+* ``ClassVar[...]``, which is understood in the context of
+  ``cdef class`` or ``@cython.cclass``.
+
+Some of the unsupported features are likely to remain
+unsupported since these type hints are not relevant for the compilation to
 efficient C code. In other cases, however, where the generated C code could
 benefit from these type hints but does not currently, help is welcome to
 improve the type analysis in Cython.
 
-
 Tips and Tricks
 ---------------
 

docs/src/userguide/extension_types.rst

@@ -1051,5 +1051,29 @@ generated containing declarations for its object struct and type object. By
 including the ``.h`` file in external C code that you write, that code can
 access the attributes of the extension type.
 
+Dataclass extension types
+=========================
 
-
+Cython supports extension types that behave like the dataclasses defined in
+the Python 3.7+ standard library. The main benefit of using a dataclass is
+that it can auto-generate simple ``__init__``, ``__repr__`` and comparison
+functions. The Cython implementation behaves as much like the Python
+standard library implementation as possible and therefore the documentation
+here only briefly outlines the differences - if you plan on using them
+then please read `the documentation for the standard library module
+<https://docs.python.org/3/library/dataclasses.html>`_.
+
+Dataclasses can be declared using the ``@cython.dataclasses.dataclass`` 
+decorator on a Cython extension type. ``@cython.dataclasses.dataclass``
+can only be applied to extension types (types marked ``cdef`` or created with the 
+``cython.cclass`` decorator) and not to regular classes. If
+you need to define special properties on a field then use ``cython.dataclasses.field``
+
+.. literalinclude:: ../../examples/userguide/extension_types/dataclass.pyx    
+
+You may use C-level types such as structs, pointers, or C++ classes.
+However, you may find these types are not compatible with the auto-generated
+special methods - for example if they cannot be converted from a Python
+type they cannot be passed to a constructor, and so you must use a 
+``default_factory`` to initialize them. Like with the Python implementation, you can also control
+which special functions an attribute is used in using ``field()``.

tests/errors/dataclass_e1.pyx

+# mode: error
+
+cimport cython
+
+@cython.dataclasses.dataclass(1, shouldnt_be_here=True, init=5, unsafe_hash=True)
+cdef class C:
+    a: list = []  # mutable
+    b: int = cython.dataclasses.field(default=5, default_factory=int)
+    c: int
+
+    def __hash__(self):
+        pass
+
+_ERRORS = """
+6:5: Arguments passed to cython.dataclasses.dataclass must be True or False
+6:5: Cannot overwrite attribute __hash__ in class C
+6:5: cython.dataclasses.dataclass() got an unexpected keyword argument 'shouldnt_be_here'
+6:5: cython.dataclasses.dataclass takes no positional arguments
+7:14: mutable default <class 'list'> for field a is not allowed: use default_factory
+8:37: cannot specify both default and default_factory
+9:4: non-default argument 'c' follows default argument in dataclass __init__
+"""

tests/errors/dataclass_e2.pyx

+# mode: error
+# tag: dataclass
+
+import dataclasses
+
+@dataclasses.dataclass
+cdef class C:
+    pass
+
+_ERRORS = """
+6:0: Cdef functions/classes cannot take arbitrary decorators.
+6:0: Use '@cython.dataclasses.dataclass' on cdef classes to create a dataclass
+"""

tests/errors/dataclass_e3.pyx

+# mode: compile
+# tag: dataclass, warnings
+
+cimport cython
+from dataclass import field
+
+@cython.dataclasses.dataclass
+cdef class E:
+    a: int = field()
+
+_WARNINGS="""
+9:18: Do you mean cython.dataclasses.field instead?
+"""

tests/errors/dataclass_e4.pyx

+# mode: error
+
+cimport cython
+
+@cython.dataclasses.dataclass
+cdef class C:
+    a: int = cython.dataclasses.field(unexpected=True)
+
+_ERRORS = """
+7:49: cython.dataclasses.field() got an unexpected keyword argument 'unexpected'
+"""

tests/run/cdef_class_dataclass.pyx

+# mode: run
+# tag: dataclass
+
+from cython cimport dataclasses
+from cython.dataclasses cimport dataclass, field
+try:
+    import typing
+    from typing import ClassVar
+    from dataclasses import InitVar
+    import dataclasses as py_dataclasses
+except ImportError:
+    pass
+import cython
+from libc.stdlib cimport malloc, free
+
+include "../testsupport/cythonarrayutil.pxi"
+
+cdef class NotADataclass:
+    cdef cython.int a
+    b: float
+
+    def __repr__(self):
+        return "NADC"
+
+    def __str__(self):
+        return "string of NotADataclass"  # should be called - repr is called!
+
+    def __eq__(self, other):
+        return type(self) == type(other)
+
+    def __hash__(self):
+        return 1
+
+@dataclass(unsafe_hash=True)
+cdef class BasicDataclass:
+    """
+    >>> sorted(list(BasicDataclass.__dataclass_fields__.keys()))
+    ['a', 'b', 'c', 'd']
+
+    # Check the field type attribute - this is currently a string since
+    # it's taken from the annotation, but if we drop PEP563 in future
+    # then it may change
+    >>> BasicDataclass.__dataclass_fields__["a"].type
+    'float'
+    >>> BasicDataclass.__dataclass_fields__["b"].type
+    'NotADataclass'
+    >>> BasicDataclass.__dataclass_fields__["c"].type
+    'object'
+    >>> BasicDataclass.__dataclass_fields__["d"].type
+    'list'
+
+    >>> inst1 = BasicDataclass() # doctest: +ELLIPSIS
+    Traceback (most recent call last):
+    TypeError: __init__() takes at least 1 ...
+    >>> inst1 = BasicDataclass(2.0)
+
+    # The error at-least demonstrates that the hash function has been created
+    >>> hash(inst1) # doctest: +ELLIPSIS
+    Traceback (most recent call last):
+    TypeError: ...unhashable...
+    >>> inst2 = BasicDataclass(2.0)
+    >>> inst1 == inst2
+    True
+    >>> inst2 = BasicDataclass(2.0, NotADataclass(), [])
+    >>> inst1 == inst2
+    False
+    >>> inst2 = BasicDataclass(2.0, NotADataclass(), [], [1,2,3])
+    >>> inst2
+    BasicDataclass(a=2.0, b=NADC, c=[], d=[1, 2, 3])
+    >>> inst2.c = "Some string"
+    >>> inst2
+    BasicDataclass(a=2.0, b=NADC, c='Some string', d=[1, 2, 3])
+    """
+    a: float
+    b: NotADataclass = field(default_factory=NotADataclass)
+    c: object = field(default=0)
+    d: list = dataclasses.field(default_factory=list)
+
+@dataclasses.dataclass
+cdef class InheritsFromDataclass(BasicDataclass):
+    """
+    >>> sorted(list(InheritsFromDataclass.__dataclass_fields__.keys()))
+    ['a', 'b', 'c', 'd', 'e']
+    >>> InheritsFromDataclass(a=1.0, e=5)
+    In __post_init__
+    InheritsFromDataclass(a=1.0, b=NADC, c=0, d=[], e=5)
+    """
+    e: cython.int = 0
+
+    def __post_init__(self):
+        print "In __post_init__"
+
+@cython.dataclasses.dataclass
+cdef class InheritsFromNotADataclass(NotADataclass):
+    """
+    >>> sorted(list(InheritsFromNotADataclass.__dataclass_fields__.keys()))
+    ['c']
+    >>> InheritsFromNotADataclass()
+    InheritsFromNotADataclass(c=1)
+    >>> InheritsFromNotADataclass(5)
+    InheritsFromNotADataclass(c=5)
+    """
+
+    c: cython.int = 1
+
+cdef struct S:
+    int a
+
+ctypedef S* S_ptr
+
+cdef S_ptr malloc_a_struct():
+    return <S_ptr>malloc(sizeof(S))
+
+@dataclass
+cdef class ContainsNonPyFields:
+    """
+    >>> ContainsNonPyFields()  # doctest: +ELLIPSIS
+    Traceback (most recent call last):
+    TypeError: __init__() takes ... 1 positional ...
+    >>> ContainsNonPyFields(mystruct={'a': 1 })  # doctest: +ELLIPSIS
+    ContainsNonPyFields(mystruct={'a': 1}, memview=<MemoryView of 'array' at ...>)
+    >>> ContainsNonPyFields(mystruct={'a': 1 }, memview=create_array((2,2), "c"))  # doctest: +ELLIPSIS
+    ContainsNonPyFields(mystruct={'a': 1}, memview=<MemoryView of 'array' at ...>)
+    >>> ContainsNonPyFields(mystruct={'a': 1 }, mystruct_ptr=0)
+    Traceback (most recent call last):
+    TypeError: __init__() got an unexpected keyword argument 'mystruct_ptr'
+    """
+    mystruct: S = cython.dataclasses.field(compare=False)
+    mystruct_ptr: S_ptr = field(init=False, repr=False, default_factory=malloc_a_struct)
+    memview: int[:, ::1] = field(default=create_array((3,1), "c"),  # mutable so not great but OK for a test
+                                 compare=False)
+
+    def __dealloc__(self):
+        free(self.mystruct_ptr)
+
+@dataclass
+cdef class InitClassVars:
+    """
+    Private (i.e. defined with "cdef") members deliberately don't appear
+    TODO - ideally c1 and c2 should also be listed here
+    >>> sorted(list(InitClassVars.__dataclass_fields__.keys()))
+    ['a', 'b1', 'b2']
+    >>> InitClassVars.c1
+    2.0
+    >>> InitClassVars.e1
+    []
+    >>> inst1 = InitClassVars()
+    In __post_init__
+    >>> inst1  # init vars don't appear in string
+    InitClassVars(a=0)
+    >>> inst2 = InitClassVars(b1=5, d2=100)
+    In __post_init__
+    >>> inst1 == inst2  # comparison ignores the initvar
+    True
+    """
+    a: cython.int = 0
+    b1: InitVar[double] = 1.0
+    b2: py_dataclasses.InitVar[double] = 1.0
+    c1: ClassVar[float] = 2.0
+    c2: typing.ClassVar[float] = 2.0
+    cdef InitVar[cython.int] d1
+    cdef py_dataclasses.InitVar[cython.int] d2
+    d1 = 5
+    d2 = 5
+    cdef ClassVar[list] e1
+    cdef typing.ClassVar[list] e2
+    e1 = []
+    e2 = []
+
+    def __post_init__(self, b1, b2, d1, d2):
+         # Check that the initvars haven't been assigned yet
+        assert self.b1==0, self.b1
+        assert self.b2==0, self.b2
+        assert self.d1==0, self.d1
+        assert self.d2==0, self.d2
+        self.b1 = b1
+        self.b2 = b2
+        self.d1 = d1
+        self.d2 = d2
+        print "In __post_init__"
+
+@dataclass
+cdef class TestVisibility:
+    """
+    >>> inst = TestVisibility()
+    >>> "a" in TestVisibility.__dataclass_fields__
+    False
+    >>> hasattr(inst, "a")
+    False
+    >>> "b" in TestVisibility.__dataclass_fields__
+    True
+    >>> hasattr(inst, "b")
+    True
+    >>> "c" in TestVisibility.__dataclass_fields__
+    True
+    >>> TestVisibility.__dataclass_fields__["c"].type
+    'double'
+    >>> hasattr(inst, "c")
+    True
+    """
+    cdef double a
+    a = 1.0
+    b: double = 2.0
+    cdef public double c
+    c = 3.0
+
+@dataclass(frozen=True)
+cdef class TestFrozen:
+    """
+    >>> inst = TestFrozen(a=5)
+    >>> inst.a
+    5.0
+    >>> inst.a = 2.  # doctest: +ELLIPSIS
+    Traceback (most recent call last):
+    AttributeError: attribute 'a' of '...TestFrozen' objects is not writable
+    """
+    a: double = 2.0
+
+import sys
+if sys.version_info >= (3, 7):
+    __doc__ = """
+    >>> from dataclasses import Field, is_dataclass, fields
+
+    # It uses the types from the standard library where available
+    >>> all(isinstance(v, Field) for v in BasicDataclass.__dataclass_fields__.values())
+    True
+
+    # check out Cython dataclasses are close enough to convince it
+    >>> is_dataclass(BasicDataclass)
+    True
+    >>> is_dataclass(BasicDataclass(1.5))
+    True
+    >>> is_dataclass(InheritsFromDataclass)
+    True
+    >>> is_dataclass(NotADataclass)
+    False
+    >>> is_dataclass(InheritsFromNotADataclass)
+    True
+    >>> [ f.name for f in fields(BasicDataclass)]
+    ['a', 'b', 'c', 'd']
+    >>> [ f.name for f in fields(InitClassVars)]
+    ['a']
+    """

tests/run/ext_type_none_arg.pyx

 
 cimport cython
+try:
+    import typing
+    from typing import Optional
+except ImportError:
+    pass  # Cython can still identify the use of "typing" even if the module doesn't exist
 
 
 ### extension types
@@ -79,6 +84,39 @@ def ext_not_none(MyExtType x not None):
     """
     return attr(x)
 
+def ext_annotations(x: MyExtType):
+    """
+    Behaves the same as "MyExtType x not None"
+    >>> ext_annotations(MyExtType())
+    123
+    >>> ext_annotations(None)
+    Traceback (most recent call last):
+    TypeError: Argument 'x' has incorrect type (expected ext_type_none_arg.MyExtType, got NoneType)
+    """
+    return attr(x)
+
+@cython.allow_none_for_extension_args(False)
+def ext_annotations_check_on(x: MyExtType):
+    """
+    >>> ext_annotations_check_on(MyExtType())
+    123
+    >>> ext_annotations_check_on(None)
+    Traceback (most recent call last):
+    TypeError: Argument 'x' has incorrect type (expected ext_type_none_arg.MyExtType, got NoneType)
+    """
+    return attr(x)
+
+def ext_optional(x: typing.Optional[MyExtType], y: Optional[MyExtType]):
+    """
+    Behaves the same as "or None"
+    >>> ext_optional(MyExtType(), MyExtType())
+    246
+    >>> ext_optional(MyExtType(), None)
+    444
+    >>> ext_optional(None, MyExtType())
+    444
+    """
+    return attr(x) + attr(y)
 
 ### builtin types (using list)
 

tests/run/pep526_variable_annotations.py

@@ -5,10 +5,13 @@
 import cython
 
 from typing import Dict, List, TypeVar, Optional, Generic, Tuple
+
 try:
+    import typing
+    from typing import Set as _SET_
     from typing import ClassVar
 except ImportError:
-    ClassVar = Optional  # fake it in Py3.5
+    pass  # this should allow Cython to interpret the directives even when the module doesn't exist
 
 
 var = 1  # type: annotation
@@ -51,6 +54,8 @@ class BasicStarship(object):
     'Picard'
     >>> bs.stats
     {}
+    >>> BasicStarship.stats
+    {}
     """
     captain: str = 'Picard'               # instance variable with default
     damage: cython.int                    # instance variable without default
@@ -117,11 +122,7 @@ def iter_declared_dict(d):
     >>> iter_declared_dict(d)
     7.0
 
-    >>> class D(object):
-    ...     def __getitem__(self, x): return 2
-    ...     def __iter__(self): return iter([1, 2, 3])
-    >>> iter_declared_dict(D())
-    6.0
+    # specialized "compiled" test in module-level __doc__
     """
     typed_dict : Dict[float, float] = d
     s = 0.0
@@ -140,11 +141,7 @@ def iter_declared_dict_arg(d : Dict[float, float]):
     >>> iter_declared_dict_arg(d)
     7.0
 
-    >>> class D(object):
-    ...     def __getitem__(self, x): return 2
-    ...     def __iter__(self): return iter([1, 2, 3])
-    >>> iter_declared_dict_arg(D())
-    6.0
+    # module level "compiled" test in __doc__ below
     """
     s = 0.0
     for key in d:
@@ -161,13 +158,58 @@ def literal_list_ptr():
     return a[3]
 
 
+def test_subscripted_types():
+    """
+    >>> test_subscripted_types()
+    dict object
+    list object
+    set object
+    """
+    a: typing.Dict[int, float] = {}
+    b: List[int] = []
+    c: _SET_[object] = set()
+
+    print(cython.typeof(a) + (" object" if not cython.compiled else ""))
+    print(cython.typeof(b) + (" object" if not cython.compiled else ""))
+    print(cython.typeof(c) + (" object" if not cython.compiled else ""))
+
+# because tuple is specifically special cased to go to ctuple where possible
+def test_tuple(a: typing.Tuple[int, float], b: typing.Tuple[int, ...],
+               c: Tuple[int, object]  # cannot be a ctuple
+               ):
+    """
+    >>> test_tuple((1, 1.0), (1, 1.0), (1, 1.0))
+    int
+    int
+    tuple object
+    tuple object
+    """
+    x: typing.Tuple[int, float] = (a[0], a[1])
+    y: Tuple[int, ...] = (1,2.)
+    z = a[0]  # should infer to int
+
+    print(cython.typeof(z))
+    print(cython.typeof(x[0]))
+    print(cython.typeof(y) + (" object" if not cython.compiled else ""))
+    print(cython.typeof(c) + (" object" if not cython.compiled else ""))
+
+
+if cython.compiled:
+    __doc__ = """
+    # passing non-dicts to variables declared as dict now fails
+    >>> class D(object):
+    ...     def __getitem__(self, x): return 2
+    ...     def __iter__(self): return iter([1, 2, 3])
+    >>> iter_declared_dict(D())  # doctest:+IGNORE_EXCEPTION_DETAIL
+    Traceback (most recent call last):
+    ...
+    TypeError: Expected dict, got D
+    >>> iter_declared_dict_arg(D())  # doctest:+IGNORE_EXCEPTION_DETAIL
+    Traceback (most recent call last):
+    ...
+    TypeError: Expected dict, got D
+    """
+
+
 _WARNINGS = """
-37:19: Unknown type declaration in annotation, ignoring
-38:12: Unknown type declaration in annotation, ignoring
-39:18: Unknown type declaration in annotation, ignoring
-73:11: Annotation ignored since class-level attributes must be Python objects. Were you trying to set up an instance attribute?
-73:19: Unknown type declaration in annotation, ignoring
-# FIXME: these are sort-of evaluated now, so the warning is misleading
-126:21: Unknown type declaration in annotation, ignoring
-137:35: Unknown type declaration in annotation, ignoring
 """

tests/run/pep526_variable_annotations_cy.pyx

+# mode: run
+
+import cython
+
+try:
+    import typing
+    from typing import List, Tuple
+    from typing import Set as _SET_
+except:
+    pass  # this should allow Cython to interpret the directives even when the module doesn't exist
+
+def test_subscripted_types():
+    """
+    >>> test_subscripted_types()
+    dict object
+    list object
+    set object
+    """
+    cdef typing.Dict[int, float] a = {}
+    cdef List[int] b = []
+    cdef _SET_[object] c = set()
+
+    print(cython.typeof(a))
+    print(cython.typeof(b))
+    print(cython.typeof(c))
+
+cdef class TestClassVar:
+    """
+    >>> TestClassVar.cls
+    5
+    >>> TestClassVar.regular  # doctest: +IGNORE_EXCEPTION_DETAIL
+    Traceback (most recent call last):
+        ...
+    AttributeError:
+    """
+    cdef int regular
+    cdef typing.ClassVar[int] cls
+    cls = 5
+
+# because tuple is specifically special cased to go to ctuple where possible
+def test_tuple(typing.Tuple[int, float] a,  typing.Tuple[int, ...] b,
+               Tuple[int, object] c  # cannot be a ctuple
+               ):
+    """
+    >>> test_tuple((1, 1.0), (1, 1.0), (1, 1.0))
+    int
+    int
+    tuple object
+    tuple object
+    """
+    cdef typing.Tuple[int, float] x = (a[0], a[1])
+    cdef Tuple[int, ...] y = (1,2.)
+    z = a[0]  # should infer to int
+
+    print(cython.typeof(z))
+    print(cython.typeof(x[0]))
+    print(cython.typeof(y))
+    print(cython.typeof(c))

tests/run/pure_cdef_class_dataclass.py

+# mode: run
+# tag: dataclass, pure3.7
+
+from __future__ import print_function
+
+import cython
+
+@cython.dataclasses.dataclass(order=True, unsafe_hash=True)
+@cython.cclass
+class MyDataclass:
+    """
+    >>> sorted(list(MyDataclass.__dataclass_fields__.keys()))
+    ['a', 'self']
+    >>> inst1 = MyDataclass(2.0, ['a', 'b'])
+    >>> print(inst1)
+    MyDataclass(a=2.0, self=['a', 'b'])
+    >>> inst2 = MyDataclass()
+    >>> print(inst2)
+    MyDataclass(a=1, self=[])
+    >>> inst1 == inst2
+    False
+    >>> inst1 > inst2
+    True
+    >>> inst2 == MyDataclass()
+    True
+    >>> hash(inst1) != id(inst1)
+    True
+    """
+
+    a: int = 1
+    self: list = cython.dataclasses.field(default_factory=list, hash=False)  # test that arguments of init don't conflict

tests/run/typing_module.py

+# mode: run
+# tag: pure3.6
+
+from __future__ import print_function
+
+import cython
+
+try:
+    import typing
+    from typing import List
+    from typing import Set as _SET_
+except ImportError:
+    pass  # this should allow Cython to interpret the directives even when the module doesn't exist
+
+
+def test_subscripted_types():
+    """
+    >>> test_subscripted_types()
+    dict object
+    list object
+    set object
+    """
+    a: typing.Dict[int, float] = {}
+    b: List[int] = []
+    c: _SET_[object] = set()
+
+    print(cython.typeof(a) + (" object" if not cython.compiled else ""))
+    print(cython.typeof(b) + (" object" if not cython.compiled else ""))
+    print(cython.typeof(c) + (" object" if not cython.compiled else ""))
+
+@cython.cclass
+class TestClassVar:
+    """
+    >>> TestClassVar.cls
+    5
+    >>> TestClassVar.regular  # doctest: +IGNORE_EXCEPTION_DETAIL
+    Traceback (most recent call last):
+        ...
+    AttributeError:
+    """
+    regular: int
+    cls: typing.ClassVar[int] = 5  # this is a little redundant really because the assignment ensures it

tests/run/typing_module_cy.pyx

+# mode: run
+
+import cython
+
+try:
+    import typing
+    from typing import List
+    from typing import Set as _SET_
+except ImportError:
+    pass  # this should allow Cython to interpret the directives even when the module doesn't exist
+
+def test_subscripted_types():
+    """
+    >>> test_subscripted_types()
+    dict object
+    list object
+    set object
+    """
+    cdef typing.Dict[int, float] a = {}
+    cdef List[int] b = []
+    cdef _SET_[object] c = set()
+
+    print(cython.typeof(a))
+    print(cython.typeof(b))
+    print(cython.typeof(c))
+
+cdef class TestClassVar:
+    """
+    >>> TestClassVar.cls
+    5
+    >>> TestClassVar.regular  # doctest: +IGNORE_EXCEPTION_DETAIL
+    Traceback (most recent call last):
+        ...
+    AttributeError:
+    """
+    cdef int regular
+    cdef typing.ClassVar[int] cls
+    cls = 5