reduce code size of parser and speed it up a little by statically switching to...

reduce code size of parser and speed it up a little by statically switching to unicode in more places

Cython/Compiler/Lexicon.py

+# cython: language_level=3
 #
 #   Cython Scanner - Lexical Definitions
 #

Cython/Compiler/Parsing.pxd

@@ -3,6 +3,8 @@
 cimport cython
 from Cython.Compiler.Scanning cimport PyrexScanner
 
+ctypedef object (*p_sub_expr_func)(object)
+
 # entry points
 
 cpdef p_module(PyrexScanner s, pxd, full_module_name)
@@ -13,8 +15,8 @@ cpdef p_code(PyrexScanner s, level= *)
 cdef p_ident(PyrexScanner s, message =*)
 cdef p_ident_list(PyrexScanner s)
 
-cdef p_binop_operator(PyrexScanner s)
-cdef p_binop_expr(PyrexScanner s, ops, p_sub_expr)
+cdef tuple p_binop_operator(PyrexScanner s)
+cdef p_binop_expr(PyrexScanner s, ops, p_sub_expr_func p_sub_expr)
 cpdef p_lambdef(PyrexScanner s, bint allow_conditional=*)
 cdef p_lambdef_nocond(PyrexScanner s)
 cdef p_test(PyrexScanner s)
@@ -29,12 +31,13 @@ cdef p_starred_expr(PyrexScanner s)
 cdef p_cascaded_cmp(PyrexScanner s)
 cdef p_cmp_op(PyrexScanner s)
 cdef p_bit_expr(PyrexScanner s)
-cpdef p_xor_expr(PyrexScanner s)
-cpdef p_and_expr(PyrexScanner s)
-cpdef p_shift_expr(PyrexScanner s)
-cpdef p_arith_expr(PyrexScanner s)
-cpdef p_term(PyrexScanner s)
-cpdef p_factor(PyrexScanner s)
+cdef p_xor_expr(s)
+cdef p_and_expr(s)
+cdef p_shift_expr(s)
+cdef p_arith_expr(s)
+cdef p_term(s)
+cdef p_factor(s)
+cdef _p_factor(PyrexScanner s)
 cdef p_typecast(PyrexScanner s)
 cdef p_sizeof(PyrexScanner s)
 cdef p_yield_expression(PyrexScanner s)

Cython/Compiler/Parsing.py

-# cython: auto_cpdef=True, infer_types=True
+# cython: auto_cpdef=True, infer_types=True, language_level=3
 #
 #   Pyrex Parser
 #
@@ -269,6 +269,10 @@ def p_term(s):
 #factor: ('+'|'-'|'~'|'&'|typecast|sizeof) factor | power
 
 def p_factor(s):
+    # little indirection for C-ification purposes
+    return _p_factor(s)
+
+def _p_factor(s):
     sy = s.sy
     if sy in ('+', '-', '~'):
         op = s.sy
@@ -1701,7 +1705,6 @@ def p_statement(s, ctx, first_statement = 0):
         return p_IF_statement(s, ctx)
     elif s.sy == 'DECORATOR':
         if ctx.level not in ('module', 'class', 'c_class', 'function', 'property', 'module_pxd', 'c_class_pxd'):
-            print ctx.level
             s.error('decorator not allowed here')
         s.level = ctx.level
         decorators = p_decorators(s)

Cython/Compiler/Scanning.py

+# cython: infer_types=True, language_level=3
 #
 #   Cython Scanner
 #
@@ -7,7 +8,7 @@ import os
 import platform
 
 import cython
-cython.declare(EncodedString=object, string_prefixes=object, raw_prefixes=object, IDENT=object,
+cython.declare(EncodedString=object, string_prefixes=object, raw_prefixes=object, IDENT=unicode,
                print_function=object)
 
 from Cython import Plex, Utils

setup.py

@@ -88,6 +88,7 @@ def compile_cython_modules(profile=False, compile_more=False, cython_with_refnan
     source_root = os.path.abspath(os.path.dirname(__file__))
     compiled_modules = ["Cython.Plex.Scanners",
                         "Cython.Plex.Actions",
+                        "Cython.Compiler.Lexicon",
                         "Cython.Compiler.Scanning",
                         "Cython.Compiler.Parsing",
                         "Cython.Compiler.Visitor",