minor source modernisations and cleanups in Scanning.py

Cython/Compiler/Scanning.pxd

@@ -4,6 +4,9 @@ import cython
 
 from ..Plex.Scanners cimport Scanner
 
+cdef get_lexicon()
+cdef initial_compile_time_env()
+
 cdef class Method:
     cdef object name
     cdef object __name__

Cython/Compiler/Scanning.py

@@ -5,13 +5,15 @@
 
 from __future__ import absolute_import
 
+import cython
+cython.declare(EncodedString=object, make_lexicon=object, lexicon=object,
+               any_string_prefix=unicode, IDENT=unicode,
+               print_function=object, error=object, warning=object,
+               os=object, platform=object)
+
 import os
 import platform
 
-import cython
-cython.declare(EncodedString=object, any_string_prefix=unicode, IDENT=unicode,
-               print_function=object, error=object, warning=object)
-
 from .. import Utils
 from ..Plex.Scanners import Scanner
 from ..Plex.Errors import UnrecognizedInput
@@ -28,12 +30,14 @@ scanner_dump_file = None
 
 lexicon = None
 
+
 def get_lexicon():
     global lexicon
     if not lexicon:
         lexicon = make_lexicon()
     return lexicon
 
+
 #------------------------------------------------------------------
 
 py_reserved_words = [
@@ -49,6 +53,7 @@ pyx_reserved_words = py_reserved_words + [
     "cimport", "DEF", "IF", "ELIF", "ELSE"
 ]
 
+
 class Method(object):
 
     def __init__(self, name):
@@ -58,6 +63,7 @@ class Method(object):
     def __call__(self, stream, text):
         return getattr(stream, self.name)(text)
 
+
 #------------------------------------------------------------------
 
 class CompileTimeScope(object):
@@ -88,6 +94,7 @@ class CompileTimeScope(object):
             else:
                 raise
 
+
 def initial_compile_time_env():
     benv = CompileTimeScope()
     names = ('UNAME_SYSNAME', 'UNAME_NODENAME', 'UNAME_RELEASE',
@@ -116,6 +123,7 @@ def initial_compile_time_env():
     denv = CompileTimeScope(benv)
     return denv
 
+
 #------------------------------------------------------------------
 
 class SourceDescriptor(object):
@@ -166,6 +174,7 @@ class SourceDescriptor(object):
         except AttributeError:
             return False
 
+
 class FileSourceDescriptor(SourceDescriptor):
     """
     Represents a code source. A code source is a more generic abstraction
@@ -235,6 +244,7 @@ class FileSourceDescriptor(SourceDescriptor):
     def __repr__(self):
         return "<FileSourceDescriptor:%s>" % self.filename
 
+
 class StringSourceDescriptor(SourceDescriptor):
     """
     Instances of this class can be used instead of a filenames if the
@@ -275,6 +285,7 @@ class StringSourceDescriptor(SourceDescriptor):
     def __repr__(self):
         return "<StringSourceDescriptor:%s>" % self.name
 
+
 #------------------------------------------------------------------
 
 class PyrexScanner(Scanner):
@@ -284,8 +295,8 @@ class PyrexScanner(Scanner):
     #  compile_time_eval  boolean  In a true conditional compilation context
     #  compile_time_expr  boolean  In a compile-time expression context
 
-    def __init__(self, file, filename, parent_scanner = None,
-                 scope = None, context = None, source_encoding=None, parse_comments=True, initial_pos=None):
+    def __init__(self, file, filename, parent_scanner=None,
+                 scope=None, context=None, source_encoding=None, parse_comments=True, initial_pos=None):
         Scanner.__init__(self, get_lexicon(), file, filename, initial_pos)
         if parent_scanner:
             self.context = parent_scanner.context
@@ -299,8 +310,8 @@ class PyrexScanner(Scanner):
             self.compile_time_env = initial_compile_time_env()
             self.compile_time_eval = 1
             self.compile_time_expr = 0
-            if hasattr(context.options, 'compile_time_env') and \
-               context.options.compile_time_env is not None:
+            if (hasattr(context.options, 'compile_time_env') and
+                    context.options.compile_time_env is not None):
                 self.compile_time_env.update(context.options.compile_time_env)
         self.parse_comments = parse_comments
         self.source_encoding = source_encoding
@@ -326,11 +337,11 @@ class PyrexScanner(Scanner):
         return self.indentation_stack[-1]
 
     def open_bracket_action(self, text):
-        self.bracket_nesting_level = self.bracket_nesting_level + 1
+        self.bracket_nesting_level += 1
         return text
 
     def close_bracket_action(self, text):
-        self.bracket_nesting_level = self.bracket_nesting_level - 1
+        self.bracket_nesting_level -= 1
         return text
 
     def newline_action(self, text):
@@ -406,6 +417,7 @@ class PyrexScanner(Scanner):
             sy, systring = self.read()
         except UnrecognizedInput:
             self.error("Unrecognized character")
+            return  # just a marker, error() always raises
         if sy == IDENT:
             if systring in self.keywords:
                 if systring == u'print' and print_function in self.context.future_directives:
@@ -445,21 +457,21 @@ class PyrexScanner(Scanner):
         # This method should be added to Plex
         self.queue.insert(0, (token, value))
 
-    def error(self, message, pos = None, fatal = True):
+    def error(self, message, pos=None, fatal=True):
         if pos is None:
             pos = self.position()
         if self.sy == 'INDENT':
-            err = error(pos, "Possible inconsistent indentation")
+            error(pos, "Possible inconsistent indentation")
         err = error(pos, message)
         if fatal: raise err
 
-    def expect(self, what, message = None):
+    def expect(self, what, message=None):
         if self.sy == what:
             self.next()
         else:
             self.expected(what, message)
 
-    def expect_keyword(self, what, message = None):
+    def expect_keyword(self, what, message=None):
         if self.sy == IDENT and self.systring == what:
             self.next()
         else:
@@ -476,12 +488,10 @@ class PyrexScanner(Scanner):
             self.error("Expected '%s', found '%s'" % (what, found))
 
     def expect_indent(self):
-        self.expect('INDENT',
-            "Expected an increase in indentation level")
+        self.expect('INDENT', "Expected an increase in indentation level")
 
     def expect_dedent(self):
-        self.expect('DEDENT',
-            "Expected a decrease in indentation level")
+        self.expect('DEDENT', "Expected a decrease in indentation level")
 
     def expect_newline(self, message="Expected a newline", ignore_semicolon=False):
         # Expect either a newline or end of file