Merge pull request #2835 from jbrockmendel/refopts

REF: Move CompilationOptions and default_options to Options

Cython/Build/Dependencies.py

@@ -42,7 +42,8 @@ except:
 from .. import Utils
 from ..Utils import (cached_function, cached_method, path_exists,
     safe_makedirs, copy_file_to_dir_if_newer, is_package_dir, replace_suffix)
-from ..Compiler.Main import Context, CompilationOptions, default_options
+from ..Compiler.Main import Context
+from ..Compiler.Options import CompilationOptions, default_options
 
 join_path = cached_function(os.path.join)
 copy_once_if_newer = cached_function(copy_file_to_dir_if_newer)
@@ -955,7 +956,7 @@ def cythonize(module_list, exclude=None, nthreads=0, aliases=None, quiet=False,
 
     c_options = CompilationOptions(**options)
     cpp_options = CompilationOptions(**options); cpp_options.cplus = True
-    ctx = c_options.create_context()
+    ctx = Context.from_options(c_options)
     options = c_options
     module_list, module_metadata = create_extension_list(
         module_list,

Cython/Build/Inline.py

@@ -8,7 +8,8 @@ from distutils.core import Distribution, Extension
 from distutils.command.build_ext import build_ext
 
 import Cython
-from ..Compiler.Main import Context, CompilationOptions, default_options
+from ..Compiler.Main import Context
+from ..Compiler.Options import CompilationOptions, default_options
 
 from ..Compiler.ParseTreeTransforms import (CythonTransform,
         SkipDeclarations, AnalyseDeclarationsTransform, EnvTransform)

Cython/Compiler/CmdLine.py

@@ -65,8 +65,6 @@ def bad_usage():
 
 
 def parse_command_line(args):
-    from .Main import CompilationOptions, default_options
-
     pending_arg = []
 
     def pop_arg():
@@ -94,7 +92,7 @@ def parse_command_line(args):
         else:
             return pop_arg()
 
-    options = CompilationOptions(default_options)
+    options = Options.CompilationOptions(Options.default_options)
     sources = []
     while args:
         if args[0].startswith("-"):

Cython/Compiler/Main.py

@@ -30,6 +30,8 @@ from .Errors import PyrexError, CompileError, error, warning
 from .Symtab import ModuleScope
 from .. import Utils
 from . import Options
+from .Options import CompilationOptions, default_options
+from .CmdLine import parse_command_line
 
 from . import Version  # legacy import needed by old PyTables versions
 version = Version.version  # legacy attribute - use "Cython.__version__" instead
@@ -82,6 +84,11 @@ class Context(object):
 
         self.gdb_debug_outputwriter = None
 
+    @classmethod
+    def from_options(cls, options):
+        return cls(options.include_path, options.compiler_directives,
+                   options.cplus, options.language_level, options=options)
+
     def set_language_level(self, level):
         from .Future import print_function, unicode_literals, absolute_import, division, generator_stop
         future_directives = set()
@@ -449,7 +456,7 @@ def run_pipeline(source, options, full_module_name=None, context=None):
     source_ext = os.path.splitext(source)[1]
     options.configure_language_defaults(source_ext[1:]) # py/pyx
     if context is None:
-        context = options.create_context()
+        context = Context.from_options(options)
 
     # Set up source object
     cwd = os.getcwd()
@@ -507,146 +514,6 @@ class CompilationSource(object):
         self.cwd = cwd
 
 
-class CompilationOptions(object):
-    r"""
-    See default_options at the end of this module for a list of all possible
-    options and CmdLine.usage and CmdLine.parse_command_line() for their
-    meaning.
-    """
-    def __init__(self, defaults=None, **kw):
-        self.include_path = []
-        if defaults:
-            if isinstance(defaults, CompilationOptions):
-                defaults = defaults.__dict__
-        else:
-            defaults = default_options
-
-        options = dict(defaults)
-        options.update(kw)
-
-        # let's assume 'default_options' contains a value for most known compiler options
-        # and validate against them
-        unknown_options = set(options) - set(default_options)
-        # ignore valid options that are not in the defaults
-        unknown_options.difference_update(['include_path'])
-        if unknown_options:
-            message = "got unknown compilation option%s, please remove: %s" % (
-                's' if len(unknown_options) > 1 else '',
-                ', '.join(unknown_options))
-            raise ValueError(message)
-
-        directive_defaults = Options.get_directive_defaults()
-        directives = dict(options['compiler_directives'])  # copy mutable field
-        # check for invalid directives
-        unknown_directives = set(directives) - set(directive_defaults)
-        if unknown_directives:
-            message = "got unknown compiler directive%s: %s" % (
-                's' if len(unknown_directives) > 1 else '',
-                ', '.join(unknown_directives))
-            raise ValueError(message)
-        options['compiler_directives'] = directives
-        if directives.get('np_pythran', False) and not options['cplus']:
-            import warnings
-            warnings.warn("C++ mode forced when in Pythran mode!")
-            options['cplus'] = True
-        if 'language_level' in directives and 'language_level' not in kw:
-            options['language_level'] = directives['language_level']
-        elif not options.get('language_level'):
-            options['language_level'] = directive_defaults.get('language_level')
-        if 'formal_grammar' in directives and 'formal_grammar' not in kw:
-            options['formal_grammar'] = directives['formal_grammar']
-        if options['cache'] is True:
-            options['cache'] = os.path.join(Utils.get_cython_cache_dir(), 'compiler')
-
-        self.__dict__.update(options)
-
-    def configure_language_defaults(self, source_extension):
-        if source_extension == 'py':
-            if self.compiler_directives.get('binding') is None:
-                self.compiler_directives['binding'] = True
-
-    def create_context(self):
-        return Context(self.include_path, self.compiler_directives,
-                       self.cplus, self.language_level, options=self)
-
-    def get_fingerprint(self):
-        r"""
-        Return a string that contains all the options that are relevant for cache invalidation.
-        """
-        # Collect only the data that can affect the generated file(s).
-        data = {}
-
-        for key, value in self.__dict__.items():
-            if key in ['show_version', 'errors_to_stderr', 'verbose', 'quiet']:
-                # verbosity flags have no influence on the compilation result
-                continue
-            elif key in ['output_file', 'output_dir']:
-                # ignore the exact name of the output file
-                continue
-            elif key in ['timestamps']:
-                # the cache cares about the content of files, not about the timestamps of sources
-                continue
-            elif key in ['cache']:
-                # hopefully caching has no influence on the compilation result
-                continue
-            elif key in ['compiler_directives']:
-                # directives passed on to the C compiler do not influence the generated C code
-                continue
-            elif key in ['include_path']:
-                # this path changes which headers are tracked as dependencies,
-                # it has no influence on the generated C code
-                continue
-            elif key in ['working_path']:
-                # this path changes where modules and pxd files are found;
-                # their content is part of the fingerprint anyway, their
-                # absolute path does not matter
-                continue
-            elif key in ['create_extension']:
-                # create_extension() has already mangled the options, e.g.,
-                # embedded_metadata, when the fingerprint is computed so we
-                # ignore it here.
-                continue
-            elif key in ['build_dir']:
-                # the (temporary) directory where we collect dependencies
-                # has no influence on the C output
-                continue
-            elif key in ['use_listing_file', 'generate_pxi', 'annotate', 'annotate_coverage_xml']:
-                # all output files are contained in the cache so the types of
-                # files generated must be part of the fingerprint
-                data[key] = value
-            elif key in ['formal_grammar', 'evaluate_tree_assertions']:
-                # these bits can change whether compilation to C passes/fails
-                data[key] = value
-            elif key in ['embedded_metadata', 'emit_linenums', 'c_line_in_traceback', 'gdb_debug', 'relative_path_in_code_position_comments']:
-                # the generated code contains additional bits when these are set
-                data[key] = value
-            elif key in ['cplus', 'language_level', 'compile_time_env', 'np_pythran']:
-                # assorted bits that, e.g., influence the parser
-                data[key] = value
-            elif key == ['capi_reexport_cincludes']:
-                if self.capi_reexport_cincludes:
-                    # our caching implementation does not yet include fingerprints of all the header files
-                    raise NotImplementedError('capi_reexport_cincludes is not compatible with Cython caching')
-            elif key == ['common_utility_include_dir']:
-                if self.common_utility_include_dir:
-                    raise NotImplementedError('common_utility_include_dir is not compatible with Cython caching yet')
-            else:
-                # any unexpected option should go into the fingerprint; it's better
-                # to recompile than to return incorrect results from the cache.
-                data[key] = value
-
-        def to_fingerprint(item):
-            r"""
-            Recursively turn item into a string, turning dicts into lists with
-            deterministic ordering.
-            """
-            if isinstance(item, dict):
-                item = sorted([(repr(key), to_fingerprint(value)) for key, value in item.items()])
-            return repr(item)
-
-        return to_fingerprint(data)
-
-
 class CompilationResult(object):
     """
     Results from the Cython compiler:
@@ -709,7 +576,7 @@ def compile_multiple(sources, options):
     if these are specified in the options.
     """
     # run_pipeline creates the context
-    # context = options.create_context()
+    # context = Context.from_options(options)
     sources = [os.path.abspath(source) for source in sources]
     processed = set()
     results = CompilationResultSet()
@@ -720,7 +587,7 @@ def compile_multiple(sources, options):
     for source in sources:
         if source not in processed:
             if context is None:
-                context = options.create_context()
+                context = Context.from_options(options)
             output_filename = get_output_filename(source, cwd, options)
             out_of_date = context.c_file_out_of_date(source, output_filename)
             if (not timestamps) or out_of_date:
@@ -767,7 +634,6 @@ def main(command_line = 0):
     args = sys.argv[1:]
     any_failures = 0
     if command_line:
-        from .CmdLine import parse_command_line
         options, sources = parse_command_line(args)
     else:
         options = CompilationOptions(default_options)
@@ -786,42 +652,3 @@ def main(command_line = 0):
         any_failures = 1
     if any_failures:
         sys.exit(1)
-
-
-# ------------------------------------------------------------------------
-#
-#  Set the default options depending on the platform
-#
-# ------------------------------------------------------------------------
-
-default_options = dict(
-    show_version = 0,
-    use_listing_file = 0,
-    errors_to_stderr = 1,
-    cplus = 0,
-    output_file = None,
-    annotate = None,
-    annotate_coverage_xml = None,
-    generate_pxi = 0,
-    capi_reexport_cincludes = 0,
-    working_path = "",
-    timestamps = None,
-    verbose = 0,
-    quiet = 0,
-    compiler_directives = {},
-    embedded_metadata = {},
-    evaluate_tree_assertions = False,
-    emit_linenums = False,
-    relative_path_in_code_position_comments = True,
-    c_line_in_traceback = True,
-    language_level = None,  # warn but default to 2
-    formal_grammar = False,
-    gdb_debug = False,
-    compile_time_env = None,
-    common_utility_include_dir = None,
-    output_dir=None,
-    build_dir=None,
-    cache=None,
-    create_extension=None,
-    np_pythran=False
-)

Cython/Compiler/Options.py

@@ -4,6 +4,10 @@
 
 from __future__ import absolute_import
 
+import os
+
+from Cython import Utils
+
 
 class ShouldBeFromDirective(object):
 
@@ -543,3 +547,184 @@ def parse_compile_time_env(s, current_settings=None):
         name, value = [s.strip() for s in item.split('=', 1)]
         result[name] = parse_variable_value(value)
     return result
+
+
+# ------------------------------------------------------------------------
+# CompilationOptions are constructed from user input and are the `option`
+#  object passed throughout the compilation pipeline.
+
+class CompilationOptions(object):
+    r"""
+    See default_options at the end of this module for a list of all possible
+    options and CmdLine.usage and CmdLine.parse_command_line() for their
+    meaning.
+    """
+    def __init__(self, defaults=None, **kw):
+        self.include_path = []
+        if defaults:
+            if isinstance(defaults, CompilationOptions):
+                defaults = defaults.__dict__
+        else:
+            defaults = default_options
+
+        options = dict(defaults)
+        options.update(kw)
+
+        # let's assume 'default_options' contains a value for most known compiler options
+        # and validate against them
+        unknown_options = set(options) - set(default_options)
+        # ignore valid options that are not in the defaults
+        unknown_options.difference_update(['include_path'])
+        if unknown_options:
+            message = "got unknown compilation option%s, please remove: %s" % (
+                's' if len(unknown_options) > 1 else '',
+                ', '.join(unknown_options))
+            raise ValueError(message)
+
+        directive_defaults = get_directive_defaults()
+        directives = dict(options['compiler_directives'])  # copy mutable field
+        # check for invalid directives
+        unknown_directives = set(directives) - set(directive_defaults)
+        if unknown_directives:
+            message = "got unknown compiler directive%s: %s" % (
+                's' if len(unknown_directives) > 1 else '',
+                ', '.join(unknown_directives))
+            raise ValueError(message)
+        options['compiler_directives'] = directives
+        if directives.get('np_pythran', False) and not options['cplus']:
+            import warnings
+            warnings.warn("C++ mode forced when in Pythran mode!")
+            options['cplus'] = True
+        if 'language_level' in directives and 'language_level' not in kw:
+            options['language_level'] = directives['language_level']
+        elif not options.get('language_level'):
+            options['language_level'] = directive_defaults.get('language_level')
+        if 'formal_grammar' in directives and 'formal_grammar' not in kw:
+            options['formal_grammar'] = directives['formal_grammar']
+        if options['cache'] is True:
+            options['cache'] = os.path.join(Utils.get_cython_cache_dir(), 'compiler')
+
+        self.__dict__.update(options)
+
+    def configure_language_defaults(self, source_extension):
+        if source_extension == 'py':
+            if self.compiler_directives.get('binding') is None:
+                self.compiler_directives['binding'] = True
+
+    def get_fingerprint(self):
+        r"""
+        Return a string that contains all the options that are relevant for cache invalidation.
+        """
+        # Collect only the data that can affect the generated file(s).
+        data = {}
+
+        for key, value in self.__dict__.items():
+            if key in ['show_version', 'errors_to_stderr', 'verbose', 'quiet']:
+                # verbosity flags have no influence on the compilation result
+                continue
+            elif key in ['output_file', 'output_dir']:
+                # ignore the exact name of the output file
+                continue
+            elif key in ['timestamps']:
+                # the cache cares about the content of files, not about the timestamps of sources
+                continue
+            elif key in ['cache']:
+                # hopefully caching has no influence on the compilation result
+                continue
+            elif key in ['compiler_directives']:
+                # directives passed on to the C compiler do not influence the generated C code
+                continue
+            elif key in ['include_path']:
+                # this path changes which headers are tracked as dependencies,
+                # it has no influence on the generated C code
+                continue
+            elif key in ['working_path']:
+                # this path changes where modules and pxd files are found;
+                # their content is part of the fingerprint anyway, their
+                # absolute path does not matter
+                continue
+            elif key in ['create_extension']:
+                # create_extension() has already mangled the options, e.g.,
+                # embedded_metadata, when the fingerprint is computed so we
+                # ignore it here.
+                continue
+            elif key in ['build_dir']:
+                # the (temporary) directory where we collect dependencies
+                # has no influence on the C output
+                continue
+            elif key in ['use_listing_file', 'generate_pxi', 'annotate', 'annotate_coverage_xml']:
+                # all output files are contained in the cache so the types of
+                # files generated must be part of the fingerprint
+                data[key] = value
+            elif key in ['formal_grammar', 'evaluate_tree_assertions']:
+                # these bits can change whether compilation to C passes/fails
+                data[key] = value
+            elif key in ['embedded_metadata', 'emit_linenums',
+                         'c_line_in_traceback', 'gdb_debug',
+                         'relative_path_in_code_position_comments']:
+                # the generated code contains additional bits when these are set
+                data[key] = value
+            elif key in ['cplus', 'language_level', 'compile_time_env', 'np_pythran']:
+                # assorted bits that, e.g., influence the parser
+                data[key] = value
+            elif key == ['capi_reexport_cincludes']:
+                if self.capi_reexport_cincludes:
+                    # our caching implementation does not yet include fingerprints of all the header files
+                    raise NotImplementedError('capi_reexport_cincludes is not compatible with Cython caching')
+            elif key == ['common_utility_include_dir']:
+                if self.common_utility_include_dir:
+                    raise NotImplementedError('common_utility_include_dir is not compatible with Cython caching yet')
+            else:
+                # any unexpected option should go into the fingerprint; it's better
+                # to recompile than to return incorrect results from the cache.
+                data[key] = value
+
+        def to_fingerprint(item):
+            r"""
+            Recursively turn item into a string, turning dicts into lists with
+            deterministic ordering.
+            """
+            if isinstance(item, dict):
+                item = sorted([(repr(key), to_fingerprint(value)) for key, value in item.items()])
+            return repr(item)
+
+        return to_fingerprint(data)
+
+
+# ------------------------------------------------------------------------
+#
+#  Set the default options depending on the platform
+#
+# ------------------------------------------------------------------------
+
+default_options = dict(
+    show_version=0,
+    use_listing_file=0,
+    errors_to_stderr=1,
+    cplus=0,
+    output_file=None,
+    annotate=None,
+    annotate_coverage_xml=None,
+    generate_pxi=0,
+    capi_reexport_cincludes=0,
+    working_path="",
+    timestamps=None,
+    verbose=0,
+    quiet=0,
+    compiler_directives={},
+    embedded_metadata={},
+    evaluate_tree_assertions=False,
+    emit_linenums=False,
+    relative_path_in_code_position_comments=True,
+    c_line_in_traceback=True,
+    language_level=None,  # warn but default to 2
+    formal_grammar=False,
+    gdb_debug=False,
+    compile_time_env=None,
+    common_utility_include_dir=None,
+    output_dir=None,
+    build_dir=None,
+    cache=None,
+    create_extension=None,
+    np_pythran=False
+)

Cython/Compiler/Tests/TestParseTreeTransforms.py

@@ -3,7 +3,7 @@ import os
 from Cython.TestUtils import TransformTest
 from Cython.Compiler.ParseTreeTransforms import *
 from Cython.Compiler.Nodes import *
-from Cython.Compiler import Main, Symtab
+from Cython.Compiler import Main, Symtab, Options
 
 
 class TestNormalizeTree(TransformTest):
@@ -177,8 +177,8 @@ class TestInterpretCompilerDirectives(TransformTest):
     def setUp(self):
         super(TestInterpretCompilerDirectives, self).setUp()
 
-        compilation_options = Main.CompilationOptions(Main.default_options)
-        ctx = compilation_options.create_context()
+        compilation_options = Options.CompilationOptions(Options.default_options)
+        ctx = Main.Context.from_options(compilation_options)
 
         transform = InterpretCompilerDirectives(ctx, ctx.compiler_directives)
         transform.module_scope = Symtab.ModuleScope('__main__', None, ctx)

Cython/Tests/TestJediTyper.py

@@ -11,7 +11,7 @@ from contextlib import contextmanager
 from tempfile import NamedTemporaryFile
 
 from Cython.Compiler.ParseTreeTransforms import NormalizeTree, InterpretCompilerDirectives
-from Cython.Compiler import Main, Symtab, Visitor
+from Cython.Compiler import Main, Symtab, Visitor, Options
 from Cython.TestUtils import TransformTest
 
 TOOLS_DIR = os.path.abspath(os.path.join(os.path.dirname(__file__), '..', '..', 'Tools'))
@@ -210,8 +210,8 @@ class TestTypeInjection(TestJediTyper):
     """
     def setUp(self):
         super(TestTypeInjection, self).setUp()
-        compilation_options = Main.CompilationOptions(Main.default_options)
-        ctx = compilation_options.create_context()
+        compilation_options = Options.CompilationOptions(Options.default_options)
+        ctx = Main.Context.from_options(compilation_options)
         transform = InterpretCompilerDirectives(ctx, ctx.compiler_directives)
         transform.module_scope = Symtab.ModuleScope('__main__', None, ctx)
         self.declarations_finder = DeclarationsFinder()

runtests.py

@@ -1029,9 +1029,9 @@ class CythonCompileTestCase(unittest.TestCase):
         try:
             CompilationOptions
         except NameError:
-            from Cython.Compiler.Main import CompilationOptions
+            from Cython.Compiler.Options import CompilationOptions
             from Cython.Compiler.Main import compile as cython_compile
-            from Cython.Compiler.Main import default_options
+            from Cython.Compiler.Options import default_options
         common_utility_include_dir = self.common_utility_dir
 
         options = CompilationOptions(
@@ -2232,7 +2232,7 @@ def time_stamper_thread(interval=10):
 
 def configure_cython(options):
     global CompilationOptions, pyrex_default_options, cython_compile
-    from Cython.Compiler.Main import \
+    from Cython.Compiler.Options import \
         CompilationOptions, \
         default_options as pyrex_default_options
     from Cython.Compiler.Options import _directive_defaults as directive_defaults
@@ -2304,7 +2304,7 @@ def runtests(options, cmd_args, coverage=None):
         options.cleanup_sharedlibs = False
         options.fork = False
         if WITH_CYTHON and include_debugger:
-            from Cython.Compiler.Main import default_options as compiler_default_options
+            from Cython.Compiler.Options import default_options as compiler_default_options
             compiler_default_options['gdb_debug'] = True
             compiler_default_options['output_dir'] = os.getcwd()