Merge pull request #2835 from jbrockmendel/refopts

REF: Move CompilationOptions and default_options to Options

Cython/Build/Dependencies.py

@@ -42,7 +42,8 @@ except:
 from .. import Utils
 from ..Utils import (cached_function, cached_method, path_exists,
     safe_makedirs, copy_file_to_dir_if_newer, is_package_dir, replace_suffix)
-from ..Compiler.Main import Context, CompilationOptions, default_options
+from ..Compiler.Main import Context
+from ..Compiler.Options import CompilationOptions, default_options
 
 join_path = cached_function(os.path.join)
 copy_once_if_newer = cached_function(copy_file_to_dir_if_newer)
@@ -955,7 +956,7 @@ def cythonize(module_list, exclude=None, nthreads=0, aliases=None, quiet=False,
 
     c_options = CompilationOptions(**options)
     cpp_options = CompilationOptions(**options); cpp_options.cplus = True
-    ctx = c_options.create_context()
+    ctx = Context.from_options(c_options)
     options = c_options
     module_list, module_metadata = create_extension_list(
         module_list,

Cython/Build/Inline.py

@@ -8,7 +8,8 @@ from distutils.core import Distribution, Extension
 from distutils.command.build_ext import build_ext
 
 import Cython
-from ..Compiler.Main import Context, CompilationOptions, default_options
+from ..Compiler.Main import Context
+from ..Compiler.Options import CompilationOptions, default_options
 
 from ..Compiler.ParseTreeTransforms import (CythonTransform,
         SkipDeclarations, AnalyseDeclarationsTransform, EnvTransform)

Cython/Compiler/CmdLine.py

@@ -65,8 +65,6 @@ def bad_usage():
 
 
 def parse_command_line(args):
-    from .Main import CompilationOptions, default_options
-
     pending_arg = []
 
     def pop_arg():
@@ -94,7 +92,7 @@ def parse_command_line(args):
         else:
             return pop_arg()
 
-    options = CompilationOptions(default_options)
+    options = Options.CompilationOptions(Options.default_options)
     sources = []
     while args:
         if args[0].startswith("-"):

Cython/Compiler/Options.py

@@ -4,6 +4,10 @@
 
 from __future__ import absolute_import
 
+import os
+
+from Cython import Utils
+
 
 class ShouldBeFromDirective(object):
 
@@ -543,3 +547,184 @@ def parse_compile_time_env(s, current_settings=None):
         name, value = [s.strip() for s in item.split('=', 1)]
         result[name] = parse_variable_value(value)
     return result
+
+
+# ------------------------------------------------------------------------
+# CompilationOptions are constructed from user input and are the `option`
+#  object passed throughout the compilation pipeline.
+
+class CompilationOptions(object):
+    r"""
+    See default_options at the end of this module for a list of all possible
+    options and CmdLine.usage and CmdLine.parse_command_line() for their
+    meaning.
+    """
+    def __init__(self, defaults=None, **kw):
+        self.include_path = []
+        if defaults:
+            if isinstance(defaults, CompilationOptions):
+                defaults = defaults.__dict__
+        else:
+            defaults = default_options
+
+        options = dict(defaults)
+        options.update(kw)
+
+        # let's assume 'default_options' contains a value for most known compiler options
+        # and validate against them
+        unknown_options = set(options) - set(default_options)
+        # ignore valid options that are not in the defaults
+        unknown_options.difference_update(['include_path'])
+        if unknown_options:
+            message = "got unknown compilation option%s, please remove: %s" % (
+                's' if len(unknown_options) > 1 else '',
+                ', '.join(unknown_options))
+            raise ValueError(message)
+
+        directive_defaults = get_directive_defaults()
+        directives = dict(options['compiler_directives'])  # copy mutable field
+        # check for invalid directives
+        unknown_directives = set(directives) - set(directive_defaults)
+        if unknown_directives:
+            message = "got unknown compiler directive%s: %s" % (
+                's' if len(unknown_directives) > 1 else '',
+                ', '.join(unknown_directives))
+            raise ValueError(message)
+        options['compiler_directives'] = directives
+        if directives.get('np_pythran', False) and not options['cplus']:
+            import warnings
+            warnings.warn("C++ mode forced when in Pythran mode!")
+            options['cplus'] = True
+        if 'language_level' in directives and 'language_level' not in kw:
+            options['language_level'] = directives['language_level']
+        elif not options.get('language_level'):
+            options['language_level'] = directive_defaults.get('language_level')
+        if 'formal_grammar' in directives and 'formal_grammar' not in kw:
+            options['formal_grammar'] = directives['formal_grammar']
+        if options['cache'] is True:
+            options['cache'] = os.path.join(Utils.get_cython_cache_dir(), 'compiler')
+
+        self.__dict__.update(options)
+
+    def configure_language_defaults(self, source_extension):
+        if source_extension == 'py':
+            if self.compiler_directives.get('binding') is None:
+                self.compiler_directives['binding'] = True
+
+    def get_fingerprint(self):
+        r"""
+        Return a string that contains all the options that are relevant for cache invalidation.
+        """
+        # Collect only the data that can affect the generated file(s).
+        data = {}
+
+        for key, value in self.__dict__.items():
+            if key in ['show_version', 'errors_to_stderr', 'verbose', 'quiet']:
+                # verbosity flags have no influence on the compilation result
+                continue
+            elif key in ['output_file', 'output_dir']:
+                # ignore the exact name of the output file
+                continue
+            elif key in ['timestamps']:
+                # the cache cares about the content of files, not about the timestamps of sources
+                continue
+            elif key in ['cache']:
+                # hopefully caching has no influence on the compilation result
+                continue
+            elif key in ['compiler_directives']:
+                # directives passed on to the C compiler do not influence the generated C code
+                continue
+            elif key in ['include_path']:
+                # this path changes which headers are tracked as dependencies,
+                # it has no influence on the generated C code
+                continue
+            elif key in ['working_path']:
+                # this path changes where modules and pxd files are found;
+                # their content is part of the fingerprint anyway, their
+                # absolute path does not matter
+                continue
+            elif key in ['create_extension']:
+                # create_extension() has already mangled the options, e.g.,
+                # embedded_metadata, when the fingerprint is computed so we
+                # ignore it here.
+                continue
+            elif key in ['build_dir']:
+                # the (temporary) directory where we collect dependencies
+                # has no influence on the C output
+                continue
+            elif key in ['use_listing_file', 'generate_pxi', 'annotate', 'annotate_coverage_xml']:
+                # all output files are contained in the cache so the types of
+                # files generated must be part of the fingerprint
+                data[key] = value
+            elif key in ['formal_grammar', 'evaluate_tree_assertions']:
+                # these bits can change whether compilation to C passes/fails
+                data[key] = value
+            elif key in ['embedded_metadata', 'emit_linenums',
+                         'c_line_in_traceback', 'gdb_debug',
+                         'relative_path_in_code_position_comments']:
+                # the generated code contains additional bits when these are set
+                data[key] = value
+            elif key in ['cplus', 'language_level', 'compile_time_env', 'np_pythran']:
+                # assorted bits that, e.g., influence the parser
+                data[key] = value
+            elif key == ['capi_reexport_cincludes']:
+                if self.capi_reexport_cincludes:
+                    # our caching implementation does not yet include fingerprints of all the header files
+                    raise NotImplementedError('capi_reexport_cincludes is not compatible with Cython caching')
+            elif key == ['common_utility_include_dir']:
+                if self.common_utility_include_dir:
+                    raise NotImplementedError('common_utility_include_dir is not compatible with Cython caching yet')
+            else:
+                # any unexpected option should go into the fingerprint; it's better
+                # to recompile than to return incorrect results from the cache.
+                data[key] = value
+
+        def to_fingerprint(item):
+            r"""
+            Recursively turn item into a string, turning dicts into lists with
+            deterministic ordering.
+            """
+            if isinstance(item, dict):
+                item = sorted([(repr(key), to_fingerprint(value)) for key, value in item.items()])
+            return repr(item)
+
+        return to_fingerprint(data)
+
+
+# ------------------------------------------------------------------------
+#
+#  Set the default options depending on the platform
+#
+# ------------------------------------------------------------------------
+
+default_options = dict(
+    show_version=0,
+    use_listing_file=0,
+    errors_to_stderr=1,
+    cplus=0,
+    output_file=None,
+    annotate=None,
+    annotate_coverage_xml=None,
+    generate_pxi=0,
+    capi_reexport_cincludes=0,
+    working_path="",
+    timestamps=None,
+    verbose=0,
+    quiet=0,
+    compiler_directives={},
+    embedded_metadata={},
+    evaluate_tree_assertions=False,
+    emit_linenums=False,
+    relative_path_in_code_position_comments=True,
+    c_line_in_traceback=True,
+    language_level=None,  # warn but default to 2
+    formal_grammar=False,
+    gdb_debug=False,
+    compile_time_env=None,
+    common_utility_include_dir=None,
+    output_dir=None,
+    build_dir=None,
+    cache=None,
+    create_extension=None,
+    np_pythran=False
+)

Cython/Compiler/Tests/TestParseTreeTransforms.py

@@ -3,7 +3,7 @@ import os
 from Cython.TestUtils import TransformTest
 from Cython.Compiler.ParseTreeTransforms import *
 from Cython.Compiler.Nodes import *
-from Cython.Compiler import Main, Symtab
+from Cython.Compiler import Main, Symtab, Options
 
 
 class TestNormalizeTree(TransformTest):
@@ -177,8 +177,8 @@ class TestInterpretCompilerDirectives(TransformTest):
     def setUp(self):
         super(TestInterpretCompilerDirectives, self).setUp()
 
-        compilation_options = Main.CompilationOptions(Main.default_options)
-        ctx = compilation_options.create_context()
+        compilation_options = Options.CompilationOptions(Options.default_options)
+        ctx = Main.Context.from_options(compilation_options)
 
         transform = InterpretCompilerDirectives(ctx, ctx.compiler_directives)
         transform.module_scope = Symtab.ModuleScope('__main__', None, ctx)

Cython/Tests/TestJediTyper.py

@@ -11,7 +11,7 @@ from contextlib import contextmanager
 from tempfile import NamedTemporaryFile
 
 from Cython.Compiler.ParseTreeTransforms import NormalizeTree, InterpretCompilerDirectives
-from Cython.Compiler import Main, Symtab, Visitor
+from Cython.Compiler import Main, Symtab, Visitor, Options
 from Cython.TestUtils import TransformTest
 
 TOOLS_DIR = os.path.abspath(os.path.join(os.path.dirname(__file__), '..', '..', 'Tools'))
@@ -210,8 +210,8 @@ class TestTypeInjection(TestJediTyper):
     """
     def setUp(self):
         super(TestTypeInjection, self).setUp()
-        compilation_options = Main.CompilationOptions(Main.default_options)
-        ctx = compilation_options.create_context()
+        compilation_options = Options.CompilationOptions(Options.default_options)
+        ctx = Main.Context.from_options(compilation_options)
         transform = InterpretCompilerDirectives(ctx, ctx.compiler_directives)
         transform.module_scope = Symtab.ModuleScope('__main__', None, ctx)
         self.declarations_finder = DeclarationsFinder()

runtests.py

@@ -1029,9 +1029,9 @@ class CythonCompileTestCase(unittest.TestCase):
         try:
             CompilationOptions
         except NameError:
-            from Cython.Compiler.Main import CompilationOptions
+            from Cython.Compiler.Options import CompilationOptions
             from Cython.Compiler.Main import compile as cython_compile
-            from Cython.Compiler.Main import default_options
+            from Cython.Compiler.Options import default_options
         common_utility_include_dir = self.common_utility_dir
 
         options = CompilationOptions(
@@ -2232,7 +2232,7 @@ def time_stamper_thread(interval=10):
 
 def configure_cython(options):
     global CompilationOptions, pyrex_default_options, cython_compile
-    from Cython.Compiler.Main import \
+    from Cython.Compiler.Options import \
         CompilationOptions, \
         default_options as pyrex_default_options
     from Cython.Compiler.Options import _directive_defaults as directive_defaults
@@ -2304,7 +2304,7 @@ def runtests(options, cmd_args, coverage=None):
         options.cleanup_sharedlibs = False
         options.fork = False
         if WITH_CYTHON and include_debugger:
-            from Cython.Compiler.Main import default_options as compiler_default_options
+            from Cython.Compiler.Options import default_options as compiler_default_options
             compiler_default_options['gdb_debug'] = True
             compiler_default_options['output_dir'] = os.getcwd()