Skip to content

C
cython
Commit 8f1833a3 authored by Stefan Behnel's avatar Stefan Behnel Committed by GitHub
Browse files
Options

Merge pull request #2835 from jbrockmendel/refopts 

REF: Move CompilationOptions and default_options to Options
parents e80bbb3e 6318423f
Hide whitespace changes
Inline Side-by-side
Showing with 211 additions and 199 deletions
Cython/Build/Dependencies.py
View file @ 8f1833a3
......@@ -42,7 +42,8 @@ except:
from .. import Utils
from ..Utils import (cached_function, cached_method, path_exists,
safe_makedirs, copy_file_to_dir_if_newer, is_package_dir, replace_suffix)
from ..Compiler.Main import Context, CompilationOptions, default_options
from ..Compiler.Main import Context
from ..Compiler.Options import CompilationOptions, default_options
join_path = cached_function(os.path.join)
copy_once_if_newer = cached_function(copy_file_to_dir_if_newer)
......@@ -955,7 +956,7 @@ def cythonize(module_list, exclude=None, nthreads=0, aliases=None, quiet=False,
c_options = CompilationOptions(**options)
cpp_options = CompilationOptions(**options); cpp_options.cplus = True
ctx = c_options.create_context()
ctx = Context.from_options(c_options)
options = c_options
module_list, module_metadata = create_extension_list(
module_list,
......
Cython/Build/Inline.py
View file @ 8f1833a3
......@@ -8,7 +8,8 @@ from distutils.core import Distribution, Extension
from distutils.command.build_ext import build_ext
import Cython
from ..Compiler.Main import Context, CompilationOptions, default_options
from ..Compiler.Main import Context
from ..Compiler.Options import CompilationOptions, default_options
from ..Compiler.ParseTreeTransforms import (CythonTransform,
SkipDeclarations, AnalyseDeclarationsTransform, EnvTransform)
......
Cython/Compiler/CmdLine.py
View file @ 8f1833a3
......@@ -65,8 +65,6 @@ def bad_usage():
def parse_command_line(args):
from .Main import CompilationOptions, default_options
pending_arg = []
def pop_arg():
......@@ -94,7 +92,7 @@ def parse_command_line(args):
else:
return pop_arg()
options = CompilationOptions(default_options)
options = Options.CompilationOptions(Options.default_options)
sources = []
while args:
if args[0].startswith("-"):
......
Cython/Compiler/Main.py
View file @ 8f1833a3
......@@ -30,6 +30,8 @@ from .Errors import PyrexError, CompileError, error, warning
from .Symtab import ModuleScope
from .. import Utils
from . import Options
from .Options import CompilationOptions, default_options
from .CmdLine import parse_command_line
from . import Version # legacy import needed by old PyTables versions
version = Version.version # legacy attribute - use "Cython.__version__" instead
......@@ -82,6 +84,11 @@ class Context(object):
self.gdb_debug_outputwriter = None
@classmethod
def from_options(cls, options):
return cls(options.include_path, options.compiler_directives,
options.cplus, options.language_level, options=options)
def set_language_level(self, level):
from .Future import print_function, unicode_literals, absolute_import, division, generator_stop
future_directives = set()
......@@ -449,7 +456,7 @@ def run_pipeline(source, options, full_module_name=None, context=None):
source_ext = os.path.splitext(source)[1]
options.configure_language_defaults(source_ext[1:]) # py/pyx
if context is None:
context = options.create_context()
context = Context.from_options(options)
# Set up source object
cwd = os.getcwd()
......@@ -507,146 +514,6 @@ class CompilationSource(object):
self.cwd = cwd
class CompilationOptions(object):
r"""
See default_options at the end of this module for a list of all possible
options and CmdLine.usage and CmdLine.parse_command_line() for their
meaning.
"""
def __init__(self, defaults=None, **kw):
self.include_path = []
if defaults:
if isinstance(defaults, CompilationOptions):
defaults = defaults.__dict__
else:
defaults = default_options
options = dict(defaults)
options.update(kw)
# let's assume 'default_options' contains a value for most known compiler options
# and validate against them
unknown_options = set(options) - set(default_options)
# ignore valid options that are not in the defaults
unknown_options.difference_update(['include_path'])
if unknown_options:
message = "got unknown compilation option%s, please remove: %s" % (
's' if len(unknown_options) > 1 else '',
', '.join(unknown_options))
raise ValueError(message)
directive_defaults = Options.get_directive_defaults()
directives = dict(options['compiler_directives']) # copy mutable field
# check for invalid directives
unknown_directives = set(directives) - set(directive_defaults)
if unknown_directives:
message = "got unknown compiler directive%s: %s" % (
's' if len(unknown_directives) > 1 else '',
', '.join(unknown_directives))
raise ValueError(message)
options['compiler_directives'] = directives
if directives.get('np_pythran', False) and not options['cplus']:
import warnings
warnings.warn("C++ mode forced when in Pythran mode!")
options['cplus'] = True
if 'language_level' in directives and 'language_level' not in kw:
options['language_level'] = directives['language_level']
elif not options.get('language_level'):
options['language_level'] = directive_defaults.get('language_level')
if 'formal_grammar' in directives and 'formal_grammar' not in kw:
options['formal_grammar'] = directives['formal_grammar']
if options['cache'] is True:
options['cache'] = os.path.join(Utils.get_cython_cache_dir(), 'compiler')
self.__dict__.update(options)
def configure_language_defaults(self, source_extension):
if source_extension == 'py':
if self.compiler_directives.get('binding') is None:
self.compiler_directives['binding'] = True
def create_context(self):
return Context(self.include_path, self.compiler_directives,
self.cplus, self.language_level, options=self)
def get_fingerprint(self):
r"""
Return a string that contains all the options that are relevant for cache invalidation.
"""
# Collect only the data that can affect the generated file(s).
data = {}
for key, value in self.__dict__.items():
if key in ['show_version', 'errors_to_stderr', 'verbose', 'quiet']:
# verbosity flags have no influence on the compilation result
continue
elif key in ['output_file', 'output_dir']:
# ignore the exact name of the output file
continue
elif key in ['timestamps']:
# the cache cares about the content of files, not about the timestamps of sources
continue
elif key in ['cache']:
# hopefully caching has no influence on the compilation result
continue
elif key in ['compiler_directives']:
# directives passed on to the C compiler do not influence the generated C code
continue
elif key in ['include_path']:
# this path changes which headers are tracked as dependencies,
# it has no influence on the generated C code
continue
elif key in ['working_path']:
# this path changes where modules and pxd files are found;
# their content is part of the fingerprint anyway, their
# absolute path does not matter
continue
elif key in ['create_extension']:
# create_extension() has already mangled the options, e.g.,
# embedded_metadata, when the fingerprint is computed so we
# ignore it here.
continue
elif key in ['build_dir']:
# the (temporary) directory where we collect dependencies
# has no influence on the C output
continue
elif key in ['use_listing_file', 'generate_pxi', 'annotate', 'annotate_coverage_xml']:
# all output files are contained in the cache so the types of
# files generated must be part of the fingerprint
data[key] = value
elif key in ['formal_grammar', 'evaluate_tree_assertions']:
# these bits can change whether compilation to C passes/fails
data[key] = value
elif key in ['embedded_metadata', 'emit_linenums', 'c_line_in_traceback', 'gdb_debug', 'relative_path_in_code_position_comments']:
# the generated code contains additional bits when these are set
data[key] = value
elif key in ['cplus', 'language_level', 'compile_time_env', 'np_pythran']:
# assorted bits that, e.g., influence the parser
data[key] = value
elif key == ['capi_reexport_cincludes']:
if self.capi_reexport_cincludes:
# our caching implementation does not yet include fingerprints of all the header files
raise NotImplementedError('capi_reexport_cincludes is not compatible with Cython caching')
elif key == ['common_utility_include_dir']:
if self.common_utility_include_dir:
raise NotImplementedError('common_utility_include_dir is not compatible with Cython caching yet')
else:
# any unexpected option should go into the fingerprint; it's better
# to recompile than to return incorrect results from the cache.
data[key] = value
def to_fingerprint(item):
r"""
Recursively turn item into a string, turning dicts into lists with
deterministic ordering.
"""
if isinstance(item, dict):
item = sorted([(repr(key), to_fingerprint(value)) for key, value in item.items()])
return repr(item)
return to_fingerprint(data)
class CompilationResult(object):
"""
Results from the Cython compiler:
......@@ -709,7 +576,7 @@ def compile_multiple(sources, options):
if these are specified in the options.
"""
# run_pipeline creates the context
# context = options.create_context()
# context = Context.from_options(options)
sources = [os.path.abspath(source) for source in sources]
processed = set()
results = CompilationResultSet()
......@@ -720,7 +587,7 @@ def compile_multiple(sources, options):
for source in sources:
if source not in processed:
if context is None:
context = options.create_context()
context = Context.from_options(options)
output_filename = get_output_filename(source, cwd, options)
out_of_date = context.c_file_out_of_date(source, output_filename)
if (not timestamps) or out_of_date:
......@@ -767,7 +634,6 @@ def main(command_line = 0):
args = sys.argv[1:]
any_failures = 0
if command_line:
from .CmdLine import parse_command_line
options, sources = parse_command_line(args)
else:
options = CompilationOptions(default_options)
......@@ -786,42 +652,3 @@ def main(command_line = 0):
any_failures = 1
if any_failures:
sys.exit(1)
# ------------------------------------------------------------------------
#
# Set the default options depending on the platform
#
# ------------------------------------------------------------------------
default_options = dict(
show_version = 0,
use_listing_file = 0,
errors_to_stderr = 1,
cplus = 0,
output_file = None,
annotate = None,
annotate_coverage_xml = None,
generate_pxi = 0,
capi_reexport_cincludes = 0,
working_path = "",
timestamps = None,
verbose = 0,
quiet = 0,
compiler_directives = {},
embedded_metadata = {},
evaluate_tree_assertions = False,
emit_linenums = False,
relative_path_in_code_position_comments = True,
c_line_in_traceback = True,
language_level = None, # warn but default to 2
formal_grammar = False,
gdb_debug = False,
compile_time_env = None,
common_utility_include_dir = None,
output_dir=None,
build_dir=None,
cache=None,
create_extension=None,
np_pythran=False
)
Cython/Compiler/Options.py
View file @ 8f1833a3
......@@ -4,6 +4,10 @@
from __future__ import absolute_import
import os
from Cython import Utils
class ShouldBeFromDirective(object):
......@@ -543,3 +547,184 @@ def parse_compile_time_env(s, current_settings=None):
name, value = [s.strip() for s in item.split('=', 1)]
result[name] = parse_variable_value(value)
return result
# ------------------------------------------------------------------------
# CompilationOptions are constructed from user input and are the `option`
# object passed throughout the compilation pipeline.
class CompilationOptions(object):
r"""
See default_options at the end of this module for a list of all possible
options and CmdLine.usage and CmdLine.parse_command_line() for their
meaning.
"""
def __init__(self, defaults=None, **kw):
self.include_path = []
if defaults:
if isinstance(defaults, CompilationOptions):
defaults = defaults.__dict__
else:
defaults = default_options
options = dict(defaults)
options.update(kw)
# let's assume 'default_options' contains a value for most known compiler options
# and validate against them
unknown_options = set(options) - set(default_options)
# ignore valid options that are not in the defaults
unknown_options.difference_update(['include_path'])
if unknown_options:
message = "got unknown compilation option%s, please remove: %s" % (
's' if len(unknown_options) > 1 else '',
', '.join(unknown_options))
raise ValueError(message)
directive_defaults = get_directive_defaults()
directives = dict(options['compiler_directives']) # copy mutable field
# check for invalid directives
unknown_directives = set(directives) - set(directive_defaults)
if unknown_directives:
message = "got unknown compiler directive%s: %s" % (
's' if len(unknown_directives) > 1 else '',
', '.join(unknown_directives))
raise ValueError(message)
options['compiler_directives'] = directives
if directives.get('np_pythran', False) and not options['cplus']:
import warnings
warnings.warn("C++ mode forced when in Pythran mode!")
options['cplus'] = True
if 'language_level' in directives and 'language_level' not in kw:
options['language_level'] = directives['language_level']
elif not options.get('language_level'):
options['language_level'] = directive_defaults.get('language_level')
if 'formal_grammar' in directives and 'formal_grammar' not in kw:
options['formal_grammar'] = directives['formal_grammar']
if options['cache'] is True:
options['cache'] = os.path.join(Utils.get_cython_cache_dir(), 'compiler')
self.__dict__.update(options)
def configure_language_defaults(self, source_extension):
if source_extension == 'py':
if self.compiler_directives.get('binding') is None:
self.compiler_directives['binding'] = True
def get_fingerprint(self):
r"""
Return a string that contains all the options that are relevant for cache invalidation.
"""
# Collect only the data that can affect the generated file(s).
data = {}
for key, value in self.__dict__.items():
if key in ['show_version', 'errors_to_stderr', 'verbose', 'quiet']:
# verbosity flags have no influence on the compilation result
continue
elif key in ['output_file', 'output_dir']:
# ignore the exact name of the output file
continue
elif key in ['timestamps']:
# the cache cares about the content of files, not about the timestamps of sources
continue
elif key in ['cache']:
# hopefully caching has no influence on the compilation result
continue
elif key in ['compiler_directives']:
# directives passed on to the C compiler do not influence the generated C code
continue
elif key in ['include_path']:
# this path changes which headers are tracked as dependencies,
# it has no influence on the generated C code
continue
elif key in ['working_path']:
# this path changes where modules and pxd files are found;
# their content is part of the fingerprint anyway, their
# absolute path does not matter
continue
elif key in ['create_extension']:
# create_extension() has already mangled the options, e.g.,
# embedded_metadata, when the fingerprint is computed so we
# ignore it here.
continue
elif key in ['build_dir']:
# the (temporary) directory where we collect dependencies
# has no influence on the C output
continue
elif key in ['use_listing_file', 'generate_pxi', 'annotate', 'annotate_coverage_xml']:
# all output files are contained in the cache so the types of
# files generated must be part of the fingerprint
data[key] = value
elif key in ['formal_grammar', 'evaluate_tree_assertions']:
# these bits can change whether compilation to C passes/fails
data[key] = value
elif key in ['embedded_metadata', 'emit_linenums',
'c_line_in_traceback', 'gdb_debug',
'relative_path_in_code_position_comments']:
# the generated code contains additional bits when these are set
data[key] = value
elif key in ['cplus', 'language_level', 'compile_time_env', 'np_pythran']:
# assorted bits that, e.g., influence the parser
data[key] = value
elif key == ['capi_reexport_cincludes']:
if self.capi_reexport_cincludes:
# our caching implementation does not yet include fingerprints of all the header files
raise NotImplementedError('capi_reexport_cincludes is not compatible with Cython caching')
elif key == ['common_utility_include_dir']:
if self.common_utility_include_dir:
raise NotImplementedError('common_utility_include_dir is not compatible with Cython caching yet')
else:
# any unexpected option should go into the fingerprint; it's better
# to recompile than to return incorrect results from the cache.
data[key] = value
def to_fingerprint(item):
r"""
Recursively turn item into a string, turning dicts into lists with
deterministic ordering.
"""
if isinstance(item, dict):
item = sorted([(repr(key), to_fingerprint(value)) for key, value in item.items()])
return repr(item)
return to_fingerprint(data)
# ------------------------------------------------------------------------
#
# Set the default options depending on the platform
#
# ------------------------------------------------------------------------
default_options = dict(
show_version=0,
use_listing_file=0,
errors_to_stderr=1,
cplus=0,
output_file=None,
annotate=None,
annotate_coverage_xml=None,
generate_pxi=0,
capi_reexport_cincludes=0,
working_path="",
timestamps=None,
verbose=0,
quiet=0,
compiler_directives={},
embedded_metadata={},
evaluate_tree_assertions=False,
emit_linenums=False,
relative_path_in_code_position_comments=True,
c_line_in_traceback=True,
language_level=None, # warn but default to 2
formal_grammar=False,
gdb_debug=False,
compile_time_env=None,
common_utility_include_dir=None,
output_dir=None,
build_dir=None,
cache=None,
create_extension=None,
np_pythran=False
)
Cython/Compiler/Tests/TestParseTreeTransforms.py
View file @ 8f1833a3
......@@ -3,7 +3,7 @@ import os
from Cython.TestUtils import TransformTest
from Cython.Compiler.ParseTreeTransforms import *
from Cython.Compiler.Nodes import *
from Cython.Compiler import Main, Symtab
from Cython.Compiler import Main, Symtab, Options
class TestNormalizeTree(TransformTest):
......@@ -177,8 +177,8 @@ class TestInterpretCompilerDirectives(TransformTest):
def setUp(self):
super(TestInterpretCompilerDirectives, self).setUp()
compilation_options = Main.CompilationOptions(Main.default_options)
ctx = compilation_options.create_context()
compilation_options = Options.CompilationOptions(Options.default_options)
ctx = Main.Context.from_options(compilation_options)
transform = InterpretCompilerDirectives(ctx, ctx.compiler_directives)
transform.module_scope = Symtab.ModuleScope('__main__', None, ctx)
......
Cython/Tests/TestJediTyper.py
View file @ 8f1833a3
......@@ -11,7 +11,7 @@ from contextlib import contextmanager
from tempfile import NamedTemporaryFile
from Cython.Compiler.ParseTreeTransforms import NormalizeTree, InterpretCompilerDirectives
from Cython.Compiler import Main, Symtab, Visitor
from Cython.Compiler import Main, Symtab, Visitor, Options
from Cython.TestUtils import TransformTest
TOOLS_DIR = os.path.abspath(os.path.join(os.path.dirname(__file__), '..', '..', 'Tools'))
......@@ -210,8 +210,8 @@ class TestTypeInjection(TestJediTyper):
"""
def setUp(self):
super(TestTypeInjection, self).setUp()
compilation_options = Main.CompilationOptions(Main.default_options)
ctx = compilation_options.create_context()
compilation_options = Options.CompilationOptions(Options.default_options)
ctx = Main.Context.from_options(compilation_options)
transform = InterpretCompilerDirectives(ctx, ctx.compiler_directives)
transform.module_scope = Symtab.ModuleScope('__main__', None, ctx)
self.declarations_finder = DeclarationsFinder()
......
runtests.py
View file @ 8f1833a3
......@@ -1029,9 +1029,9 @@ class CythonCompileTestCase(unittest.TestCase):
try:
CompilationOptions
except NameError:
from Cython.Compiler.Main import CompilationOptions
from Cython.Compiler.Options import CompilationOptions
from Cython.Compiler.Main import compile as cython_compile
from Cython.Compiler.Main import default_options
from Cython.Compiler.Options import default_options
common_utility_include_dir = self.common_utility_dir
options = CompilationOptions(
......@@ -2232,7 +2232,7 @@ def time_stamper_thread(interval=10):
def configure_cython(options):
global CompilationOptions, pyrex_default_options, cython_compile
from Cython.Compiler.Main import \
from Cython.Compiler.Options import \
CompilationOptions, \
default_options as pyrex_default_options
from Cython.Compiler.Options import _directive_defaults as directive_defaults
......@@ -2304,7 +2304,7 @@ def runtests(options, cmd_args, coverage=None):
options.cleanup_sharedlibs = False
options.fork = False
if WITH_CYTHON and include_debugger:
from Cython.Compiler.Main import default_options as compiler_default_options
from Cython.Compiler.Options import default_options as compiler_default_options
compiler_default_options['gdb_debug'] = True
compiler_default_options['output_dir'] = os.getcwd()
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment
GitLab Nexedi Edition | About GitLab | About Nexedi | 沪ICP备2021021310号-2 | 沪ICP备2021021310号-7