Merge pull request #2315 from kif/2314_compile_time_env

Compile time env available from cython command line options

Cython/Build/Cythonize.py

@@ -25,9 +25,14 @@ class _FakePool(object):
         for _ in imap(func, args):
             pass
 
-    def close(self): pass
-    def terminate(self): pass
-    def join(self): pass
+    def close(self):
+        pass
+
+    def terminate(self):
+        pass
+
+    def join(self):
+        pass
 
 
 def parse_directives(option, name, value, parser):
@@ -52,6 +57,13 @@ def parse_options(option, name, value, parser):
     setattr(parser.values, dest, options)
 
 
+def parse_compile_time_env(option, name, value, parser):
+    dest = option.dest
+    old_env = dict(getattr(parser.values, dest, {}))
+    directives = Options.parse_compile_time_env(value, current_settings=old_env)
+    setattr(parser.values, dest, directives)
+
+
 def find_package_base(path):
     base_dir, package_path = os.path.split(path)
     while os.path.isfile(os.path.join(base_dir, '__init__.py')):
@@ -85,6 +97,7 @@ def cython_compile(path_pattern, options):
                 exclude_failures=options.keep_going,
                 exclude=options.excludes,
                 compiler_directives=options.directives,
+                compile_time_env=options.compile_time_env,
                 force=options.force,
                 quiet=options.quiet,
                 **options.options)
@@ -136,11 +149,17 @@ def parse_args(args):
     from optparse import OptionParser
     parser = OptionParser(usage='%prog [options] [sources and packages]+')
 
-    parser.add_option('-X', '--directive', metavar='NAME=VALUE,...', dest='directives',
-                      type=str, action='callback', callback=parse_directives, default={},
+    parser.add_option('-X', '--directive', metavar='NAME=VALUE,...',
+                      dest='directives', default={}, type="str",
+                      action='callback', callback=parse_directives,
                       help='set a compiler directive')
-    parser.add_option('-s', '--option', metavar='NAME=VALUE', dest='options',
-                      type=str, action='callback', callback=parse_options, default={},
+    parser.add_option('-E', '--compile-time-env', metavar='NAME=VALUE,...',
+                      dest='compile_time_env', default={}, type="str",
+                      action='callback', callback=parse_compile_time_env,
+                      help='set a compile time environment variable')
+    parser.add_option('-s', '--option', metavar='NAME=VALUE',
+                      dest='options', default={}, type="str",
+                      action='callback', callback=parse_options,
                       help='set a cythonize option')
     parser.add_option('-3', dest='python3_mode', action='store_true',
                       help='use Python 3 syntax mode by default')

Cython/Compiler/CmdLine.py

@@ -47,10 +47,11 @@ Options:
   --warning-errors, -Werror      Make all warnings into errors
   --warning-extra, -Wextra       Enable extra warnings
   -X, --directive <name>=<value>[,<name=value,...] Overrides a compiler directive
+  -E, --compile-time-env name=value[,<name=value,...] Provides compile time env like DEF would do.
 """
 
 
-#The following experimental options are supported only on MacOSX:
+# The following experimental options are supported only on MacOSX:
 #  -C, --compile    Compile generated .c file to .o file
 #  --link           Link .o file to produce extension module (implies -C)
 #  -+, --cplus      Use C++ compiler for compiling and linking
@@ -174,6 +175,17 @@ def parse_command_line(args):
                 except ValueError as e:
                     sys.stderr.write("Error in compiler directive: %s\n" % e.args[0])
                     sys.exit(1)
+            elif option == "--compile-time-env" or option.startswith('-E'):
+                if option.startswith('-E') and option[2:].strip():
+                    x_args = option[2:]
+                else:
+                    x_args = pop_value()
+                try:
+                    options.compile_time_env = Options.parse_compile_time_env(
+                        x_args, current_settings=options.compile_time_env)
+                except ValueError as e:
+                    sys.stderr.write("Error in compile-time-env: %s\n" % e.args[0])
+                    sys.exit(1)
             elif option.startswith('--debug'):
                 option = option[2:].replace('-', '_')
                 from . import DebugFlags

Cython/Compiler/Options.py

@@ -4,6 +4,7 @@
 
 from __future__ import absolute_import
 
+
 class ShouldBeFromDirective(object):
 
     known_directives = []
@@ -120,21 +121,21 @@ closure_freelist_size = 8
 
 
 def get_directive_defaults():
-  # To add an item to this list, all accesses should be changed to use the new
-  # directive, and the global option itself should be set to an instance of
-  # ShouldBeFromDirective.
-  for old_option in ShouldBeFromDirective.known_directives:
-    value = globals().get(old_option.options_name)
-    assert old_option.directive_name in _directive_defaults
-    if not isinstance(value, ShouldBeFromDirective):
-        if old_option.disallow:
-            raise RuntimeError(
-                "Option '%s' must be set from directive '%s'" % (
-                old_option.option_name, old_option.directive_name))
-        else:
-            # Warn?
-            _directive_defaults[old_option.directive_name] = value
-  return _directive_defaults
+    # To add an item to this list, all accesses should be changed to use the new
+    # directive, and the global option itself should be set to an instance of
+    # ShouldBeFromDirective.
+    for old_option in ShouldBeFromDirective.known_directives:
+        value = globals().get(old_option.options_name)
+        assert old_option.directive_name in _directive_defaults
+        if not isinstance(value, ShouldBeFromDirective):
+            if old_option.disallow:
+                raise RuntimeError(
+                    "Option '%s' must be set from directive '%s'" % (
+                    old_option.option_name, old_option.directive_name))
+            else:
+                # Warn?
+                _directive_defaults[old_option.directive_name] = value
+    return _directive_defaults
 
 # Declare compiler directives
 _directive_defaults = {
@@ -146,14 +147,14 @@ _directive_defaults = {
     'exceptval' : None,  # (except value=None, check=True)
     'auto_cpdef': False,
     'auto_pickle': None,
-    'cdivision': False, # was True before 0.12
+    'cdivision': False,  # was True before 0.12
     'cdivision_warnings': False,
     'overflowcheck': False,
     'overflowcheck.fold': True,
     'always_allow_keywords': False,
     'allow_none_for_extension_args': True,
     'wraparound' : True,
-    'ccomplex' : False, # use C99/C++ for complex types and arith
+    'ccomplex' : False,  # use C99/C++ for complex types and arith
     'callspec' : "",
     'final' : False,
     'internal' : False,
@@ -162,20 +163,20 @@ _directive_defaults = {
     'no_gc': False,
     'linetrace': False,
     'emit_code_comments': True,  # copy original source code into C code comments
-    'annotation_typing': True,   # read type declarations from Python function annotations
+    'annotation_typing': True,  # read type declarations from Python function annotations
     'infer_types': None,
     'infer_types.verbose': False,
     'autotestdict': True,
     'autotestdict.cdef': False,
     'autotestdict.all': False,
     'language_level': 2,
-    'fast_getattr': False, # Undocumented until we come up with a better way to handle this everywhere.
-    'py2_import': False, # For backward compatibility of Cython's source code in Py3 source mode
+    'fast_getattr': False,  # Undocumented until we come up with a better way to handle this everywhere.
+    'py2_import': False,  # For backward compatibility of Cython's source code in Py3 source mode
     'preliminary_late_includes_cy28': False,  # Temporary directive in 0.28, to be removed in a later version (see GH#2079).
     'iterable_coroutine': False,  # Make async coroutines backwards compatible with the old asyncio yield-from syntax.
     'c_string_type': 'bytes',
     'c_string_encoding': '',
-    'type_version_tag': True,   # enables Py_TPFLAGS_HAVE_VERSION_TAG on extension types
+    'type_version_tag': True,  # enables Py_TPFLAGS_HAVE_VERSION_TAG on extension types
     'unraisable_tracebacks': True,
     'old_style_globals': False,
     'np_pythran': False,
@@ -195,16 +196,16 @@ _directive_defaults = {
 
 # optimizations
     'optimize.inline_defnode_calls': True,
-    'optimize.unpack_method_calls': True,   # increases code size when True
-    'optimize.unpack_method_calls_in_pyinit': False,   # uselessly increases code size when True
+    'optimize.unpack_method_calls': True,  # increases code size when True
+    'optimize.unpack_method_calls_in_pyinit': False,  # uselessly increases code size when True
     'optimize.use_switch': True,
 
 # remove unreachable code
     'remove_unreachable': True,
 
 # control flow debug directives
-    'control_flow.dot_output': "", # Graphviz output filename
-    'control_flow.dot_annotate_defs': False, # Annotate definitions
+    'control_flow.dot_output': "",  # Graphviz output filename
+    'control_flow.dot_annotate_defs': False,  # Annotate definitions
 
 # test support
     'test_assert_path_exists' : [],
@@ -273,9 +274,9 @@ directive_types = {
     'auto_pickle': bool,
     'final' : bool,  # final cdef classes and methods
     'internal' : bool,  # cdef class visibility in the module dict
-    'infer_types' : bool, # values can be True/None/False
+    'infer_types' : bool,  # values can be True/None/False
     'binding' : bool,
-    'cfunc' : None, # decorators do not take directive value
+    'cfunc' : None,  # decorators do not take directive value
     'ccall' : None,
     'inline' : None,
     'staticmethod' : None,
@@ -291,7 +292,7 @@ for key, val in _directive_defaults.items():
     if key not in directive_types:
         directive_types[key] = type(val)
 
-directive_scopes = { # defaults to available everywhere
+directive_scopes = {  # defaults to available everywhere
     # 'module', 'function', 'class', 'with statement'
     'auto_pickle': ('module', 'cclass'),
     'final' : ('cclass', 'function'),
@@ -423,7 +424,7 @@ def parse_directive_list(s, relaxed_bool=False, ignore_unknown=False,
         item = item.strip()
         if not item:
             continue
-        if not '=' in item:
+        if '=' not in item:
             raise ValueError('Expected "=" in option "%s"' % item)
         name, value = [s.strip() for s in item.strip().split('=', 1)]
         if name not in _directive_defaults:
@@ -441,3 +442,73 @@ def parse_directive_list(s, relaxed_bool=False, ignore_unknown=False,
             parsed_value = parse_directive_value(name, value, relaxed_bool=relaxed_bool)
             result[name] = parsed_value
     return result
+
+
+def parse_variable_value(value):
+    """
+    Parses value as an option value for the given name and returns
+    the interpreted value.
+
+    >>> parse_variable_value('True')
+    True
+    >>> parse_variable_value('true')
+    'true'
+    >>> parse_variable_value('us-ascii')
+    'us-ascii'
+    >>> parse_variable_value('str')
+    'str'
+    >>> parse_variable_value('123')
+    123
+    >>> parse_variable_value('1.23')
+    1.23
+
+    """
+    if value == "True":
+        return True
+    elif value == "False":
+        return False
+    elif value == "None":
+        return None
+    elif value.isdigit():
+        return int(value)
+    else:
+        try:
+            value = float(value)
+        except Exception:
+            # Not a float
+            pass
+        return value
+
+
+def parse_compile_time_env(s, current_settings=None):
+    """
+    Parses a comma-separated list of pragma options. Whitespace
+    is not considered.
+
+    >>> parse_compile_time_env('      ')
+    {}
+    >>> (parse_compile_time_env('HAVE_OPENMP=True') ==
+    ... {'HAVE_OPENMP': True})
+    True
+    >>> parse_compile_time_env('  asdf')
+    Traceback (most recent call last):
+       ...
+    ValueError: Expected "=" in option "asdf"
+    >>> parse_compile_time_env('NUM_THREADS=4') == {'NUM_THREADS': 4}
+    True
+    >>> parse_compile_time_env('unknown=anything') == {'unknown': 'anything'}
+    True
+    """
+    if current_settings is None:
+        result = {}
+    else:
+        result = current_settings
+    for item in s.split(','):
+        item = item.strip()
+        if not item:
+            continue
+        if '=' not in item:
+            raise ValueError('Expected "=" in option "%s"' % item)
+        name, value = [s.strip() for s in item.split('=', 1)]
+        result[name] = parse_variable_value(value)
+    return result