gpython: Add way to run it with threads runtime

Until now gpython was always activating gevent on startup + adding
goodness such as "always UTF-8 default encoding"; go, chan, b/u and
friends available from builtin namespace, etc... While those goodness
are sometimes useful on their own, it is not always appropriate to force
a project to switch from threads to gevent.

For this reason add a flag to select which runtime - either gevent
or threads - gpython should use.

	gpython -Xgpython.runtime=gevent	selects gevent,

while

	gpython -Xgpython.runtime=threads	selects threads.

Gevent remains the default.

It is also possible to specify desired runtime via $GPYTHON_RUNTIME
environment variable.

/reviewed-on nexedi/pygolang!5

README.rst

@@ -53,6 +53,8 @@ Additionally GPython sets UTF-8 to be default encoding always, and puts `go`,
 
    GPython is optional and the rest of Pygolang can be used from under standard Python too.
    However without gevent integration `go` spawns full - not lightweight - OS thread.
+   GPython can be also used with threads - not gevent - runtime. Please see
+   `GPython options`_ for details.
 
 
 Goroutines and channels
@@ -502,3 +504,22 @@ such benchmarking data in Python.
 .. _golang.x.perf.benchlib: https://lab.nexedi.com/nexedi/pygolang/tree/master/golang/x/perf/benchlib.py
 __ https://github.com/golang/proposal/blob/master/design/14313-benchmark-format.md
 __ https://godoc.org/golang.org/x/perf/cmd/benchstat
+
+
+--------
+
+GPython options
+---------------
+
+GPython mimics and supports most of Python command-line options, like `gpython
+-c <commands>` to run Python statements from command line, or `gpython -m
+<module>` to execute a module. Such options have the same meaning as in
+standard Python and are not documented here.
+
+GPython-specific options and environment variables are listed below:
+
+`-X gpython.runtime=(gevent|threads)`
+    Specify which runtime GPython should use. `gevent` provides lightweight
+    coroutines, while with `threads` `go` spawns full OS thread. `gevent` is
+    default. The runtime to use can be also specified via `$GPYTHON_RUNTIME`
+    environment variable.

gpython/__init__.py

@@ -26,6 +26,9 @@ differences:
 - gevent is pre-activated and stdlib is patched to be gevent aware;
 - go, chan, select etc are put into builtin namespace;
 - default string encoding is always set to UTF-8.
+
+Gevent activation can be disabled via `-X gpython.runtime=threads`, or
+$GPYTHON_RUNTIME=threads.
 """
 
 # NOTE gpython is kept out of golang/ , since even just importing e.g. golang.cmd.gpython,
@@ -75,8 +78,12 @@ def pymain(argv):
         ver = []
         if 'GPython' in sys.version:
             golang = sys.modules['golang'] # must be already imported
-            gevent = sys.modules['gevent'] # must be already imported
-            gpyver = 'GPython %s [gevent %s]' % (golang.__version__, gevent.__version__)
+            gevent = sys.modules.get('gevent', None)
+            gpyver = 'GPython %s' % golang.__version__
+            if gevent is not None:
+                gpyver += ' [gevent %s]' % gevent.__version__
+            else:
+                gpyver += ' [threads]'
             ver.append(gpyver)
 
         import platform
@@ -213,20 +220,65 @@ def main():
         exe = exe + '.exe'
     sys.executable  = exe
 
-    # make gevent pre-available & stdlib patched
-    import gevent
-    from gevent import monkey
-    # XXX workaround for gevent vs pypy2 crash.
-    # XXX remove when gevent-1.4.1 is relased (https://github.com/gevent/gevent/pull/1357).
-    patch_thread=True
-    if pypy and sys.version_info.major == 2:
-        _ = monkey.patch_thread(existing_locks=False)
-        assert _ in (True, None)
-        patch_thread=False
-    _ = monkey.patch_all(thread=patch_thread)      # XXX sys=True ?
-    if _ not in (True, None):   # patched or nothing to do
-        # XXX provide details
-        raise RuntimeError('gevent monkey-patching failed')
+    # import os to get access to environment.
+    # it is practically ok to import os before gevent, because os is always
+    # imported by site. Yes, `import site` can be disabled by -S, but there is
+    # no harm wrt gevent monkey-patching even if we import os first.
+    import os
+
+    # extract and process -X
+    # -X gpython.runtime=(gevent|threads)    + $GPYTHON_RUNTIME
+    sys._xoptions = getattr(sys, '_xoptions', {})
+    argv_ = []
+    gpy_runtime = os.getenv('GPYTHON_RUNTIME', 'gevent')
+    while len(argv) > 0:
+        arg  = argv[0]
+        argv = argv[1:]
+
+        if not arg.startswith('-X'):
+            argv_.append(arg)
+            # continue looking for -X only until options end
+            if not arg.startswith('-'):
+                break
+            continue
+
+        # -X <opt>
+        opt = arg[2:]       # -X<opt>
+        if opt == '':
+            opt  = argv[0]  # -X <opt>
+            argv = argv[1:]
+
+        if opt.startswith('gpython.runtime='):
+            gpy_runtime = opt[len('gpython.runtime='):]
+            sys._xoptions['gpython.runtime'] = gpy_runtime
+
+        else:
+            raise RuntimeError('gpython: unknown -X option %s' % opt)
+    argv = argv_ + argv
+
+    # initialize according to selected runtime
+    if gpy_runtime == 'gevent':
+        # make gevent pre-available & stdlib patched
+        import gevent
+        from gevent import monkey
+        # XXX workaround for gevent vs pypy2 crash.
+        # XXX remove when gevent-1.4.1 is relased (https://github.com/gevent/gevent/pull/1357).
+        patch_thread=True
+        if pypy and sys.version_info.major == 2:
+            _ = monkey.patch_thread(existing_locks=False)
+            assert _ in (True, None)
+            patch_thread=False
+        _ = monkey.patch_all(thread=patch_thread)      # XXX sys=True ?
+        if _ not in (True, None):   # patched or nothing to do
+            # XXX provide details
+            raise RuntimeError('gevent monkey-patching failed')
+        gpy_verextra = 'gevent %s' % gevent.__version__
+
+    elif gpy_runtime == 'threads':
+        gpy_verextra = 'threads'
+
+    else:
+        raise RuntimeError('gpython: invalid runtime %s' % gpy_runtime)
 
     # put go, chan, select, ... into builtin namespace
     import golang
@@ -235,7 +287,7 @@ def main():
         setattr(builtins, k, getattr(golang, k))
 
     # sys.version
-    sys.version += (' [GPython %s] [gevent %s]' % (golang.__version__, gevent.__version__))
+    sys.version += (' [GPython %s] [%s]' % (golang.__version__, gpy_verextra))
 
     # tail to pymain
     pymain(argv)

gpython/gpython_test.py

@@ -33,6 +33,24 @@ is_cpython = (platform.python_implementation() == 'CPython')
 # @gpython_only is marker to run a test only under gpython
 gpython_only = pytest.mark.skipif('GPython' not in sys.version, reason="gpython-only test")
 
+# runtime is pytest fixture that yields all variants of should be supported gpython runtimes:
+# '' - not specified (gpython should autoselect)
+# 'gevent'
+# 'threads'
+@pytest.fixture(scope="function", params=['', 'gevent', 'threads'])
+def runtime(request):
+    yield request.param
+
+# gpyenv returns environment appropriate for spawning gpython with
+# specified runtime.
+def gpyenv(runtime): # -> env
+    env = os.environ.copy()
+    if runtime != '':
+        env['GPYTHON_RUNTIME'] = runtime
+    else:
+        env.pop('GPYTHON_RUNTIME', None)
+    return env
+
 
 @gpython_only
 def test_defaultencoding_utf8():
@@ -54,6 +72,12 @@ def test_golang_builtins():
 
 @gpython_only
 def test_gevent_activated():
+    # gpython, by default, acticates gevent.
+    # handling of various runtime modes is explicitly tested in test_Xruntime.
+    assert_gevent_activated()
+
+def assert_gevent_activated():
+    assert 'gevent' in sys.modules
     from gevent.monkey import is_module_patched as patched, is_object_patched as obj_patched
 
     # builtin (gevent: only on py2 - on py3 __import__ uses fine-grained locking)
@@ -91,15 +115,38 @@ def test_gevent_activated():
     if sys.hexversion >= 0x03070000: # >= 3.7.0
         assert patched('queue')
 
+def assert_gevent_not_activated():
+    assert 'gevent' not in sys.modules
+    from gevent.monkey import is_module_patched as patched, is_object_patched as obj_patched
+
+    assert not patched('socket')
+    assert not patched('time')
+    assert not patched('select')
+    assert not patched('os')
+    assert not patched('signal')
+    assert not patched('thread' if PY2 else '_thread')
+    assert not patched('threading')
+    assert not patched('_threading_local')
+    assert not patched('ssl')
+    assert not patched('subprocess')
+    assert not patched('sys')
+
 
 @gpython_only
-def test_executable():
+def test_executable(runtime):
     # sys.executable must point to gpython and we must be able to execute it.
     import gevent
     assert 'gpython' in sys.executable
-    out = pyout(['-c', 'import sys; print(sys.version)'])
-    assert ('[GPython %s]' % golang.__version__) in str(out)
-    assert ('[gevent %s]'  % gevent.__version__) in str(out)
+    ver = pyout(['-c', 'import sys; print(sys.version)'], env=gpyenv(runtime))
+    ver = str(ver)
+    assert ('[GPython %s]' % golang.__version__) in ver
+    if runtime != 'threads':
+        assert ('[gevent %s]'  % gevent.__version__)     in ver
+        assert ('[threads]')                         not in ver
+    else:
+        assert ('[gevent ')                          not in ver
+        assert ('[threads]')                             in ver
+
 
 # verify pymain.
 #
@@ -138,11 +185,15 @@ def test_pymain():
 # pymain -V/--version
 # gpython_only because output differs from !gpython.
 @gpython_only
-def test_pymain_ver():
+def test_pymain_ver(runtime):
     from golang import b
     from gpython import _version_info_str as V
     import gevent
-    vok = 'GPython %s [gevent %s]' % (golang.__version__, gevent.__version__)
+    vok = 'GPython %s' % golang.__version__
+    if runtime != 'threads':
+        vok += ' [gevent %s]' % gevent.__version__
+    else:
+        vok += ' [threads]'
 
     if is_cpython:
         vok += ' / CPython %s' % platform.python_version()
@@ -153,8 +204,29 @@ def test_pymain_ver():
 
     vok += '\n'
 
-    ret, out, err = _pyrun(['-V'], stdout=PIPE, stderr=PIPE)
+    ret, out, err = _pyrun(['-V'], stdout=PIPE, stderr=PIPE, env=gpyenv(runtime))
     assert (ret, out, b(err)) == (0, b'', b(vok))
 
-    ret, out, err = _pyrun(['--version'], stdout=PIPE, stderr=PIPE)
+    ret, out, err = _pyrun(['--version'], stdout=PIPE, stderr=PIPE, env=gpyenv(runtime))
     assert (ret, out, b(err)) == (0, b'', b(vok))
+
+
+# verify -X gpython.runtime=...
+@gpython_only
+def test_Xruntime(runtime):
+    env = os.environ.copy()
+    env.pop('GPYTHON_RUNTIME', None) # del
+
+    argv = []
+    if runtime != '':
+        argv += ['-X', 'gpython.runtime='+runtime]
+    prog = 'from gpython import gpython_test as t; '
+    if runtime != 'threads':
+        prog += 't.assert_gevent_activated(); '
+    else:
+        prog += 't.assert_gevent_not_activated(); '
+    prog += 'print("ok")'
+    argv += ['-c', prog]
+
+    out = pyout(argv, env=env)
+    assert out == b'ok\n'