gpython: Python interpreter with support for lightweight threads

Provide gpython interpreter, that sets UTF-8 as default encoding,
integrates gevent and puts go, chan, select etc into builtin namespace.
Please see details in the documentation added by this patch.

pymain was taken from kirr/go123@7a476082
(go123: tracing/pyruntraced: New tool to run Python code with tracepoints activated (draft))

README.rst

@@ -4,6 +4,7 @@
 
 Package `golang` provides Go-like features for Python:
 
+- `gpython` is Python interpreter with support for lightweight threads.
 - `go` spawns lightweight thread.
 - `chan` and `select` provide channels with Go semantic.
 - `method` allows to define methods separate from class.
@@ -16,10 +17,38 @@ between Python and Go environments.
 __ `String conversion`_
 __ `Benchmarking and testing`_
 
+
+GPython
+-------
+
+Command `gpython` provides Python interpreter that supports lightweight threads
+via tight integration with gevent__. The standard library of GPython is API
+compatible with Python standard library, but inplace of OS threads lightweight
+coroutines are provided, and IO is internally organized via
+libuv__/libev__-based IO scheduler. Consequently programs can spawn lots of
+coroutines cheaply, and modules like `time`, `socket`, `ssl`, `subprocess` etc
+all could be used from all coroutines simultaneously, in the same blocking way
+as if every coroutine was a full OS thread. This gives ability to scale servers
+without changing concurrency model and existing code.
+
+__ http://www.gevent.org/
+__ http://libuv.org/
+__ http://software.schmorp.de/pkg/libev.html
+
+
+Additionally GPython sets UTF-8 to be default encoding always, and puts `go`,
+`chan`, `select` etc into builtin namespace.
+
+.. note::
+
+   GPython is optional and the rest of Pygolang can be used from under standard Python too.
+   However without gevent integration `go` spawns full - not lightweight - OS thread.
+
+
 Goroutines and channels
 -----------------------
 
-`go` spawns a thread, or a coroutine if gevent was activated. It is possible to
+`go` spawns a coroutine, or thread if gevent was not activated. It is possible to
 exchange data in between either threads or coroutines via channels. `chan`
 creates a new channel with Go semantic - either synchronous or buffered. Use
 `chan.recv`, `chan.send` and `chan.close` for communication. `select` can be

golang/__init__.py

@@ -172,8 +172,12 @@ def defer(f):
 
 # go spawns lightweight thread.
 #
-# NOTE it spawns threading.Thread, but if gevent was activated via
-# `gevent.monkey.patch_all`, it will spawn greenlet, not full OS thread.
+# go spawns:
+#
+# - lightweight thread (with    gevent integration), or
+# - full OS thread     (without gevent integration).
+#
+# Use gpython to run Python with integrated gevent, or use gevent directly to do so.
 def go(f, *argv, **kw):
     t = threading.Thread(target=f, args=argv, kwargs=kw)
     t.daemon = True # leaked goroutines don't prevent program to exit

gpython/__init__.py

+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+# Copyright (C) 2018-2019  Nexedi SA and Contributors.
+#                          Kirill Smelkov <kirr@nexedi.com>
+#
+# This program is free software: you can Use, Study, Modify and Redistribute
+# it under the terms of the GNU General Public License version 3, or (at your
+# option) any later version, as published by the Free Software Foundation.
+#
+# You can also Link and Combine this program with other software covered by
+# the terms of any of the Free Software licenses or any of the Open Source
+# Initiative approved licenses and Convey the resulting work. Corresponding
+# source of such a combination shall include the source code for all other
+# software used.
+#
+# This program is distributed WITHOUT ANY WARRANTY; without even the implied
+# warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
+#
+# See COPYING file for full licensing terms.
+# See https://www.nexedi.com/licensing for rationale and options.
+"""gpython ... - run python ... with gevent & golang activated
+
+gpython is substitute for standard python interpreter with the following
+differences:
+
+- gevent is pre-activated and stdlib is patched to be gevent aware;
+- go, chan, select etc are put into builtin namespace;
+- default string encoding is always set to UTF-8.
+"""
+
+# NOTE gpython is kept out of golang/ , since even just importing e.g. golang.cmd.gpython,
+# would import golang, and we need to do gevent monkey-patching ASAP - before that.
+#
+# we also keep it in gpython/__init__.py instead of gpython.py, since the latter does not
+# work correctly with `pip install` (gpython script is installed, but gpython module is not).
+
+# NOTE don't import anything at global scope - we need gevent to be imported first.
+from __future__ import print_function
+
+
+# pymain mimics `python ...`
+#
+# argv is what comes via `...` without first [0] for python.
+def pymain(argv):
+    import sys, code, runpy, six
+
+    # interactive console
+    if not argv:
+        code.interact()
+        return
+
+    # -c command
+    if argv[0] == '-c':
+        sys.argv = argv[0:1] + argv[2:] # python leaves '-c' as argv[0]
+
+        # exec with the same globals `python -c ...` does
+        g = {'__name__':    '__main__',
+             '__doc__':     None,
+             '__package__': None}
+        six.exec_(argv[1], g)
+
+    # -m module
+    elif argv[0] == '-m':
+        # search sys.path for module and run corresponding .py file as script
+        sys.argv = argv[1:]
+        runpy.run_module(sys.argv[0], init_globals={'__doc__': None}, run_name='__main__')
+
+    elif argv[0].startswith('-'):
+        print("unknown option: '%s'" % argv[0], file=sys.stderr)
+        sys.exit(2)
+
+    # file
+    else:
+        sys.argv = argv
+        filepath = argv[0]
+
+        # exec with same globals `python file.py` does
+        # XXX use runpy.run_path() instead?
+        g = {'__name__':    '__main__',
+             '__file__':    filepath,
+             '__doc__':     None,
+             '__package__': None}
+        _execfile(filepath, g)
+
+    return
+
+# execfile was removed in py3
+def _execfile(path, globals=None, locals=None):
+    import six
+    with open(path, "rb") as f:
+        src = f.read()
+    code = compile(src, path, 'exec')
+    six.exec_(code, globals, locals)
+
+
+def main():
+    # set UTF-8 as default encoding.
+    # It is ok to import sys before gevent because sys is anyway always
+    # imported first, e.g. to support sys.modules.
+    import sys
+    if sys.getdefaultencoding() != 'utf-8':
+        reload(sys)
+        sys.setdefaultencoding('utf-8')
+        delattr(sys, 'setdefaultencoding')
+
+    # safety check that we are not running from a setuptools entrypoint, where
+    # it would be too late to monkey-patch stdlib.
+    #
+    # (os and signal are imported by python startup itself)
+    # (on py3 _thread is imported by the interpreter early to support fine-grained import lock)
+    bad = []
+    for mod in ('pkg_resources', 'golang', 'socket', 'time', 'select',
+                'threading', 'thread', 'ssl', 'subprocess'):
+        if mod in sys.modules:
+            bad.append(mod)
+    if bad:
+        sysmodv = list(sys.modules.keys())
+        sysmodv.sort()
+        raise RuntimeError('gpython: internal error: the following modules are pre-imported, but must be not:'
+                '\n\n\t%s\n\nsys.modules:\n\n\t%s' % (bad, sysmodv))
+
+    # make gevent pre-available & stdlib patched
+    from gevent import monkey
+    _ = monkey.patch_all()      # XXX sys=True ?
+    if _ not in (True, None):   # patched or nothing to do
+        # XXX provide details
+        raise RuntimeError('gevent monkey-patching failed')
+
+    # put go, chan, select, ... into builtin namespace
+    import golang
+    from six.moves import builtins
+    for k in golang.__all__:
+        setattr(builtins, k, getattr(golang, k))
+
+    # sys.executable & friends
+    exe = sys.argv[0]
+
+    # on windows there are
+    #   gpython-script.py
+    #   gpython.exe
+    #   gpython.manifest
+    # and argv[0] is gpython-script.py
+    if exe.endswith('-script.py'):
+        exe = exe[:-len('-script.py')]
+        exe = exe + '.exe'
+
+    import sys
+    sys.executable  = exe
+    sys.version    += (' [GPython %s]' % golang.__version__)
+
+    # tail to pymain
+    pymain(sys.argv[1:])
+
+if __name__ == '__main__':
+    main()

gpython/gpython_test.py

+# -*- coding: utf-8 -*-
+# Copyright (C) 2019  Nexedi SA and Contributors.
+#                     Kirill Smelkov <kirr@nexedi.com>
+#
+# This program is free software: you can Use, Study, Modify and Redistribute
+# it under the terms of the GNU General Public License version 3, or (at your
+# option) any later version, as published by the Free Software Foundation.
+#
+# You can also Link and Combine this program with other software covered by
+# the terms of any of the Free Software licenses or any of the Open Source
+# Initiative approved licenses and Convey the resulting work. Corresponding
+# source of such a combination shall include the source code for all other
+# software used.
+#
+# This program is distributed WITHOUT ANY WARRANTY; without even the implied
+# warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
+#
+# See COPYING file for full licensing terms.
+# See https://www.nexedi.com/licensing for rationale and options.
+
+import sys, os, golang, subprocess
+from six import PY2, PY3
+from six.moves import builtins
+import pytest
+
+# @gpython_only is marker to run a test only under gpython
+gpython_only = pytest.mark.skipif('GPython' not in sys.version, reason="gpython-only test")
+
+
+@gpython_only
+def test_defaultencoding_utf8():
+    assert sys.getdefaultencoding() == 'utf-8'
+
+@gpython_only
+def test_golang_builtins():
+    # some direct accesses
+    assert go     is golang.go
+    assert chan   is golang.chan
+    assert select is golang.select
+
+    # indirectly verify golang.__all__
+    for k in golang.__all__:
+        assert getattr(builtins, k) is getattr(golang, k)
+
+@gpython_only
+def test_gevent_activated():
+    from gevent.monkey import is_module_patched as patched, is_object_patched as obj_patched
+
+    # builtin (gevent: only on py2 - on py3 __import__ uses fine-grained locking)
+    if PY2:
+        assert obj_patched('__builtin__', '__import__')
+
+    assert patched('socket')
+    # patch_socket(dns=True) also patches vvv
+    assert obj_patched('socket', 'getaddrinfo')
+    assert obj_patched('socket', 'gethostbyname')
+    # ...
+
+    assert patched('time')
+
+    assert patched('select')
+    import select as select_mod # patch_select(aggressive=True) removes vvv
+    assert not hasattr(select_mod, 'epoll')
+    assert not hasattr(select_mod, 'kqueue')
+    assert not hasattr(select_mod, 'kevent')
+    assert not hasattr(select_mod, 'devpoll')
+
+    # XXX on native windows, patch_{os,signal} do nothing currently
+    if os.name != 'nt':
+        assert patched('os')
+        assert patched('signal')
+
+    assert patched('thread' if PY2 else '_thread')
+    assert patched('threading')
+    assert patched('_threading_local')
+
+    assert patched('ssl')
+    assert patched('subprocess')
+    #assert patched('sys')       # currently disabled
+
+    if sys.hexversion >= 0x03070000: # >= 3.7.0
+        assert patched('queue')
+
+@gpython_only
+def test_executable():
+    # sys.executable must point to gpython and we must be able to execute it.
+    assert 'gpython' in sys.executable
+    out = subprocess.check_output([sys.executable, '-c', 'import sys; print(sys.version)'])
+    assert ('[GPython %s]' % golang.__version__) in str(out)
+
+# TODO test_pymain (it is not gpython_only)

setup.py

 # pygolang | pythonic package setup
+# Copyright (C) 2018-2019  Nexedi SA and Contributors.
+#                          Kirill Smelkov <kirr@nexedi.com>
+#
+# This program is free software: you can Use, Study, Modify and Redistribute
+# it under the terms of the GNU General Public License version 3, or (at your
+# option) any later version, as published by the Free Software Foundation.
+#
+# You can also Link and Combine this program with other software covered by
+# the terms of any of the Free Software licenses or any of the Open Source
+# Initiative approved licenses and Convey the resulting work. Corresponding
+# source of such a combination shall include the source code for all other
+# software used.
+#
+# This program is distributed WITHOUT ANY WARRANTY; without even the implied
+# warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
+#
+# See COPYING file for full licensing terms.
+# See https://www.nexedi.com/licensing for rationale and options.
 from setuptools import setup, find_packages
+from setuptools.command.install_scripts import install_scripts as _install_scripts
+from setuptools.command.develop import develop as _develop
 from os.path import dirname, join
-import re
+import sys, re
 
 # read file content
 def readfile(path):
@@ -22,6 +42,102 @@ _ = readfile(join(dirname(__file__), 'golang/__init__.py'))
 _ = grep1('^__version__ = "(.*)"$', _)
 version = _.group(1)
 
+# XInstallGPython customly installs bin/gpython.
+#
+# console_scripts generated by setuptools do lots of imports. However we need
+# gevent.monkey.patch_all() to be done first - before all other imports. We
+# could use plain scripts for gpython, however even for plain scripts
+# setuptools wants to inject pkg_resources import for develop install, and
+# pkg_resources does import lots of modules.
+#
+# -> generate the script via our custom install, but keep gpython listed as
+# console_scripts entry point, so that pip knows to remove the file on develop
+# uninstall.
+#
+# NOTE in some cases (see below e.g. about bdist_wheel) we accept for gpython
+# to be generated not via XInstallGPython - becuase in those cases pkg_resources
+# and entry points are not used - just plain `import gpython`.
+class XInstallGPython:
+    gpython_installed = 0
+
+    # NOTE cannot override write_script, because base class - _install_scripts
+    # or _develop, is old-style and super does not work with it.
+    #def write_script(self, script_name, script, mode="t", blockers=()):
+    #    script_name, script = self.transform_script(script_name, script)
+    #    super(XInstallGPython, self).write_script(script_name, script, mode, blockers)
+
+    # transform_script transform to-be installed script to override installed gpython content.
+    #
+    # (script_name, script) -> (script_name, script)
+    def transform_script(self, script_name, script):
+        # on windows setuptools installs 3 files:
+        #   gpython-script.py
+        #   gpython.exe
+        #   gpython.exe.manifest
+        # we want to override .py only.
+        #
+        # for-windows build could be cross - e.g. from linux via bdist_wininst -
+        # -> we can't rely on os.name. Rely on just script name.
+        if script_name in ('gpython', 'gpython-script.py'):
+            script  = '#!%s\n' % sys.executable
+            script += '\nfrom gpython import main; main()\n'
+            self.gpython_installed += 1
+
+        return script_name, script
+
+
+# install_scripts is custom scripts installer that takes gpython into account.
+class install_scripts(XInstallGPython, _install_scripts):
+    def write_script(self, script_name, script, mode="t", blockers=()):
+        script_name, script = self.transform_script(script_name, script)
+        _install_scripts.write_script(self, script_name, script, mode, blockers)
+
+    def run(self):
+        _install_scripts.run(self)
+        # bdist_wheel disables generation of scripts for entry-points[1]
+        # and pip/setuptools regenerate them when installing the wheel[2].
+        #
+        #   [1] https://github.com/pypa/wheel/commit/0d7f398b
+        #   [2] https://github.com/pypa/wheel/commit/9aaa6628
+        #
+        # since setup.py is not included into the wheel, we cannot control
+        # entry-point installation when the wheel is installed. However,
+        # console script generated when installing the wheel looks like:
+        #
+        #   #!/path/to/python
+        #   # -*- coding: utf-8 -*-
+        #   import re
+        #   import sys
+        #
+        #   from gpython import main
+        #
+        #   if __name__ == '__main__':
+        #       sys.argv[0] = re.sub(r'(-script\.pyw?|\.exe)?$', '', sys.argv[0])
+        #       sys.exit(main())
+        #
+        # which does not import pkg_resources. Since we also double-check in
+        # gpython itself that pkg_resources and other modules are not imported,
+        # we are ok with this.
+        if not self.no_ep:
+            # regular install
+            assert self.gpython_installed == 1
+        else:
+            # bdist_wheel
+            assert self.gpython_installed == 0
+            assert len(self.outfiles) == 0
+
+
+# develop, similarly to install_scripts, is used to handle gpython in `pip install -e` mode.
+class develop(XInstallGPython, _develop):
+    def write_script(self, script_name, script, mode="t", blockers=()):
+        script_name, script = self.transform_script(script_name, script)
+        _develop.write_script(self, script_name, script, mode, blockers)
+
+    def install_egg_scripts(self, dist):
+        _develop.install_egg_scripts(self, dist)
+        assert self.gpython_installed == 1
+
+
 setup(
     name        = 'pygolang',
     version     = version,
@@ -38,17 +154,25 @@ setup(
     packages    = find_packages(),
     include_package_data = True,
 
-    install_requires = ['six', 'decorator'],
+    install_requires = ['gevent', 'six', 'decorator'],
 
     extras_require = {
                   'test': ['pytest'],
     },
 
     entry_points= {'console_scripts': [
+                        # NOTE gpython is handled specially - see XInstallGPython.
+                        'gpython  = gpython:main',
+
                         'py.bench = golang.cmd.pybench:main',
                       ]
                   },
 
+    cmdclass    = {
+        'install_scripts':  install_scripts,
+        'develop':          develop,
+    },
+
     classifiers = [_.strip() for _ in """\
         Development Status :: 3 - Alpha
         Intended Audience :: Developers

tox.ini

 [tox]
 # TODO pypy -> pypy2, pypy3 once pypy3 lands in Debian
 # https://bugs.debian.org/825970
-envlist = py27, py36, py37, pypy
+envlist = {py27,py36,py37,pypy}-{thread,gevent}
 
 [testenv]
 deps    =
     # why tox does not get it from extras_require['test'] ?
     pytest
 
-commands= {envpython} -m pytest
+# gpython pre-imports installed golang, will get into conflict with
+# golang/ if we run pytest from pygolang worktree. Avoid that.
+changedir = {envsitepackagesdir}
+
+commands=
+    thread: {envpython} -m pytest gpython/ golang/
+    gevent: gpython     -m pytest gpython/ golang/