#!/usr/bin/env python
# -*- coding: utf-8 -*-
# Copyright (C) 2018-2021 Nexedi SA and Contributors.
# Kirill Smelkov <kirr@nexedi.com>
#
# This program is free software: you can Use, Study, Modify and Redistribute
# it under the terms of the GNU General Public License version 3, or (at your
# option) any later version, as published by the Free Software Foundation.
#
# You can also Link and Combine this program with other software covered by
# the terms of any of the Free Software licenses or any of the Open Source
# Initiative approved licenses and Convey the resulting work. Corresponding
# source of such a combination shall include the source code for all other
# software used.
#
# This program is distributed WITHOUT ANY WARRANTY; without even the implied
# warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
#
# See COPYING file for full licensing terms.
# See https://www.nexedi.com/licensing for rationale and options.
"""gpython ... - run python ... with gevent & golang activated.
gpython is substitute for standard python interpreter with the following
differences:
- gevent is pre-activated and stdlib is patched to be gevent aware;
- go, chan, select etc are put into builtin namespace;
- default string encoding is always set to UTF-8.
Gevent activation can be disabled via `-X gpython.runtime=threads`, or
$GPYTHON_RUNTIME=threads.
"""
# NOTE gpython is kept out of golang/ , since even just importing e.g. golang.cmd.gpython,
# would import golang, and we need to do gevent monkey-patching ASAP - before that.
#
# we also keep it in gpython/__init__.py instead of gpython.py, since the latter does not
# work correctly with `pip install` (gpython script is installed, but gpython module is not).
# NOTE don't import anything at global scope - we need gevent to be imported first.
from __future__ import print_function, absolute_import
_pyopt = "c:m:OVW:X:"
_pyopt_long = ('version',)
# pymain mimics `python ...`
#
# argv is full argument vector including first [0] for path to main program itself.
# init, if provided, is called after options are parsed, but before interpreter start.
def pymain(argv, init=None):
import sys
from os.path import dirname, realpath
# sys.executable
# on windows there are
# gpython-script.py
# gpython.exe
# gpython.manifest
# and argv[0] is gpython-script.py
exe = realpath(argv[0])
argv = argv[1:]
if exe.endswith('-script.py'):
exe = exe[:-len('-script.py')]
exe = exe + '.exe'
sys._gpy_underlying_executable = sys.executable
sys.executable = exe
# `python /path/to/gpython` adds /path/to to sys.path[0] - remove it.
# `gpython file` will add path-to-file to sys.path[0] by itself, and
# /path/to/gpython is unneccessary and would create difference in behaviour
# in between gpython and python.
exedir = dirname(exe)
if sys.path[0] == exedir:
del sys.path[0]
else:
# buildout injects `sys.path[0:0] = eggs` into python scripts.
# detect that and remove sys.path entry corresponding to exedir.
if not _is_buildout_script(exe):
raise RuntimeError('pymain: internal error: sys.path[0] was not set by underlying python to dirname(exe):'
'\n\n\texe:\t%s\n\tsys.path[0]:\t%s' % (exe, sys.path[0]))
else:
if exedir in sys.path:
sys.path.remove(exedir)
else:
raise RuntimeError('pymain: internal error: sys.path does not contain dirname(exe):'
'\n\n\texe:\t%s\n\tsys.path:\t%s' % (exe, sys.path))
run = None # function to run according to -c/-m/file/stdin/interactive
version = False # set if `-V`
warnoptions = [] # collected `-W arg`
reexec_with = [] # reexecute underlying python with those options (e.g. -O, -S, ...)
reexec_argv = [] # if reexecuting, reexecute with this application-level argv
igetopt = _IGetOpt(argv, _pyopt, _pyopt_long)
for (opt, arg) in igetopt:
# options that require reexecuting through underlying python with that -<opt>
if opt in (
'-O', # optimize
):
reexec_with.append(opt)
if arg is not None:
reexec_with.append(arg)
continue
reexec_argv.append(opt)
if arg is not None:
reexec_argv.append(arg)
# -V / --version
if opt in ('-V', '--version'):
version = True
break
# -c command
elif opt == '-c':
cmd = arg
sys.argv = ['-c'] + igetopt.argv # python leaves '-c' as argv[0]
sys.path.insert(0, '') # cwd
def run():
import six
# exec with the same globals `python -c ...` does
g = {'__name__': '__main__',
'__doc__': None,
'__package__': None}
six.exec_(cmd, g)
break
# -m module
elif opt == '-m':
mod = arg
sys.argv = [mod] + igetopt.argv
# sys.path <- cwd
# NOTE python2 injects '', while python3 injects realpath('')
# we stick to python3 behaviour, as it is more sane because e.g.
# import path does not change after chdir.
sys.path.insert(0, realpath('')) # realpath(cwd)
def run():
import runpy
# search sys.path for module and run corresponding .py file as script
runpy.run_module(mod, init_globals={'__doc__': None},
run_name='__main__', alter_sys=True)
break
# -W arg (warning control)
elif opt == '-W':
warnoptions.append(arg)
else:
print("unknown option: '%s'" % opt, file=sys.stderr)
sys.exit(2)
argv = igetopt.argv
reexec_argv += argv
if run is None:
# file
if len(argv) > 0 and argv[0] != '-':
sys.argv = argv
filepath = argv[0]
# starting from cpython 3.9 __file__ is always absolute
# https://bugs.python.org/issue20443
if sys.version_info >= (3, 9):
filepath = realpath(filepath)
sys.path.insert(0, realpath(dirname(filepath))) # not abspath -> see PySys_SetArgvEx
def run():
# exec with same globals `python file.py` does
# XXX use runpy.run_path() instead?
g = {'__name__': '__main__',
'__file__': filepath,
'__doc__': None,
'__package__': None}
_execfile(filepath, g)
# interactive console / program on non-tty stdin
else:
sys.argv = [''] if len(argv) == 0 else argv # e.g. ['-']
sys.path.insert(0, '') # cwd
if sys.stdin.isatty():
def run():
_interact()
else:
def run():
import six
prog = sys.stdin.read()
g = {'__name__': '__main__',
'__file__': '<stdin>'}
six.exec_(prog, g)
# ---- options processed -> start the interpreter ----
# reexec underlying interpreter on options that we cannot handle at python
# level after underlying interpreter is already started. For example
#
# gpython -O file.py
#
# is reexecuted as
#
# python -O gpython file.py
if len(reexec_with) > 0:
import os
argv = [sys._gpy_underlying_executable] + reexec_with + [sys.executable] + reexec_argv
os.execv(argv[0], argv)
if init is not None:
init()
# handle -V/--version
if version:
ver = []
if 'GPython' in sys.version:
golang = sys.modules['golang'] # must be already imported
gevent = sys.modules.get('gevent', None)
gpyver = 'GPython %s' % golang.__version__
if gevent is not None:
gpyver += ' [gevent %s]' % gevent.__version__
else:
gpyver += ' [threads]'
ver.append(gpyver)
import platform
pyimpl = platform.python_implementation()
v = _version_info_str
if pyimpl == 'CPython':
ver.append('CPython %s' % v(sys.version_info))
elif pyimpl == 'PyPy':
ver.append('PyPy %s' % v(sys.pypy_version_info))
ver.append('Python %s' % v(sys.version_info))
else:
ver = [] # unknown
ver = ' / '.join(ver)
if ver == '':
# unknown implementation: just print full sys.version
ver = sys.version
print(ver, file=sys.stderr)
return
# init warnings
if len(warnoptions) > 0:
# NOTE warnings might be already imported by code that calls pymain.
# This way we cannot set `sys.warnoptions = warnoptions` and just
# import/reload warnings (if we reload warnings, it will loose all
# previous setup that pymain caller might have done to it).
# -> only amend warnings setup
#
# NOTE $PYTHONWARNINGS is handled by underlying python natively.
import warnings
sys.warnoptions += warnoptions
warnings._processoptions(warnoptions)
# execute -m/-c/file/interactive
run()
# _interact runs interactive console.
def _interact():
import code, sys
from six.moves import input as raw_input
# like code.interact() but with overridden console.raw_input _and_
# readline imported (code.interact mutually excludes those two).
try:
import readline # enable interactive editing
except ImportError:
pass
console = code.InteractiveConsole()
def _(prompt):
# python behaviour:
# - use stdout for prompt by default;
# - use stderr for prompt if any of stdin/stderr is not a tty
# (builtin raw_input always prints prompt)
promptio = sys.stdout
if (not sys.stdin.isatty()) or (not sys.stdout.isatty()):
promptio = sys.stderr
promptio.write(prompt)
promptio.flush()
return raw_input('')
console.raw_input = _
console.interact()
# execfile was removed in py3
def _execfile(path, globals=None, locals=None):
import six
with open(path, "rb") as f:
src = f.read()
code = compile(src, path, 'exec', dont_inherit=True)
six.exec_(code, globals, locals)
# _version_info_str converts version_info -> str.
def _version_info_str(vi):
major, minor, micro, release, serial = vi
v = '%d.%d.%d' % (major, minor, micro)
if (release, serial) != ('final', 0):
v += '.%s%s' % (release, serial)
return v
def main():
# import sys early.
# it is ok to import sys before gevent because sys is anyway always
# imported first, e.g. to support sys.modules.
import sys
# safety check that we are not running from a setuptools entrypoint, where
# it would be too late to monkey-patch stdlib.
#
# (os and signal are imported by python startup itself)
# (on py3 _thread is imported by the interpreter early to support fine-grained import lock)
avoid = ['pkg_resources', 'golang', 'socket', 'select', 'threading',
'thread', 'ssl', 'subprocess']
# pypy7 made time always pre-imported (https://foss.heptapod.net/pypy/pypy/-/commit/f4fa167b)
# cpython3.8 made time always pre-imported via zipimport hook:
# https://github.com/python/cpython/commit/79d1c2e6c9d1 (`import time` in zipimport.py)
pypy = ('PyPy' in sys.version)
if (not pypy) and (sys.version_info < (3, 8)):
avoid.append('time')
bad = []
for mod in avoid:
if mod in sys.modules:
bad.append(mod)
if bad:
sysmodv = list(sys.modules.keys())
sysmodv.sort()
raise RuntimeError('gpython: internal error: the following modules are pre-imported, but must be not:'
'\n\n\t%s\n\nsys.modules:\n\n\t%s' % (bad, sysmodv))
# set UTF-8 as default encoding.
if sys.getdefaultencoding() != 'utf-8':
reload(sys)
sys.setdefaultencoding('utf-8')
delattr(sys, 'setdefaultencoding')
# import os to get access to environment.
# it is practically ok to import os before gevent, because os is always
# imported by site. Yes, `import site` can be disabled by -S, but there is
# no harm wrt gevent monkey-patching even if we import os first.
import os
# extract and process `-X gpython.*`
# -X gpython.runtime=(gevent|threads) + $GPYTHON_RUNTIME
sys._xoptions = getattr(sys, '_xoptions', {})
argv_ = []
gpy_runtime = os.getenv('GPYTHON_RUNTIME', 'gevent')
igetopt = _IGetOpt(sys.argv[1:], _pyopt, _pyopt_long)
for (opt, arg) in igetopt:
if opt == '-X':
if arg.startswith('gpython.'):
if arg.startswith('gpython.runtime='):
gpy_runtime = arg[len('gpython.runtime='):]
sys._xoptions['gpython.runtime'] = gpy_runtime
else:
raise RuntimeError('gpython: unknown -X option %s' % opt)
continue
argv_.append(opt)
if arg is not None:
argv_.append(arg)
# options after -c / -m are not for python itself
if opt in ('-c', '-m'):
break
argv = [sys.argv[0]] + argv_ + igetopt.argv
# init initializes according to selected runtime
# it is called after options are parsed and sys.path is setup correspondingly.
# this way golang and gevent are imported from exactly the same place as
# they would be in standard python after regular import (ex from golang/
# under cwd if run under `python -c ...` or interactive console.
def init():
if gpy_runtime == 'gevent':
# make gevent pre-available & stdlib patched
import gevent
from gevent import monkey
# XXX workaround for gevent vs pypy2 crash.
# XXX remove when gevent-1.4.1 is relased (https://github.com/gevent/gevent/pull/1357).
patch_thread=True
if pypy and sys.version_info.major == 2:
_ = monkey.patch_thread(existing_locks=False)
assert _ in (True, None)
patch_thread=False
_ = monkey.patch_all(thread=patch_thread) # XXX sys=True ?
if _ not in (True, None): # patched or nothing to do
# XXX provide details
raise RuntimeError('gevent monkey-patching failed')
gpy_verextra = 'gevent %s' % gevent.__version__
elif gpy_runtime == 'threads':
gpy_verextra = 'threads'
else:
raise RuntimeError('gpython: invalid runtime %s' % gpy_runtime)
# put go, chan, select, ... into builtin namespace
import golang
from six.moves import builtins
for k in golang.__all__:
setattr(builtins, k, getattr(golang, k))
# sys.version
sys.version += (' [GPython %s] [%s]' % (golang.__version__, gpy_verextra))
# tail to pymain
pymain(argv, init)
# _is_buildout_script returns whether file @path is generated as python buildout script.
def _is_buildout_script(path):
with open(path, 'r') as f:
src = f.read()
# buildout injects the following prologues into python scripts:
# sys.path[0:0] = [
# ...
# ]
return ('\nsys.path[0:0] = [\n' in src)
# _IGetOpt provides getopt-style incremental options parsing.
# ( we cannot use getopt directly, because it complains about "unrecognized options"
# on e.g. `gpython file.py -opt` )
class _IGetOpt:
def __init__(self, argv, shortopts, longopts):
self.argv = argv
self._opts = {} # opt -> bool(arg-required)
self._shortopttail = '' # current tail of short options from e.g. -abcd
# parse shortopts -> ._opts
opt = None
for _ in shortopts:
if _ == ':':
if opt is None:
raise RuntimeError("invalid shortopts: unexpected ':'")
self._opts['-'+opt] = True
opt = None # prevent ::
else:
opt = _
if opt in self._opts:
raise RuntimeError("invalid shortopts: double '%s'" % opt)
self._opts['-'+opt] = False
# parse longopts -> ._opts
for opt in longopts:
arg_required = (opt[-1:] == '=')
if arg_required:
opt = opt[:-1]
self._opts['--'+opt] = arg_required
def __iter__(self):
return self
def __next__(self):
# yield e.g. -b -c -d from -abcd
if len(self._shortopttail) > 0:
opt = '-'+self._shortopttail[0]
self._shortopttail = self._shortopttail[1:]
if opt not in self._opts:
raise RuntimeError('unexpected option %s' % opt)
arg = None
if self._opts[opt]: # arg required
if len(self._shortopttail) > 0:
# -o<arg>
arg = self._shortopttail
self._shortopttail = ''
else:
# -o <arg>
if len(self.argv) == 0:
raise RuntimeError('option %s requires an argument' % opt)
arg = self.argv[0]
self.argv = self.argv[1:]
return (opt, arg)
# ._shortopttail is empty - proceed with .argv
if len(self.argv) == 0:
raise StopIteration # end of argv
opt = self.argv[0]
if not opt.startswith('-'):
raise StopIteration # not an option
if opt == '-':
raise StopIteration # not an option
self.argv = self.argv[1:]
if opt == '--':
raise StopIteration # options -- args delimiter
# short option
if not opt.startswith('--'):
self._shortopttail = opt[1:]
return self.__next__()
# long option
arg = None
if '=' in opt:
opt, arg = opt.split('=')
if opt not in self._opts:
raise RuntimeError('unexpected option %s' % opt)
arg_required = self._opts[opt]
if not arg_required:
if arg is not None:
raise RuntimeError('option %s requires no argument' % opt)
else:
if arg is None:
if len(self.argv) == 0:
raise RuntimeError('option %s requires no argument' % opt)
arg = self.argv[0]
self.argv[0] = self.argv[1:]
return (opt, arg)
next = __next__ # for py2
if __name__ == '__main__':
main()