gpython: Run code in carefully prepared __main__ module

Because else sys.modules['__main__'] points to the module of gpython
itself, not user program. This difference can be important when e.g.
user defines classes in main module and tries to pickle them. Such
pickling will fail if gpython is not careful to run user's code in its
own main.

Without this patch added test fails as

      File "check_main.py", line 51, in <module>
        main()
      File "check_main.py", line 39, in main
        assert hasattr(mainmod, 'MyUniqueClassXYZ'),        dir(mainmod)
    AssertionError: ['__builtins__', '__doc__', '__file__', '__name__', '__package__', 'main']

The problem was discovered while trying to run test program from
https://github.com/zopefoundation/ZEO/issues/166 :

    $ gpython zopenrace-zeo.py
    Traceback (most recent call last):
      File "/home/kirr/tmp/trashme/Z/py2.venv/bin/gpython", line 8, in <module>
        sys.exit(main())
      File "/home/kirr/tmp/trashme/Z/py2.venv/lib/python2.7/site-packages/gpython/__init__.py", line 395, in main
        pymain(argv, init)
      File "/home/kirr/tmp/trashme/Z/py2.venv/lib/python2.7/site-packages/gpython/__init__.py", line 266, in pymain
        run()
      File "/home/kirr/tmp/trashme/Z/py2.venv/lib/python2.7/site-packages/gpython/__init__.py", line 172, in run
        _execfile(filepath, g)
      File "/home/kirr/tmp/trashme/Z/py2.venv/lib/python2.7/site-packages/gpython/__init__.py", line 275, in _execfile
        six.exec_(code, globals, locals)
      File "/home/kirr/tmp/trashme/Z/py2.venv/lib/python2.7/site-packages/six.py", line 719, in exec_
        exec("""exec _code_ in _globs_, _locs_""")
      File "<string>", line 1, in <module>
      File "zopenrace-zeo.py", line 205, in <module>
        main()
      File "zopenrace-zeo.py", line 190, in main
        init()
      File "zopenrace-zeo.py", line 126, in init
        transaction.commit()
      File "/home/kirr/tmp/trashme/Z/py2.venv/lib/python2.7/site-packages/transaction/_manager.py", line 257, in commit
        return self.manager.commit()
      File "/home/kirr/tmp/trashme/Z/py2.venv/lib/python2.7/site-packages/transaction/_manager.py", line 134, in commit
        return self.get().commit()
      File "/home/kirr/tmp/trashme/Z/py2.venv/lib/python2.7/site-packages/transaction/_transaction.py", line 282, in commit
        reraise(t, v, tb)
      File "/home/kirr/tmp/trashme/Z/py2.venv/lib/python2.7/site-packages/transaction/_transaction.py", line 273, in commit
        self._commitResources()
      File "/home/kirr/tmp/trashme/Z/py2.venv/lib/python2.7/site-packages/transaction/_transaction.py", line 456, in _commitResources
        reraise(t, v, tb)
      File "/home/kirr/tmp/trashme/Z/py2.venv/lib/python2.7/site-packages/transaction/_transaction.py", line 430, in _commitResources
        rm.commit(self)
      File "/home/kirr/tmp/trashme/Z/py2.venv/lib/python2.7/site-packages/ZODB/Connection.py", line 497, in commit
        self._commit(transaction)
      File "/home/kirr/tmp/trashme/Z/py2.venv/lib/python2.7/site-packages/ZODB/Connection.py", line 546, in _commit
        self._store_objects(ObjectWriter(obj), transaction)
      File "/home/kirr/tmp/trashme/Z/py2.venv/lib/python2.7/site-packages/ZODB/Connection.py", line 578, in _store_objects
        p = writer.serialize(obj)  # This calls __getstate__ of obj
      File "/home/kirr/tmp/trashme/Z/py2.venv/lib/python2.7/site-packages/ZODB/serialize.py", line 430, in serialize
        return self._dump(meta, obj.__getstate__())
      File "/home/kirr/tmp/trashme/Z/py2.venv/lib/python2.7/site-packages/ZODB/serialize.py", line 439, in _dump
        self._p.dump(state)
    PicklingError: Can't pickle <class '__main__.PInt'>: attribute lookup __main__.PInt failed

/reviewed-by @jerome
/reviewed-on !15

gpython/__init__.py

@@ -120,13 +120,9 @@ def pymain(argv, init=None):
             cmd = arg
             sys.argv = ['-c'] + igetopt.argv # python leaves '-c' as argv[0]
             sys.path.insert(0, '')   # cwd
-            def run():
+            def run(mmain):
                 import six
-                # exec with the same globals `python -c ...` does
-                g = {'__name__':    '__main__',
-                     '__doc__':     None,
-                     '__package__': None}
-                six.exec_(cmd, g)
+                six.exec_(cmd, mmain.__dict__)
             break
 
         # -m module
@@ -138,11 +134,13 @@ def pymain(argv, init=None):
             # we stick to python3 behaviour, as it is more sane because e.g.
             # import path does not change after chdir.
             sys.path.insert(0, realpath(''))  # realpath(cwd)
-            def run():
+            def run(mmain):
                 import runpy
                 # search sys.path for module and run corresponding .py file as script
-                runpy.run_module(mod, init_globals={'__doc__': None},
-                                 run_name='__main__', alter_sys=True)
+                # NOTE runpy._run_module_as_main works on sys.modules['__main__']
+                sysmain = sys.modules['__main__']
+                assert sysmain is mmain,  (sysmain, mmain)
+                runpy._run_module_as_main(mod)
             break
 
         # -W arg  (warning control)
@@ -166,14 +164,9 @@ def pymain(argv, init=None):
                 filepath = realpath(filepath)
 
             sys.path.insert(0, realpath(dirname(filepath))) # not abspath -> see PySys_SetArgvEx
-            def run():
-                # exec with same globals `python file.py` does
-                # XXX use runpy.run_path() instead?
-                g = {'__name__':    '__main__',
-                     '__file__':    filepath,
-                     '__doc__':     None,
-                     '__package__': None}
-                _execfile(filepath, g)
+            def run(mmain):
+                mmain.__file__ = filepath
+                _execfile(filepath, mmain.__dict__)
 
         # interactive console / program on non-tty stdin
         else:
@@ -181,15 +174,15 @@ def pymain(argv, init=None):
             sys.path.insert(0, '')  # cwd
 
             if sys.stdin.isatty():
-                def run():
-                    _interact()
+                def run(mmain):
+                    mmain.__file__ = '<stdin>'
+                    _interact(mmain)
             else:
-                def run():
+                def run(mmain):
                     import six
                     prog = sys.stdin.read()
-                    g = {'__name__': '__main__',
-                         '__file__': '<stdin>'}
-                    six.exec_(prog, g)
+                    mmain.__file__ = '<stdin>'
+                    six.exec_(prog, mmain.__dict__)
 
 
     # ---- options processed -> start the interpreter ----
@@ -256,12 +249,21 @@ def pymain(argv, init=None):
         sys.warnoptions += warnoptions
         warnings._processoptions(warnoptions)
 
+    # inject new empty __main__ module instead of previous __main__
+    import types
+    mmain = types.ModuleType('__main__')
+    mmain.__file__    = None
+    mmain.__loader__  = None
+    mmain.__package__ = None
+    mmain.__doc__     = None
+    sys.modules['__main__'] = mmain
+
     # execute -m/-c/file/interactive
-    run()
+    run(mmain)
 
 
-# _interact runs interactive console.
-def _interact():
+# _interact runs interactive console in mmain namespace.
+def _interact(mmain):
     import code, sys
     from six.moves import input as raw_input
     # like code.interact() but with overridden console.raw_input _and_
@@ -271,7 +273,7 @@ def _interact():
     except ImportError:
         pass
 
-    console = code.InteractiveConsole()
+    console = code.InteractiveConsole(mmain.__dict__)
     def _(prompt):
         # python behaviour:
         # - use stdout for prompt by default;

gpython/gpython_test.py

@@ -21,7 +21,7 @@
 from __future__ import print_function, absolute_import
 
 import sys, os, platform, re, golang
-from golang.golang_test import pyout, _pyrun
+from golang.golang_test import pyout, pyrun, _pyrun, readfile
 from subprocess import PIPE
 from six import PY2
 from six.moves import builtins
@@ -224,6 +224,23 @@ def test_pymain_print_function_future():
     _ = pyout(['future_print_function.py'], cwd=testprog)
     assert _ == b"print is a function with print_function future\n"
 
+
+# verify thay pymain runs programs with __main__ module correctly setup.
+def test_pymain__main__():
+    from golang import b
+    check_main_py = readfile('%s/check_main.py' % testprog)
+
+    pyrun(['testprog/check_main.py'], cwd=here) # file
+    pyrun(['-m', 'check_main'], cwd=testprog)   # -m
+    pyrun(['-c', check_main_py])                # -c
+
+    # stdin
+    ret, out, err = _pyrun([], stdin=b(check_main_py), stdout=PIPE, stderr=PIPE)
+    assert ret == 0,    (out, err)
+    assert b"Error" not in out,    (out, err)
+    assert b"Error" not in err,    (out, err)
+
+
 # verify that pymain sets sys.path in exactly the same way as underlying python does.
 @gpython_only
 def test_pymain_syspath():

gpython/testprog/check_main.py

+# -*- coding: utf-8 -*-
+# Copyright (C) 2021  Nexedi SA and Contributors.
+#                     Kirill Smelkov <kirr@nexedi.com>
+#
+# This program is free software: you can Use, Study, Modify and Redistribute
+# it under the terms of the GNU General Public License version 3, or (at your
+# option) any later version, as published by the Free Software Foundation.
+#
+# You can also Link and Combine this program with other software covered by
+# the terms of any of the Free Software licenses or any of the Open Source
+# Initiative approved licenses and Convey the resulting work. Corresponding
+# source of such a combination shall include the source code for all other
+# software used.
+#
+# This program is distributed WITHOUT ANY WARRANTY; without even the implied
+# warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
+#
+# See COPYING file for full licensing terms.
+# See https://www.nexedi.com/licensing for rationale and options.
+"""Program check_main verifies that __main__ module is correctly installed on
+program run."""
+
+from __future__ import print_function, absolute_import
+
+import sys, pickle
+
+class MyUniqueClassXYZ(object):
+    def __init__(self, value):
+        self.value = value
+
+def main():
+    assert MyUniqueClassXYZ.__module__ == '__main__',   MyUniqueClassXYZ.__module__
+    assert '__main__' in sys.modules,                   sys.modules
+    mainmod  = sys.modules['__main__']
+    mainmod_ = __import__('__main__')
+    assert mainmod is mainmod_,                         (mainmod, mainmod_)
+
+    # verify that mainmod actually refers to current module
+    assert hasattr(mainmod, 'MyUniqueClassXYZ'),        dir(mainmod)
+
+    # pickle/unpickle would also fail if import('__main__') gives module different from current
+    obj = MyUniqueClassXYZ(123)
+    s = pickle.dumps(obj)
+    obj_ = pickle.loads(s)
+    assert type(obj_) is MyUniqueClassXYZ,              type(obj)
+    assert obj_.value == 123,                           obj_.value
+
+    # ok
+
+assert __name__ == '__main__',  __name__
+main()