cleanup the eval and exec implementation

This makes the implementation more similar to cpythons and removes some missing error handling.
It also uses and implements some missing PyRun_* and PyEval_* functions.
One behaviour change is that 'execfile' now always uses the cpython parser.

If think if we want to support other parsers the right way would be to decide inside PyParser_ASTFromFile which parser to use.

from_cpython/Include/Python.h

@@ -132,6 +132,7 @@ PyAPI_FUNC(void) PyType_SetDict(PyTypeObject*, PyObject*) PYSTON_NOEXCEPT;
 #include "abstract.h"
 
 #include "compile.h"
+#include "eval.h"
 
 #include "pyctype.h"
 #include "pystrtod.h"

from_cpython/Include/compile.h

@@ -10,7 +10,7 @@ extern "C" {
 
 /* Public interface */
 struct _node; /* Declare the existence of this type */
-PyAPI_FUNC(PyCodeObject *) PyNode_Compile(struct _node *, const char *);
+PyAPI_FUNC(PyCodeObject *) PyNode_Compile(struct _node *, const char *) PYSTON_NOEXCEPT;
 
 /* Future feature support */
 
@@ -30,8 +30,8 @@ typedef struct {
 
 struct _mod; /* Declare the existence of this type */
 PyAPI_FUNC(PyCodeObject *) PyAST_Compile(struct _mod *, const char *,
-					PyCompilerFlags *, PyArena *);
-PyAPI_FUNC(PyFutureFeatures *) PyFuture_FromAST(struct _mod *, const char *);
+                    PyCompilerFlags *, PyArena *) PYSTON_NOEXCEPT;
+PyAPI_FUNC(PyFutureFeatures *) PyFuture_FromAST(struct _mod *, const char *) PYSTON_NOEXCEPT;
 
 
 #ifdef __cplusplus

from_cpython/Include/eval.h

@@ -7,7 +7,7 @@
 extern "C" {
 #endif
 
-PyAPI_FUNC(PyObject *) PyEval_EvalCode(PyCodeObject *, PyObject *, PyObject *);
+PyAPI_FUNC(PyObject *) PyEval_EvalCode(PyCodeObject *, PyObject *, PyObject *) PYSTON_NOEXCEPT;
 
 PyAPI_FUNC(PyObject *) PyEval_EvalCodeEx(PyCodeObject *co,
 					PyObject *globals,
@@ -15,9 +15,9 @@ PyAPI_FUNC(PyObject *) PyEval_EvalCodeEx(PyCodeObject *co,
 					PyObject **args, int argc,
 					PyObject **kwds, int kwdc,
 					PyObject **defs, int defc,
-					PyObject *closure);
+                    PyObject *closure) PYSTON_NOEXCEPT;
 
-PyAPI_FUNC(PyObject *) _PyEval_CallTracing(PyObject *func, PyObject *args);
+PyAPI_FUNC(PyObject *) _PyEval_CallTracing(PyObject *func, PyObject *args) PYSTON_NOEXCEPT;
 
 #ifdef __cplusplus
 }

from_cpython/Lib/test/test_univnewlines2k.py

-# expected: fail
 # Tests universal newline support for both reading and parsing files.
 import unittest
 import os

src/codegen/irgen/hooks.h

@@ -40,9 +40,6 @@ void compileAndRunModule(AST_Module* m, BoxedModule* bm);
 CompiledFunction* cfForMachineFunctionName(const std::string&);
 
 extern "C" Box* exec(Box* boxedCode, Box* globals, Box* locals, FutureFlags caller_future_flags);
-extern "C" Box* execfile(Box* fn, Box* globals, Box* locals);
-extern "C" Box* eval(Box* boxedCode, Box* globals, Box* locals);
-extern "C" Box* compile(Box* source, Box* filename, Box* mode, Box** _args /* flags, dont_inherit */);
 }
 
 #endif

src/runtime/capi.cpp

@@ -1184,6 +1184,76 @@ extern "C" mod_ty PyParser_ASTFromFile(FILE* fp, const char* filename, int start
     }
 }
 
+extern "C" PyObject* Py_CompileStringFlags(const char* str, const char* filename, int start,
+                                           PyCompilerFlags* flags) noexcept {
+    PyCodeObject* co;
+    mod_ty mod;
+    PyArena* arena = PyArena_New();
+    if (arena == NULL)
+        return NULL;
+
+    mod = PyParser_ASTFromString(str, filename, start, flags, arena);
+    if (mod == NULL) {
+        PyArena_Free(arena);
+        return NULL;
+    }
+    if (flags && (flags->cf_flags & PyCF_ONLY_AST)) {
+        PyObject* result = PyAST_mod2obj(mod);
+        PyArena_Free(arena);
+        return result;
+    }
+    co = PyAST_Compile(mod, filename, flags, arena);
+    PyArena_Free(arena);
+    return (PyObject*)co;
+}
+
+static PyObject* run_mod(mod_ty mod, const char* filename, PyObject* globals, PyObject* locals, PyCompilerFlags* flags,
+                         PyArena* arena) noexcept {
+    PyCodeObject* co;
+    PyObject* v;
+    co = PyAST_Compile(mod, filename, flags, arena);
+    if (co == NULL)
+        return NULL;
+    v = PyEval_EvalCode(co, globals, locals);
+    Py_DECREF(co);
+    return v;
+}
+
+extern "C" PyObject* PyRun_FileExFlags(FILE* fp, const char* filename, int start, PyObject* globals, PyObject* locals,
+                                       int closeit, PyCompilerFlags* flags) noexcept {
+    PyObject* ret;
+    mod_ty mod;
+    PyArena* arena = PyArena_New();
+    if (arena == NULL)
+        return NULL;
+
+    mod = PyParser_ASTFromFile(fp, filename, start, 0, 0, flags, NULL, arena);
+    if (closeit)
+        fclose(fp);
+    if (mod == NULL) {
+        PyArena_Free(arena);
+        return NULL;
+    }
+    ret = run_mod(mod, filename, globals, locals, flags, arena);
+    PyArena_Free(arena);
+    return ret;
+}
+
+extern "C" PyObject* PyRun_StringFlags(const char* str, int start, PyObject* globals, PyObject* locals,
+                                       PyCompilerFlags* flags) noexcept {
+    PyObject* ret = NULL;
+    mod_ty mod;
+    PyArena* arena = PyArena_New();
+    if (arena == NULL)
+        return NULL;
+
+    mod = PyParser_ASTFromString(str, "<string>", start, flags, arena);
+    if (mod != NULL)
+        ret = run_mod(mod, "<string>", globals, locals, flags, arena);
+    PyArena_Free(arena);
+    return ret;
+}
+
 extern "C" int PyRun_InteractiveLoopFlags(FILE* fp, const char* filename, PyCompilerFlags* flags) noexcept {
     PyObject* v;
     int ret;

test/CPYTHON_TEST_NOTES.md

@@ -197,7 +197,6 @@ test_unicode            argument passing issue?
 test_unicodedata        [unknown]
 test_unicode_file       exit code 139, no error message
 test_unittest           serialize_ast assert
-test_univnewlines2k     [unknown]
 test_univnewlines       [unknown]
 test_userdict           segfault: repr of recursive dict?
 test_userlist           slice(1L, 1L)

test/extra/cheetah_test.py

@@ -9,5 +9,5 @@ create_virtenv(ENV_NAME, ["cheetah==2.4.4", "Markdown==2.0.1"], force_create = T
 cheetah_exe = os.path.join(ENV_NAME, "bin", "cheetah")
 env = os.environ
 env["PATH"] = os.path.join(ENV_NAME, "bin")
-expected = [{'ran': 2138, 'errors': 4, 'failures': 1}, {'ran': 2138, 'errors': 232, 'failures': 3}]
+expected = [{'ran': 2138, 'errors': 4}, {'ran': 2138, 'errors': 232, 'failures': 2}]
 run_test([cheetah_exe, "test"], cwd=ENV_NAME, expected=expected, env=env)

test/extra/geoip_test.py

@@ -7,7 +7,7 @@ ENV_NAME = "geoip_test_env_" + os.path.basename(sys.executable)
 SRC_DIR = os.path.abspath(os.path.join(ENV_NAME, "src"))
 PYTHON_EXE = os.path.abspath(os.path.join(ENV_NAME, "bin", "python"))
 
-pkg = ["-e", "git+https://github.com/maxmind/geoip-api-python.git@v1.3.2#egg=GeoIP"]
+pkg = ["nose==1.3.7", "-e", "git+http://github.com/maxmind/geoip-api-python.git@v1.3.2#egg=GeoIP"]
 create_virtenv(ENV_NAME, pkg, force_create = True)
 GEOIP_DIR = os.path.abspath(os.path.join(SRC_DIR, "geoip"))
 expected = [{'ran': 10}]

test/tests/compile_test2.py

+# this tests are from cpythons test_compile.py
+import unittest
+from test import test_support
+
+class TestSpecifics(unittest.TestCase):
+    def test_exec_functional_style(self):
+        # Exec'ing a tuple of length 2 works.
+        g = {'b': 2}
+        exec("a = b + 1", g)
+        self.assertEqual(g['a'], 3)
+
+        # As does exec'ing a tuple of length 3.
+        l = {'b': 3}
+        g = {'b': 5, 'c': 7}
+        exec("a = b + c", g, l)
+        self.assertNotIn('a', g)
+        self.assertEqual(l['a'], 10)
+
+        # Tuples not of length 2 or 3 are invalid.
+        with self.assertRaises(TypeError):
+            exec("a = b + 1",)
+
+        with self.assertRaises(TypeError):
+            exec("a = b + 1", {}, {}, {})
+
+        # Can't mix and match the two calling forms.
+        g = {'a': 3, 'b': 4}
+        l = {}
+        with self.assertRaises(TypeError):
+            exec("a = b + 1", g) in g
+        with self.assertRaises(TypeError):
+            exec("a = b + 1", g, l) in g, l
+
+    def test_exec_with_general_mapping_for_locals(self):
+
+        class M:
+            "Test mapping interface versus possible calls from eval()."
+            def __getitem__(self, key):
+                if key == 'a':
+                    return 12
+                raise KeyError
+            def __setitem__(self, key, value):
+                self.results = (key, value)
+            def keys(self):
+                return list('xyz')
+
+        m = M()
+        g = globals()
+        exec 'z = a' in g, m
+        self.assertEqual(m.results, ('z', 12))
+        try:
+            exec 'z = b' in g, m
+        except NameError:
+            pass
+        else:
+            self.fail('Did not detect a KeyError')
+        exec 'z = dir()' in g, m
+        self.assertEqual(m.results, ('z', list('xyz')))
+        exec 'z = globals()' in g, m
+        self.assertEqual(m.results, ('z', g))
+        exec 'z = locals()' in g, m
+        self.assertEqual(m.results, ('z', m))
+        try:
+            exec 'z = b' in m
+        except TypeError:
+            pass
+        else:
+            self.fail('Did not validate globals as a real dict')
+
+        class A:
+            "Non-mapping"
+            pass
+        m = A()
+        try:
+            exec 'z = a' in g, m
+        except TypeError:
+            pass
+        else:
+            self.fail('Did not validate locals as a mapping')
+
+        # Verify that dict subclasses work as well
+        class D(dict):
+            def __getitem__(self, key):
+                if key == 'a':
+                    return 12
+                return dict.__getitem__(self, key)
+        d = D()
+        exec 'z = a' in g, d
+        self.assertEqual(d['z'], 12)
+
+    def test_unicode_encoding(self):
+        code = u"# -*- coding: utf-8 -*-\npass\n"
+        self.assertRaises(SyntaxError, compile, code, "tmp", "exec")
+def test_main():
+    test_support.run_unittest(TestSpecifics)
+
+if __name__ == "__main__":
+    # pyston change: remove duration in test output
+    # test_main()
+    import sys, StringIO, re
+    orig_stdout = sys.stdout
+    out = StringIO.StringIO()
+    sys.stdout = out
+    test_main()
+    sys.stdout = orig_stdout
+    print re.sub(" [.0-9]+s", " TIME", out.getvalue())
+

test/tests/eval_bad_indentation.py

-# expected: fail
-
 try:
     eval("\n 2")
     print "bad, should have thrown an exception"