Skip to content

P
Pyston
Commit 5501fb38 authored by Marius Wachtler's avatar Marius Wachtler
Browse files
Options

cleanup the eval and exec implementation 

This makes the implementation more similar to cpythons and removes some missing error handling.
It also uses and implements some missing PyRun_* and PyEval_* functions.
One behaviour change is that 'execfile' now always uses the cpython parser.

If think if we want to support other parsers the right way would be to decide inside PyParser_ASTFromFile which parser to use.
parent c9268664
Show whitespace changes
Inline Side-by-side
Showing with 602 additions and 328 deletions
from_cpython/Include/Python.h
View file @ 5501fb38
...@@ -132,6 +132,7 @@ PyAPI_FUNC(void) PyType_SetDict(PyTypeObject*, PyObject*) PYSTON_NOEXCEPT; ...@@ -132,6 +132,7 @@ PyAPI_FUNC(void) PyType_SetDict(PyTypeObject*, PyObject*) PYSTON_NOEXCEPT;
#include "abstract.h" #include "abstract.h"
#include "compile.h" #include "compile.h"
#include "eval.h"
#include "pyctype.h" #include "pyctype.h"
#include "pystrtod.h" #include "pystrtod.h"
......
from_cpython/Include/compile.h
View file @ 5501fb38
...@@ -10,7 +10,7 @@ extern "C" { ...@@ -10,7 +10,7 @@ extern "C" {
/* Public interface */ /* Public interface */
struct _node; /* Declare the existence of this type */ struct _node; /* Declare the existence of this type */
PyAPI_FUNC(PyCodeObject *) PyNode_Compile(struct _node *, const char *); PyAPI_FUNC(PyCodeObject *) PyNode_Compile(struct _node *, const char *) PYSTON_NOEXCEPT;
/* Future feature support */ /* Future feature support */
...@@ -30,8 +30,8 @@ typedef struct { ...@@ -30,8 +30,8 @@ typedef struct {
struct _mod; /* Declare the existence of this type */ struct _mod; /* Declare the existence of this type */
PyAPI_FUNC(PyCodeObject *) PyAST_Compile(struct _mod *, const char *, PyAPI_FUNC(PyCodeObject *) PyAST_Compile(struct _mod *, const char *,
PyCompilerFlags *, PyArena *); PyCompilerFlags *, PyArena *) PYSTON_NOEXCEPT;
PyAPI_FUNC(PyFutureFeatures *) PyFuture_FromAST(struct _mod *, const char *); PyAPI_FUNC(PyFutureFeatures *) PyFuture_FromAST(struct _mod *, const char *) PYSTON_NOEXCEPT;
#ifdef __cplusplus #ifdef __cplusplus
......
from_cpython/Include/eval.h
View file @ 5501fb38
...@@ -7,7 +7,7 @@ ...@@ -7,7 +7,7 @@
extern "C" { extern "C" {
#endif #endif
PyAPI_FUNC(PyObject *) PyEval_EvalCode(PyCodeObject *, PyObject *, PyObject *); PyAPI_FUNC(PyObject *) PyEval_EvalCode(PyCodeObject *, PyObject *, PyObject *) PYSTON_NOEXCEPT;
PyAPI_FUNC(PyObject *) PyEval_EvalCodeEx(PyCodeObject *co, PyAPI_FUNC(PyObject *) PyEval_EvalCodeEx(PyCodeObject *co,
PyObject *globals, PyObject *globals,
...@@ -15,9 +15,9 @@ PyAPI_FUNC(PyObject *) PyEval_EvalCodeEx(PyCodeObject *co, ...@@ -15,9 +15,9 @@ PyAPI_FUNC(PyObject *) PyEval_EvalCodeEx(PyCodeObject *co,
PyObject **args, int argc, PyObject **args, int argc,
PyObject **kwds, int kwdc, PyObject **kwds, int kwdc,
PyObject **defs, int defc, PyObject **defs, int defc,
PyObject *closure); PyObject *closure) PYSTON_NOEXCEPT;
PyAPI_FUNC(PyObject *) _PyEval_CallTracing(PyObject *func, PyObject *args); PyAPI_FUNC(PyObject *) _PyEval_CallTracing(PyObject *func, PyObject *args) PYSTON_NOEXCEPT;
#ifdef __cplusplus #ifdef __cplusplus
} }
......
from_cpython/Lib/test/test_univnewlines2k.py
View file @ 5501fb38
# expected: fail
# Tests universal newline support for both reading and parsing files. # Tests universal newline support for both reading and parsing files.
import unittest import unittest
import os import os
......
src/codegen/irgen/hooks.cpp
View file @ 5501fb38
...@@ -380,46 +380,10 @@ static FunctionMetadata* compileForEvalOrExec(AST* source, std::vector<AST_stmt* ...@@ -380,46 +380,10 @@ static FunctionMetadata* compileForEvalOrExec(AST* source, std::vector<AST_stmt*
return md; return md;
} }
static AST_Module* parseExec(llvm::StringRef source, FutureFlags future_flags, bool interactive = false) {
// TODO error message if parse fails or if it isn't an expr
// TODO should have a cleaner interface that can parse the Expression directly
// TODO this memory leaks
const char* code = source.data();
AST_Module* parsedModule = parse_string(code, future_flags);
if (interactive)
makeModuleInteractive(parsedModule);
return parsedModule;
}
static FunctionMetadata* compileExec(AST_Module* parsedModule, BoxedString* fn, PyCompilerFlags* flags) { static FunctionMetadata* compileExec(AST_Module* parsedModule, BoxedString* fn, PyCompilerFlags* flags) {
return compileForEvalOrExec(parsedModule, parsedModule->body, fn, flags); return compileForEvalOrExec(parsedModule, parsedModule->body, fn, flags);
} }
static AST_Expression* parseEval(llvm::StringRef source, FutureFlags future_flags) {
const char* code = source.data();
// TODO error message if parse fails or if it isn't an expr
// TODO should have a cleaner interface that can parse the Expression directly
// TODO this memory leaks
// Hack: we need to support things like `eval(" 2")`.
// This is over-accepting since it will accept things like `eval("\n 2")`
while (*code == ' ' || *code == '\t' || *code == '\n' || *code == '\r')
code++;
AST_Module* parsedModule = parse_string(code, future_flags);
if (parsedModule->body.size() == 0)
raiseSyntaxError("unexpected EOF while parsing", 0, 0, "<string>", "");
RELEASE_ASSERT(parsedModule->body.size() == 1, "");
RELEASE_ASSERT(parsedModule->body[0]->type == AST_TYPE::Expr, "");
AST_Expression* parsedExpr = new AST_Expression(std::move(parsedModule->interned_strings));
parsedExpr->body = static_cast<AST_Expr*>(parsedModule->body[0])->value;
return parsedExpr;
}
static FunctionMetadata* compileEval(AST_Expression* parsedExpr, BoxedString* fn, PyCompilerFlags* flags) { static FunctionMetadata* compileEval(AST_Expression* parsedExpr, BoxedString* fn, PyCompilerFlags* flags) {
// We need body (list of statements) to compile. // We need body (list of statements) to compile.
// Obtain this by simply making a single statement which contains the expression. // Obtain this by simply making a single statement which contains the expression.
...@@ -430,140 +394,71 @@ static FunctionMetadata* compileEval(AST_Expression* parsedExpr, BoxedString* fn ...@@ -430,140 +394,71 @@ static FunctionMetadata* compileEval(AST_Expression* parsedExpr, BoxedString* fn
return compileForEvalOrExec(parsedExpr, std::move(body), fn, flags); return compileForEvalOrExec(parsedExpr, std::move(body), fn, flags);
} }
Box* compile(Box* source, Box* fn, Box* type, Box** _args) { extern "C" PyCodeObject* PyAST_Compile(struct _mod* _mod, const char* filename, PyCompilerFlags* flags,
Box* flags = _args[0]; PyArena* arena) noexcept {
try {
RELEASE_ASSERT(PyInt_Check(_args[1]), ""); mod_ty mod = _mod;
bool dont_inherit = (bool)static_cast<BoxedInt*>(_args[1])->n; AST* parsed = cpythonToPystonAST(mod, filename);
FunctionMetadata* md = NULL;
RELEASE_ASSERT(flags->cls == int_cls, ""); switch (mod->kind) {
int64_t iflags = static_cast<BoxedInt*>(flags)->n; case Module_kind:
case Interactive_kind:
// source is allowed to be an AST, unicode, or anything that supports the buffer protocol if (parsed->type != AST_TYPE::Module) {
if (source->cls == unicode_cls) { raiseExcHelper(TypeError, "expected Module node, got %s", AST_TYPE::stringify(parsed->type));
source = PyUnicode_AsUTF8String(source);
if (!source)
throwCAPIException();
// cf.cf_flags |= PyCF_SOURCE_IS_UTF8
} }
md = compileExec(static_cast<AST_Module*>(parsed), boxString(filename), flags);
if (isSubclass(fn->cls, unicode_cls)) { break;
fn = _PyUnicode_AsDefaultEncodedString(fn, NULL); case Expression_kind:
if (!fn) if (parsed->type != AST_TYPE::Expression) {
throwCAPIException(); raiseExcHelper(TypeError, "expected Expression node, got %s", AST_TYPE::stringify(parsed->type));
} }
RELEASE_ASSERT(PyString_Check(fn), ""); md = compileEval(static_cast<AST_Expression*>(parsed), boxString(filename), flags);
break;
if (isSubclass(type->cls, unicode_cls)) { case Suite_kind:
type = _PyUnicode_AsDefaultEncodedString(type, NULL); PyErr_SetString(PyExc_SystemError, "suite should not be possible");
if (!type) return NULL;
throwCAPIException(); default:
PyErr_Format(PyExc_SystemError, "module kind %d should not be possible", mod->kind);
return NULL;
} }
RELEASE_ASSERT(PyString_Check(type), "");
BoxedString* filename_str = static_cast<BoxedString*>(fn);
BoxedString* type_str = static_cast<BoxedString*>(type);
if (iflags & ~(PyCF_MASK | PyCF_MASK_OBSOLETE | /* PyCF_DONT_IMPLY_DEDENT | */ PyCF_ONLY_AST)) { return (PyCodeObject*)md->getCode();
raiseExcHelper(ValueError, "compile(): unrecognised flags"); } catch (ExcInfo e) {
setCAPIException(e);
return NULL;
} }
}
bool only_ast = (bool)(iflags & PyCF_ONLY_AST); extern "C" int PyEval_MergeCompilerFlags(PyCompilerFlags* cf) noexcept {
int result = cf->cf_flags != 0;
iflags &= ~PyCF_ONLY_AST; /* Pyston change:
PyFrameObject *current_frame = PyEval_GetFrame();
int result = cf->cf_flags != 0;
if (current_frame != NULL) {
const int codeflags = current_frame->f_code->co_flags;
*/
FutureFlags arg_future_flags = iflags & PyCF_MASK;
FutureFlags future_flags;
if (dont_inherit) {
future_flags = arg_future_flags;
} else {
FunctionMetadata* caller_cl = getTopPythonFunction(); FunctionMetadata* caller_cl = getTopPythonFunction();
assert(caller_cl != NULL); if (caller_cl != NULL) {
assert(caller_cl->source != NULL); assert(caller_cl->source != NULL);
FutureFlags caller_future_flags = caller_cl->source->future_flags; FutureFlags caller_future_flags = caller_cl->source->future_flags;
future_flags = arg_future_flags | caller_future_flags;
}
iflags &= !(PyCF_MASK | PyCF_MASK_OBSOLETE);
RELEASE_ASSERT(iflags == 0, "");
AST* parsed;
mod_ty mod;
ArenaWrapper arena; const int codeflags = caller_future_flags;
const int compilerflags = codeflags & PyCF_MASK;
if (PyAST_Check(source)) { if (compilerflags) {
int mode; result = 1;
if (type_str->s() == "exec") cf->cf_flags |= compilerflags;
mode = 0;
else if (type_str->s() == "eval")
mode = 1;
else if (type_str->s() == "single")
mode = 2;
else {
raiseExcHelper(ValueError, "compile() arg 3 must be 'exec', 'eval' or 'single'");
} }
if (only_ast) // nothing to do #if 0 /* future keyword */
return source; if (codeflags & CO_GENERATOR_ALLOWED) {
result = 1;
mod = PyAST_obj2mod(source, arena, mode); cf->cf_flags |= CO_GENERATOR_ALLOWED;
if (PyErr_Occurred())
throwCAPIException();
} else {
RELEASE_ASSERT(PyString_Check(source), "");
int mode;
if (type_str->s() == "exec")
mode = Py_file_input;
else if (type_str->s() == "eval")
mode = Py_eval_input;
else if (type_str->s() == "single")
mode = Py_single_input;
else {
raiseExcHelper(ValueError, "compile() arg 3 must be 'exec', 'eval' or 'single'");
} }
#endif
PyCompilerFlags cf;
cf.cf_flags = future_flags;
const char* code = static_cast<BoxedString*>(source)->s().data();
assert(arena);
const char* fn = filename_str->c_str();
mod = PyParser_ASTFromString(code, fn, mode, &cf, arena);
if (!mod)
throwCAPIException();
} }
if (only_ast) {
Box* result = PyAST_mod2obj(mod);
if (PyErr_Occurred())
throwCAPIException();
return result; return result;
}
// be careful when moving around this function: it does also do some additional syntax checking (=raises exception),
// which we should not do when in AST only mode.
parsed = cpythonToPystonAST(mod, filename_str->c_str());
PyCompilerFlags pcf;
pcf.cf_flags = future_flags;
FunctionMetadata* md;
if (type_str->s() == "exec" || type_str->s() == "single") {
// TODO: CPython parses execs as Modules
if (parsed->type != AST_TYPE::Module) {
raiseExcHelper(TypeError, "expected Module node, got %s", AST_TYPE::stringify(parsed->type));
}
md = compileExec(static_cast<AST_Module*>(parsed), filename_str, &pcf);
} else if (type_str->s() == "eval") {
if (parsed->type != AST_TYPE::Expression) {
raiseExcHelper(TypeError, "expected Expression node, got %s", AST_TYPE::stringify(parsed->type));
}
md = compileEval(static_cast<AST_Expression*>(parsed), filename_str, &pcf);
} else {
raiseExcHelper(ValueError, "compile() arg 3 must be 'exec', 'eval' or 'single'");
}
return (Box*)md->getCode();
} }
static void pickGlobalsAndLocals(Box*& globals, Box*& locals) { static void pickGlobalsAndLocals(Box*& globals, Box*& locals) {
...@@ -609,151 +504,111 @@ static void pickGlobalsAndLocals(Box*& globals, Box*& locals) { ...@@ -609,151 +504,111 @@ static void pickGlobalsAndLocals(Box*& globals, Box*& locals) {
} }
} }
static Box* evalMain(Box* boxedCode, Box* globals, Box* locals, PyCompilerFlags* flags) { extern "C" PyObject* PyEval_EvalCode(PyCodeObject* co, PyObject* globals, PyObject* locals) noexcept {
try {
pickGlobalsAndLocals(globals, locals); pickGlobalsAndLocals(globals, locals);
return evalOrExec(metadataFromCode((Box*)co), globals, locals);
if (boxedCode->cls == unicode_cls) { } catch (ExcInfo e) {
boxedCode = PyUnicode_AsUTF8String(boxedCode); setCAPIException(e);
if (!boxedCode) return NULL;
throwCAPIException();
// cf.cf_flags |= PyCF_SOURCE_IS_UTF8
}
FunctionMetadata* md;
if (boxedCode->cls == str_cls) {
FunctionMetadata* caller_cl = getTopPythonFunction();
assert(caller_cl != NULL);
assert(caller_cl->source != NULL);
AST_Expression* parsed = parseEval(static_cast<BoxedString*>(boxedCode)->s(), caller_cl->source->future_flags);
static BoxedString* string_string = internStringImmortal("<string>");
md = compileEval(parsed, string_string, flags);
} else if (boxedCode->cls == code_cls) {
md = metadataFromCode(boxedCode);
} else {
abort();
} }
return evalOrExec(md, globals, locals);
} }
Box* eval(Box* boxedCode, Box* globals, Box* locals) { Box* exec(Box* boxedCode, Box* globals, Box* locals, FutureFlags caller_future_flags) {
FunctionMetadata* caller_cl = getTopPythonFunction(); if (!globals)
assert(caller_cl != NULL); globals = None;
assert(caller_cl->source != NULL); if (!locals)
FutureFlags caller_future_flags = caller_cl->source->future_flags; locals = None;
PyCompilerFlags pcf;
pcf.cf_flags = caller_future_flags; // this is based on cpythons exec_statement() but (heavily) adopted
Box* v = NULL;
return evalMain(boxedCode, globals, locals, &pcf); int plain = 0;
} int n;
PyObject* prog = boxedCode;
Box* execfile(Box* _fn, Box* globals, Box* locals) { if (PyTuple_Check(prog) && globals == Py_None && locals == Py_None && ((n = PyTuple_Size(prog)) == 2 || n == 3)) {
if (!PyString_Check(_fn)) { /* Backward compatibility hack */
raiseExcHelper(TypeError, "must be string, not %s", getTypeName(_fn)); globals = PyTuple_GetItem(prog, 1);
} if (n == 3)
locals = PyTuple_GetItem(prog, 2);
BoxedString* fn = static_cast<BoxedString*>(_fn); prog = PyTuple_GetItem(prog, 0);
}
pickGlobalsAndLocals(globals, locals); if (globals == Py_None) {
globals = PyEval_GetGlobals();
#if LLVMREV < 217625 if (locals == Py_None) {
bool exists; locals = PyEval_GetLocals();
llvm_error_code code = llvm::sys::fs::exists(fn->s, exists); plain = 1;
}
#if LLVMREV < 210072 if (!globals || !locals) {
ASSERT(code == 0, "%s: %s", code.message().c_str(), fn->s.c_str()); raiseExcHelper(SystemError, "globals and locals cannot be NULL");
#else }
assert(!code); } else if (locals == Py_None)
#endif locals = globals;
if (!PyString_Check(prog) &&
#else #ifdef Py_USING_UNICODE
bool exists = llvm::sys::fs::exists(std::string(fn->s())); !PyUnicode_Check(prog) &&
#endif #endif
!PyCode_Check(prog) && !PyFile_Check(prog)) {
if (!exists) raiseExcHelper(TypeError, "exec: arg 1 must be a string, file, or code object");
raiseExcHelper(IOError, "No such file or directory: '%s'", fn->s().data());
AST_Module* parsed = caching_parse_file(fn->s().data(), /* future_flags = */ 0);
assert(parsed);
FunctionMetadata* caller_cl = getTopPythonFunction();
assert(caller_cl != NULL);
assert(caller_cl->source != NULL);
FutureFlags caller_future_flags = caller_cl->source->future_flags;
PyCompilerFlags pcf;
pcf.cf_flags = caller_future_flags;
FunctionMetadata* md = compileForEvalOrExec(parsed, parsed->body, fn, &pcf);
assert(md);
return evalOrExec(md, globals, locals);
}
Box* execMain(Box* boxedCode, Box* globals, Box* locals, PyCompilerFlags* flags) {
if (PyTuple_Check(boxedCode)) {
RELEASE_ASSERT(!globals, "");
RELEASE_ASSERT(!locals, "");
BoxedTuple* t = static_cast<BoxedTuple*>(boxedCode);
RELEASE_ASSERT(t->size() >= 2 && t->size() <= 3, "%ld", t->size());
boxedCode = t->elts[0];
globals = t->elts[1];
if (t->size() >= 3)
locals = t->elts[2];
} }
pickGlobalsAndLocals(globals, locals); if (!PyDict_Check(globals) && globals->cls != attrwrapper_cls) {
raiseExcHelper(TypeError, "exec: arg 2 must be a dictionary or None");
if (boxedCode->cls == unicode_cls) {
boxedCode = PyUnicode_AsUTF8String(boxedCode);
if (!boxedCode)
throwCAPIException();
// cf.cf_flags |= PyCF_SOURCE_IS_UTF8
} }
if (!PyMapping_Check(locals))
raiseExcHelper(TypeError, "exec: arg 3 must be a mapping or None");
FunctionMetadata* md; if (PyDict_GetItemString(globals, "__builtins__") == NULL)
if (boxedCode->cls == str_cls) { // Pyston change:
FunctionMetadata* caller_cl = getTopPythonFunction(); // PyDict_SetItemString(globals, "__builtins__", f->f_builtins);
assert(caller_cl != NULL); PyDict_SetItemString(globals, "__builtins__", builtins_module);
assert(caller_cl->source != NULL);
auto parsed = parseExec(static_cast<BoxedString*>(boxedCode)->s(), caller_cl->source->future_flags); if (PyCode_Check(prog)) {
static BoxedString* string_string = internStringImmortal("<string>"); /* Pyston change:
md = compileExec(parsed, string_string, flags); if (PyCode_GetNumFree((PyCodeObject *)prog) > 0) {
} else if (boxedCode->cls == code_cls) { PyErr_SetString(PyExc_TypeError,
md = metadataFromCode(boxedCode); "code object passed to exec may not contain free variables");
return -1;
}
*/
v = PyEval_EvalCode((PyCodeObject*)prog, globals, locals);
} else if (PyFile_Check(prog)) {
FILE* fp = PyFile_AsFile(prog);
char* name = PyString_AsString(PyFile_Name(prog));
PyCompilerFlags cf;
if (name == NULL)
throwCAPIException();
cf.cf_flags = caller_future_flags & PyCF_MASK;
if (cf.cf_flags)
v = PyRun_FileFlags(fp, name, Py_file_input, globals, locals, &cf);
else
v = PyRun_File(fp, name, Py_file_input, globals, locals);
} else { } else {
abort(); PyObject* tmp = NULL;
char* str;
PyCompilerFlags cf;
cf.cf_flags = 0;
#ifdef Py_USING_UNICODE
if (PyUnicode_Check(prog)) {
tmp = PyUnicode_AsUTF8String(prog);
if (tmp == NULL)
throwCAPIException();
prog = tmp;
cf.cf_flags |= PyCF_SOURCE_IS_UTF8;
} }
assert(md); #endif
if (PyString_AsStringAndSize(prog, &str, NULL))
return evalOrExec(md, globals, locals); return 0;
} cf.cf_flags |= caller_future_flags & PyCF_MASK;
if (cf.cf_flags)
Box* exec(Box* boxedCode, Box* globals, Box* locals, FutureFlags caller_future_flags) { v = PyRun_StringFlags(str, Py_file_input, globals, locals, &cf);
PyCompilerFlags pcf; else
pcf.cf_flags = caller_future_flags; v = PyRun_String(str, Py_file_input, globals, locals);
return execMain(boxedCode, globals, locals, &pcf); Py_XDECREF(tmp);
}
extern "C" PyObject* PyRun_StringFlags(const char* str, int start, PyObject* globals, PyObject* locals,
PyCompilerFlags* flags) noexcept {
try {
// TODO pass future_flags (the information is in PyCompilerFlags but we need to
// unify the format...)
if (start == Py_file_input)
return execMain(boxString(str), globals, locals, flags);
else if (start == Py_eval_input)
return evalMain(boxString(str), globals, locals, flags);
} catch (ExcInfo e) {
setCAPIException(e);
return NULL;
} }
// Py_single_input is not yet implemented if (!v)
RELEASE_ASSERT(0, "Unimplemented %d", start); throwCAPIException();
return 0; return v;
} }
// If a function version keeps failing its speculations, kill it (remove it // If a function version keeps failing its speculations, kill it (remove it
......
src/codegen/irgen/hooks.h
View file @ 5501fb38
...@@ -40,9 +40,6 @@ void compileAndRunModule(AST_Module* m, BoxedModule* bm); ...@@ -40,9 +40,6 @@ void compileAndRunModule(AST_Module* m, BoxedModule* bm);
CompiledFunction* cfForMachineFunctionName(const std::string&); CompiledFunction* cfForMachineFunctionName(const std::string&);
extern "C" Box* exec(Box* boxedCode, Box* globals, Box* locals, FutureFlags caller_future_flags); extern "C" Box* exec(Box* boxedCode, Box* globals, Box* locals, FutureFlags caller_future_flags);
extern "C" Box* execfile(Box* fn, Box* globals, Box* locals);
extern "C" Box* eval(Box* boxedCode, Box* globals, Box* locals);
extern "C" Box* compile(Box* source, Box* filename, Box* mode, Box** _args /* flags, dont_inherit */);
} }
#endif #endif
src/runtime/builtin_modules/builtins.cpp
View file @ 5501fb38
...@@ -19,6 +19,9 @@ ...@@ -19,6 +19,9 @@
#include "llvm/Support/FileSystem.h" #include "llvm/Support/FileSystem.h"
#include "Python.h"
#include "Python-ast.h"
#include "capi/typeobject.h" #include "capi/typeobject.h"
#include "capi/types.h" #include "capi/types.h"
#include "codegen/ast_interpreter.h" #include "codegen/ast_interpreter.h"
...@@ -1604,18 +1607,14 @@ Box* rawInput(Box* prompt) { ...@@ -1604,18 +1607,14 @@ Box* rawInput(Box* prompt) {
} }
Box* input(Box* prompt) { Box* input(Box* prompt) {
char* str; PyObject* line = rawInput(prompt);
PyObject* line = raw_input(prompt);
if (line == NULL)
throwCAPIException();
char* str = NULL;
if (!PyArg_Parse(line, "s;embedded '\\0' in input line", &str)) if (!PyArg_Parse(line, "s;embedded '\\0' in input line", &str))
throwCAPIException(); throwCAPIException();
// CPython trims the string first, but our eval function takes care of that. while (*str == ' ' || *str == '\t')
// while (*str == ' ' || *str == '\t') str++;
// str++;
Box* gbls = globals(); Box* gbls = globals();
Box* lcls = locals(); Box* lcls = locals();
...@@ -1628,7 +1627,13 @@ Box* input(Box* prompt) { ...@@ -1628,7 +1627,13 @@ Box* input(Box* prompt) {
throwCAPIException(); throwCAPIException();
} }
return eval(line, gbls, lcls); PyCompilerFlags cf;
cf.cf_flags = 0;
PyEval_MergeCompilerFlags(&cf);
Box* res = PyRun_StringFlags(str, Py_eval_input, gbls, lcls, &cf);
if (!res)
throwCAPIException();
return res;
} }
Box* builtinRound(Box* _number, Box* _ndigits) { Box* builtinRound(Box* _number, Box* _ndigits) {
...@@ -1695,6 +1700,259 @@ Box* builtinFormat(Box* value, Box* format_spec) { ...@@ -1695,6 +1700,259 @@ Box* builtinFormat(Box* value, Box* format_spec) {
return res; return res;
} }
static PyObject* builtin_compile(PyObject* self, PyObject* args, PyObject* kwds) noexcept {
char* str;
char* filename;
char* startstr;
int mode = -1;
int dont_inherit = 0;
int supplied_flags = 0;
int is_ast;
PyCompilerFlags cf;
PyObject* result = NULL, *cmd, * tmp = NULL;
Py_ssize_t length;
static const char* kwlist[] = { "source", "filename", "mode", "flags", "dont_inherit", NULL };
int start[] = { Py_file_input, Py_eval_input, Py_single_input };
if (!PyArg_ParseTupleAndKeywords(args, kwds, "Oss|ii:compile", const_cast<char**>(kwlist), &cmd, &filename,
&startstr, &supplied_flags, &dont_inherit))
return NULL;
cf.cf_flags = supplied_flags;
if (supplied_flags & ~(PyCF_MASK | PyCF_MASK_OBSOLETE | PyCF_DONT_IMPLY_DEDENT | PyCF_ONLY_AST)) {
PyErr_SetString(PyExc_ValueError, "compile(): unrecognised flags");
return NULL;
}
/* XXX Warn if (supplied_flags & PyCF_MASK_OBSOLETE) != 0? */
if (!dont_inherit) {
PyEval_MergeCompilerFlags(&cf);
}
if (strcmp(startstr, "exec") == 0)
mode = 0;
else if (strcmp(startstr, "eval") == 0)
mode = 1;
else if (strcmp(startstr, "single") == 0)
mode = 2;
else {
PyErr_SetString(PyExc_ValueError, "compile() arg 3 must be 'exec', 'eval' or 'single'");
return NULL;
}
is_ast = PyAST_Check(cmd);
if (is_ast == -1)
return NULL;
if (is_ast) {
if (supplied_flags & PyCF_ONLY_AST) {
Py_INCREF(cmd);
result = cmd;
} else {
PyArena* arena;
mod_ty mod;
arena = PyArena_New();
if (arena == NULL)
return NULL;
mod = PyAST_obj2mod(cmd, arena, mode);
if (mod == NULL) {
PyArena_Free(arena);
return NULL;
}
result = (PyObject*)PyAST_Compile(mod, filename, &cf, arena);
PyArena_Free(arena);
}
return result;
}
#ifdef Py_USING_UNICODE
if (PyUnicode_Check(cmd)) {
tmp = PyUnicode_AsUTF8String(cmd);
if (tmp == NULL)
return NULL;
cmd = tmp;
cf.cf_flags |= PyCF_SOURCE_IS_UTF8;
}
#endif
if (PyObject_AsReadBuffer(cmd, (const void**)&str, &length))
goto cleanup;
if ((size_t)length != strlen(str)) {
PyErr_SetString(PyExc_TypeError, "compile() expected string without null bytes");
goto cleanup;
}
result = Py_CompileStringFlags(str, filename, start[mode], &cf);
cleanup:
Py_XDECREF(tmp);
return result;
}
static PyObject* builtin_eval(PyObject* self, PyObject* args) noexcept {
PyObject* cmd, *result, * tmp = NULL;
PyObject* globals = Py_None, * locals = Py_None;
char* str;
PyCompilerFlags cf;
if (!PyArg_UnpackTuple(args, "eval", 1, 3, &cmd, &globals, &locals))
return NULL;
if (locals != Py_None && !PyMapping_Check(locals)) {
PyErr_SetString(PyExc_TypeError, "locals must be a mapping");
return NULL;
}
// Pyston change:
// if (globals != Py_None && !PyDict_Check(globals)) {
if (globals != Py_None && !PyDict_Check(globals) && globals->cls != attrwrapper_cls) {
PyErr_SetString(PyExc_TypeError, PyMapping_Check(globals)
? "globals must be a real dict; try eval(expr, {}, mapping)"
: "globals must be a dict");
return NULL;
}
if (globals == Py_None) {
globals = PyEval_GetGlobals();
if (locals == Py_None)
locals = PyEval_GetLocals();
} else if (locals == Py_None)
locals = globals;
if (globals == NULL || locals == NULL) {
PyErr_SetString(PyExc_TypeError, "eval must be given globals and locals "
"when called without a frame");
return NULL;
}
if (PyDict_GetItemString(globals, "__builtins__") == NULL) {
if (PyDict_SetItemString(globals, "__builtins__", PyEval_GetBuiltins()) != 0)
return NULL;
}
if (PyCode_Check(cmd)) {
// Pyston change:
#if 0
if (PyCode_GetNumFree((PyCodeObject *)cmd) > 0) {
PyErr_SetString(PyExc_TypeError,
"code object passed to eval() may not contain free variables");
return NULL;
}
#endif
return PyEval_EvalCode((PyCodeObject*)cmd, globals, locals);
}
if (!PyString_Check(cmd) && !PyUnicode_Check(cmd)) {
PyErr_SetString(PyExc_TypeError, "eval() arg 1 must be a string or code object");
return NULL;
}
cf.cf_flags = 0;
#ifdef Py_USING_UNICODE
if (PyUnicode_Check(cmd)) {
tmp = PyUnicode_AsUTF8String(cmd);
if (tmp == NULL)
return NULL;
cmd = tmp;
cf.cf_flags |= PyCF_SOURCE_IS_UTF8;
}
#endif
if (PyString_AsStringAndSize(cmd, &str, NULL)) {
Py_XDECREF(tmp);
return NULL;
}
while (*str == ' ' || *str == '\t')
str++;
(void)PyEval_MergeCompilerFlags(&cf);
result = PyRun_StringFlags(str, Py_eval_input, globals, locals, &cf);
Py_XDECREF(tmp);
return result;
}
static PyObject* builtin_execfile(PyObject* self, PyObject* args) noexcept {
char* filename;
PyObject* globals = Py_None, * locals = Py_None;
PyObject* res;
FILE* fp = NULL;
PyCompilerFlags cf;
int exists;
if (PyErr_WarnPy3k("execfile() not supported in 3.x; use exec()", 1) < 0)
return NULL;
if (!PyArg_ParseTuple(args, "s|O!O:execfile", &filename, &PyDict_Type, &globals, &locals))
return NULL;
if (locals != Py_None && !PyMapping_Check(locals)) {
PyErr_SetString(PyExc_TypeError, "locals must be a mapping");
return NULL;
}
if (globals == Py_None) {
globals = PyEval_GetGlobals();
if (locals == Py_None)
locals = PyEval_GetLocals();
} else if (locals == Py_None)
locals = globals;
if (PyDict_GetItemString(globals, "__builtins__") == NULL) {
if (PyDict_SetItemString(globals, "__builtins__", PyEval_GetBuiltins()) != 0)
return NULL;
}
exists = 0;
/* Test for existence or directory. */
#if defined(PLAN9)
{
Dir* d;
if ((d = dirstat(filename)) != nil) {
if (d->mode & DMDIR)
werrstr("is a directory");
else
exists = 1;
free(d);
}
}
#elif defined(RISCOS)
if (object_exists(filename)) {
if (isdir(filename))
errno = EISDIR;
else
exists = 1;
}
#else /* standard Posix */
{
struct stat s;
if (stat(filename, &s) == 0) {
if (S_ISDIR(s.st_mode))
#if defined(PYOS_OS2) && defined(PYCC_VACPP)
errno = EOS2ERR;
#else
errno = EISDIR;
#endif
else
exists = 1;
}
}
#endif
if (exists) {
Py_BEGIN_ALLOW_THREADS fp = fopen(filename, "r" PY_STDIOTEXTMODE);
Py_END_ALLOW_THREADS
if (fp == NULL) {
exists = 0;
}
}
if (!exists) {
PyErr_SetFromErrnoWithFilename(PyExc_IOError, filename);
return NULL;
}
cf.cf_flags = 0;
if (PyEval_MergeCompilerFlags(&cf))
res = PyRun_FileExFlags(fp, filename, Py_file_input, globals, locals, 1, &cf);
else
res = PyRun_FileEx(fp, filename, Py_file_input, globals, locals, 1);
return res;
}
extern "C" { extern "C" {
BoxedClass* ellipsis_cls; BoxedClass* ellipsis_cls;
} }
...@@ -2206,16 +2464,6 @@ void setupBuiltins() { ...@@ -2206,16 +2464,6 @@ void setupBuiltins() {
builtins_module->giveAttr("divmod", new BoxedBuiltinFunctionOrMethod( builtins_module->giveAttr("divmod", new BoxedBuiltinFunctionOrMethod(
FunctionMetadata::create((void*)divmod, UNKNOWN, 2), "divmod", divmod_doc)); FunctionMetadata::create((void*)divmod, UNKNOWN, 2), "divmod", divmod_doc));
builtins_module->giveAttr("execfile", new BoxedBuiltinFunctionOrMethod(
FunctionMetadata::create((void*)execfile, UNKNOWN, 3, false, false),
"execfile", { NULL, NULL }, NULL, execfile_doc));
FunctionMetadata* compile_func = new FunctionMetadata(
5, false, false, ParamNames({ "source", "filename", "mode", "flags", "dont_inherit" }, "", ""));
compile_func->addVersion((void*)compile, UNKNOWN, { UNKNOWN, UNKNOWN, UNKNOWN, UNKNOWN, UNKNOWN });
builtins_module->giveAttr("compile", new BoxedBuiltinFunctionOrMethod(compile_func, "compile",
{ boxInt(0), boxInt(0) }, NULL, compile_doc));
builtins_module->giveAttr("map", new BoxedBuiltinFunctionOrMethod( builtins_module->giveAttr("map", new BoxedBuiltinFunctionOrMethod(
FunctionMetadata::create((void*)map, LIST, 1, true, false), "map", map_doc)); FunctionMetadata::create((void*)map, LIST, 1, true, false), "map", map_doc));
builtins_module->giveAttr( builtins_module->giveAttr(
...@@ -2267,9 +2515,6 @@ void setupBuiltins() { ...@@ -2267,9 +2515,6 @@ void setupBuiltins() {
PyType_Ready(&PyBuffer_Type); PyType_Ready(&PyBuffer_Type);
builtins_module->giveAttr("buffer", &PyBuffer_Type); builtins_module->giveAttr("buffer", &PyBuffer_Type);
builtins_module->giveAttr(
"eval", new BoxedBuiltinFunctionOrMethod(FunctionMetadata::create((void*)eval, UNKNOWN, 3, false, false),
"eval", { NULL, NULL }, NULL, eval_doc));
builtins_module->giveAttr("callable", builtins_module->giveAttr("callable",
new BoxedBuiltinFunctionOrMethod(FunctionMetadata::create((void*)callable, UNKNOWN, 1), new BoxedBuiltinFunctionOrMethod(FunctionMetadata::create((void*)callable, UNKNOWN, 1),
"callable", callable_doc)); "callable", callable_doc));
...@@ -2288,6 +2533,9 @@ void setupBuiltins() { ...@@ -2288,6 +2533,9 @@ void setupBuiltins() {
static PyMethodDef builtin_methods[] = { static PyMethodDef builtin_methods[] = {
{ "compile", (PyCFunction)builtin_compile, METH_VARARGS | METH_KEYWORDS, compile_doc },
{ "eval", builtin_eval, METH_VARARGS, eval_doc },
{ "execfile", builtin_execfile, METH_VARARGS, execfile_doc },
{ "print", (PyCFunction)builtin_print, METH_VARARGS | METH_KEYWORDS, print_doc }, { "print", (PyCFunction)builtin_print, METH_VARARGS | METH_KEYWORDS, print_doc },
{ "reload", builtin_reload, METH_O, reload_doc }, { "reload", builtin_reload, METH_O, reload_doc },
}; };
......
src/runtime/capi.cpp
View file @ 5501fb38
...@@ -1184,6 +1184,76 @@ extern "C" mod_ty PyParser_ASTFromFile(FILE* fp, const char* filename, int start ...@@ -1184,6 +1184,76 @@ extern "C" mod_ty PyParser_ASTFromFile(FILE* fp, const char* filename, int start
} }
} }
extern "C" PyObject* Py_CompileStringFlags(const char* str, const char* filename, int start,
PyCompilerFlags* flags) noexcept {
PyCodeObject* co;
mod_ty mod;
PyArena* arena = PyArena_New();
if (arena == NULL)
return NULL;
mod = PyParser_ASTFromString(str, filename, start, flags, arena);
if (mod == NULL) {
PyArena_Free(arena);
return NULL;
}
if (flags && (flags->cf_flags & PyCF_ONLY_AST)) {
PyObject* result = PyAST_mod2obj(mod);
PyArena_Free(arena);
return result;
}
co = PyAST_Compile(mod, filename, flags, arena);
PyArena_Free(arena);
return (PyObject*)co;
}
static PyObject* run_mod(mod_ty mod, const char* filename, PyObject* globals, PyObject* locals, PyCompilerFlags* flags,
PyArena* arena) noexcept {
PyCodeObject* co;
PyObject* v;
co = PyAST_Compile(mod, filename, flags, arena);
if (co == NULL)
return NULL;
v = PyEval_EvalCode(co, globals, locals);
Py_DECREF(co);
return v;
}
extern "C" PyObject* PyRun_FileExFlags(FILE* fp, const char* filename, int start, PyObject* globals, PyObject* locals,
int closeit, PyCompilerFlags* flags) noexcept {
PyObject* ret;
mod_ty mod;
PyArena* arena = PyArena_New();
if (arena == NULL)
return NULL;
mod = PyParser_ASTFromFile(fp, filename, start, 0, 0, flags, NULL, arena);
if (closeit)
fclose(fp);
if (mod == NULL) {
PyArena_Free(arena);
return NULL;
}
ret = run_mod(mod, filename, globals, locals, flags, arena);
PyArena_Free(arena);
return ret;
}
extern "C" PyObject* PyRun_StringFlags(const char* str, int start, PyObject* globals, PyObject* locals,
PyCompilerFlags* flags) noexcept {
PyObject* ret = NULL;
mod_ty mod;
PyArena* arena = PyArena_New();
if (arena == NULL)
return NULL;
mod = PyParser_ASTFromString(str, "<string>", start, flags, arena);
if (mod != NULL)
ret = run_mod(mod, "<string>", globals, locals, flags, arena);
PyArena_Free(arena);
return ret;
}
extern "C" int PyRun_InteractiveLoopFlags(FILE* fp, const char* filename, PyCompilerFlags* flags) noexcept { extern "C" int PyRun_InteractiveLoopFlags(FILE* fp, const char* filename, PyCompilerFlags* flags) noexcept {
PyObject* v; PyObject* v;
int ret; int ret;
......
test/CPYTHON_TEST_NOTES.md
View file @ 5501fb38
...@@ -197,7 +197,6 @@ test_unicode argument passing issue? ...@@ -197,7 +197,6 @@ test_unicode argument passing issue?
test_unicodedata [unknown] test_unicodedata [unknown]
test_unicode_file exit code 139, no error message test_unicode_file exit code 139, no error message
test_unittest serialize_ast assert test_unittest serialize_ast assert
test_univnewlines2k [unknown]
test_univnewlines [unknown] test_univnewlines [unknown]
test_userdict segfault: repr of recursive dict? test_userdict segfault: repr of recursive dict?
test_userlist slice(1L, 1L) test_userlist slice(1L, 1L)
......
test/extra/cheetah_test.py
View file @ 5501fb38
...@@ -9,5 +9,5 @@ create_virtenv(ENV_NAME, ["cheetah==2.4.4", "Markdown==2.0.1"], force_create = T ...@@ -9,5 +9,5 @@ create_virtenv(ENV_NAME, ["cheetah==2.4.4", "Markdown==2.0.1"], force_create = T
cheetah_exe = os.path.join(ENV_NAME, "bin", "cheetah") cheetah_exe = os.path.join(ENV_NAME, "bin", "cheetah")
env = os.environ env = os.environ
env["PATH"] = os.path.join(ENV_NAME, "bin") env["PATH"] = os.path.join(ENV_NAME, "bin")
expected = [{'ran': 2138, 'errors': 4, 'failures': 1}, {'ran': 2138, 'errors': 232, 'failures': 3}] expected = [{'ran': 2138, 'errors': 4}, {'ran': 2138, 'errors': 232, 'failures': 2}]
run_test([cheetah_exe, "test"], cwd=ENV_NAME, expected=expected, env=env) run_test([cheetah_exe, "test"], cwd=ENV_NAME, expected=expected, env=env)
test/extra/geoip_test.py
View file @ 5501fb38
...@@ -7,7 +7,7 @@ ENV_NAME = "geoip_test_env_" + os.path.basename(sys.executable) ...@@ -7,7 +7,7 @@ ENV_NAME = "geoip_test_env_" + os.path.basename(sys.executable)
SRC_DIR = os.path.abspath(os.path.join(ENV_NAME, "src")) SRC_DIR = os.path.abspath(os.path.join(ENV_NAME, "src"))
PYTHON_EXE = os.path.abspath(os.path.join(ENV_NAME, "bin", "python")) PYTHON_EXE = os.path.abspath(os.path.join(ENV_NAME, "bin", "python"))
pkg = ["-e", "git+https://github.com/maxmind/geoip-api-python.git@v1.3.2#egg=GeoIP"] pkg = ["nose==1.3.7", "-e", "git+http://github.com/maxmind/geoip-api-python.git@v1.3.2#egg=GeoIP"]
create_virtenv(ENV_NAME, pkg, force_create = True) create_virtenv(ENV_NAME, pkg, force_create = True)
GEOIP_DIR = os.path.abspath(os.path.join(SRC_DIR, "geoip")) GEOIP_DIR = os.path.abspath(os.path.join(SRC_DIR, "geoip"))
expected = [{'ran': 10}] expected = [{'ran': 10}]
......
test/tests/compile_test2.py 0 → 100644
View file @ 5501fb38
# this tests are from cpythons test_compile.py
import unittest
from test import test_support
class TestSpecifics(unittest.TestCase):
def test_exec_functional_style(self):
# Exec'ing a tuple of length 2 works.
g = {'b': 2}
exec("a = b + 1", g)
self.assertEqual(g['a'], 3)
# As does exec'ing a tuple of length 3.
l = {'b': 3}
g = {'b': 5, 'c': 7}
exec("a = b + c", g, l)
self.assertNotIn('a', g)
self.assertEqual(l['a'], 10)
# Tuples not of length 2 or 3 are invalid.
with self.assertRaises(TypeError):
exec("a = b + 1",)
with self.assertRaises(TypeError):
exec("a = b + 1", {}, {}, {})
# Can't mix and match the two calling forms.
g = {'a': 3, 'b': 4}
l = {}
with self.assertRaises(TypeError):
exec("a = b + 1", g) in g
with self.assertRaises(TypeError):
exec("a = b + 1", g, l) in g, l
def test_exec_with_general_mapping_for_locals(self):
class M:
"Test mapping interface versus possible calls from eval()."
def __getitem__(self, key):
if key == 'a':
return 12
raise KeyError
def __setitem__(self, key, value):
self.results = (key, value)
def keys(self):
return list('xyz')
m = M()
g = globals()
exec 'z = a' in g, m
self.assertEqual(m.results, ('z', 12))
try:
exec 'z = b' in g, m
except NameError:
pass
else:
self.fail('Did not detect a KeyError')
exec 'z = dir()' in g, m
self.assertEqual(m.results, ('z', list('xyz')))
exec 'z = globals()' in g, m
self.assertEqual(m.results, ('z', g))
exec 'z = locals()' in g, m
self.assertEqual(m.results, ('z', m))
try:
exec 'z = b' in m
except TypeError:
pass
else:
self.fail('Did not validate globals as a real dict')
class A:
"Non-mapping"
pass
m = A()
try:
exec 'z = a' in g, m
except TypeError:
pass
else:
self.fail('Did not validate locals as a mapping')
# Verify that dict subclasses work as well
class D(dict):
def __getitem__(self, key):
if key == 'a':
return 12
return dict.__getitem__(self, key)
d = D()
exec 'z = a' in g, d
self.assertEqual(d['z'], 12)
def test_unicode_encoding(self):
code = u"# -*- coding: utf-8 -*-\npass\n"
self.assertRaises(SyntaxError, compile, code, "tmp", "exec")
def test_main():
test_support.run_unittest(TestSpecifics)
if __name__ == "__main__":
# pyston change: remove duration in test output
# test_main()
import sys, StringIO, re
orig_stdout = sys.stdout
out = StringIO.StringIO()
sys.stdout = out
test_main()
sys.stdout = orig_stdout
print re.sub(" [.0-9]+s", " TIME", out.getvalue())
test/tests/eval_bad_indentation.py
View file @ 5501fb38
# expected: fail
try: try:
eval("\n 2") eval("\n 2")
print "bad, should have thrown an exception" print "bad, should have thrown an exception"
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment
GitLab Nexedi Edition | About GitLab | About Nexedi | 沪ICP备2021021310号-2 | 沪ICP备2021021310号-7