Support CPython's errnomodule.c, our first extension module!

errnomodule.c is extremely simple and doesn't do anything complicated,
so this is more of a proof-of-concept for now.
The tricky stuff will most likely be around newly-defined types.

include/Python.h

@@ -16,6 +16,7 @@
 #define PYSTON_EXTINCLUDE_PYTHON_H
 
 #include <assert.h>
+#include <errno.h>
 #include <stdio.h>
 #include <stdlib.h>
 #include <stdint.h>
@@ -26,7 +27,6 @@
 //typedef struct PyObject PyObject;
 typedef void PyObject;
 
-#define Py_INCREF(x)
 bool PyArg_ParseTuple(PyObject*, const char*, ...);
 PyObject* Py_BuildValue(const char*, ...);
 
@@ -40,6 +40,22 @@ struct PyMethodDef {
 };
 typedef struct PyMethodDef PyMethodDef;
 
+PyObject* PyString_FromString(const char*);
+PyObject* PyInt_FromLong(long);
+int PyDict_SetItem(PyObject* mp, PyObject* key, PyObject* item);
+int PyDict_SetItemString(PyObject* mp, const char* key, PyObject* item);
+
+PyObject* PyModule_GetDict(PyObject*);
+PyObject* PyDict_New(void);
+
+#define Py_XDECREF(op) (op)
+#define Py_INCREF(op) (op)
+#define Py_DECREF(op) (op)
+
+#define PyDoc_VAR(name) static char name[]
+#define PyDoc_STRVAR(name, str) PyDoc_VAR(name) = PyDoc_STR(str)
+#define PyDoc_STR(str) str
+
 #define METH_VARARGS  0x0001
 
 #ifdef __cplusplus
@@ -56,4 +72,8 @@ PyObject* Py_InitModule4(const char *arg0, PyMethodDef *arg1, const char *arg2,
 	Py_InitModule4(name, methods, (char *)NULL, (PyObject *)NULL, \
 		       PYTHON_API_VERSION)
 
+#define Py_InitModule3(name, methods, doc) \
+	Py_InitModule4(name, methods, doc, (PyObject *)NULL, \
+		       PYTHON_API_VERSION)
+
 #endif

src/Makefile

@@ -204,6 +204,7 @@ CLANGFLAGS := $(CXXFLAGS_DBG) $(CLANG_EXTRA_FLAGS)
 CLANGFLAGS_RELEASE := $(CXXFLAGS_RELEASE) $(CLANG_EXTRA_FLAGS)
 
 EXT_CFLAGS := $(COMMON_CFLAGS) -fPIC -Wimplicit -O2 -I../include $(CLANG_EXTRA_FLAGS)
+EXT_CFLAGS += -Wno-missing-field-initializers
 
 # Extra flags to enable soon:
 CLANGFLAGS += -Wno-sign-conversion -Wnon-virtual-dtor -Winit-self -Wimplicit-int -Wmissing-include-dirs -Wstrict-overflow=5 -Wundef -Wpointer-arith -Wtype-limits -Wwrite-strings -Wempty-body -Waggregate-return -Wstrict-prototypes -Wold-style-definition -Wmissing-field-initializers -Wredundant-decls -Wnested-externs -Winline -Wint-to-pointer-cast -Wpointer-to-int-cast -Wlong-long -Wvla
@@ -259,11 +260,12 @@ STDLIB_SRCS := $(wildcard runtime/inline/*.cpp)
 SRCS := $(MAIN_SRCS) $(STDLIB_SRCS)
 STDLIB_OBJS := stdlib.bc.o stdlib.stripped.bc.o
 STDLIB_RELEASE_OBJS := stdlib.release.bc.o
+STDMODULES_SRCS := ../lib_python/2.7_Modules/errnomodule.c
 # The stdlib objects have slightly longer dependency chains,
 # so put them first in the list:
-OBJS := $(STDLIB_OBJS) $(SRCS:.cpp=.o)
-PROFILE_OBJS := $(STDLIB_RELEASE_OBJS) $(MAIN_SRCS:.cpp=.prof.o) $(STDLIB_SRCS:.cpp=.release.o)
-OPT_OBJS := $(STDLIB_RELEASE_OBJS) $(SRCS:.cpp=.release.o)
+OBJS := $(STDLIB_OBJS) $(SRCS:.cpp=.o) $(STDMODULES_SRCS:.c=.o)
+PROFILE_OBJS := $(STDLIB_RELEASE_OBJS) $(MAIN_SRCS:.cpp=.prof.o) $(STDLIB_SRCS:.cpp=.release.o) $(STDMODULES_SRCS:.c=.o)
+OPT_OBJS := $(STDLIB_RELEASE_OBJS) $(SRCS:.cpp=.release.o) $(STDMODULES_SRCS:.c=.o)
 
 OPTIONAL_SRCS := codegen/profiling/oprofile.cpp codegen/profiling/pprof.cpp
 TOOL_SRCS := $(wildcard $(TOOLS_DIR)/*.cpp)
@@ -283,7 +285,7 @@ all: llvm
 	$(MAKE) pyston_all
 pyston_all: pyston_dbg pyston pyston_oprof pyston_prof $(OPTIONAL_SRCS:.cpp=.o)
 
-ALL_HEADERS := $(wildcard */*.h) $(wildcard */*/*.h)
+ALL_HEADERS := $(wildcard */*.h) $(wildcard */*/*.h) $(wildcard ../include/*.h)
 tags: $(SRCS) $(OPTIONAL_SRCS) $(ALL_HEADERS)
 	$(ECHO) Calculating tags...
 	$(VERB) ctags $^

src/core/stats.cpp

@@ -65,5 +65,13 @@ void Stats::dump() {
     }
 }
 
+void Stats::endOfInit() {
+    int orig_names = names->size();
+    for (int orig_id = 0; orig_id < orig_names; orig_id++) {
+        int init_id = getStatId("_init_" + (*names)[orig_id]);
+        log(init_id, (*counts)[orig_id]);
+    }
+};
+
 #endif
 }

src/core/stats.h

@@ -39,6 +39,7 @@ public:
     static void log(int id, int count = 1) { (*counts)[id] += count; }
 
     static void dump();
+    static void endOfInit();
 };
 
 struct StatCounter {

src/runtime/builtin_modules/errno.cpp

-// Copyright (c) 2014 Dropbox, Inc.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//    http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-#include <cmath>
-
-#include "core/types.h"
-#include "gc/collector.h"
-#include "runtime/inline/boxing.h"
-#include "runtime/types.h"
-#include "runtime/util.h"
-
-namespace pyston {
-
-BoxedModule* errno_module;
-
-void setupErrno() {
-    errno_module = createModule("errno", "__builtin__");
-}
-}

src/runtime/capi.cpp

@@ -21,20 +21,87 @@
 #include "codegen/compvars.h"
 #include "core/threading.h"
 #include "core/types.h"
+#include "runtime/objmodel.h"
 #include "runtime/types.h"
 
 namespace pyston {
 
-static BoxedModule* test_module = NULL;
+// A dictionary-like wrapper around the attributes array.
+// Not sure if this will be enough to satisfy users who expect __dict__
+// or PyModule_GetDict to return real dicts.
+BoxedClass* attrwrapper_cls;
+class AttrWrapper : public Box {
+private:
+    Box* b;
+
+public:
+    AttrWrapper(Box* b) : Box(attrwrapper_cls), b(b) {}
+
+    static void gcHandler(GCVisitor* v, Box* b) {
+        boxGCHandler(v, b);
+
+        AttrWrapper* aw = (AttrWrapper*)b;
+        v->visit(aw->b);
+    }
+
+    static Box* setitem(Box* _self, Box* _key, Box* value) {
+        assert(_self->cls == attrwrapper_cls);
+        AttrWrapper* self = static_cast<AttrWrapper*>(_self);
+
+        RELEASE_ASSERT(_key->cls == str_cls, "");
+        BoxedString* key = static_cast<BoxedString*>(_key);
+        self->b->setattr(key->s, value, NULL);
+
+        return None;
+    }
+};
+
+extern "C" PyObject* PyModule_GetDict(PyObject* _m) {
+    BoxedModule* m = static_cast<BoxedModule*>(_m);
+    assert(m->cls == module_cls);
+
+    return new AttrWrapper(m);
+}
+
+extern "C" PyObject* PyDict_New() {
+    return new BoxedDict();
+}
+
+extern "C" PyObject* PyString_FromString(const char* s) {
+    return boxStrConstant(s);
+}
+
+extern "C" PyObject* PyInt_FromLong(long n) {
+    return boxInt(n);
+}
+
+extern "C" int PyDict_SetItem(PyObject* mp, PyObject* _key, PyObject* _item) {
+    Box* b = static_cast<Box*>(mp);
+    Box* key = static_cast<Box*>(_key);
+    Box* item = static_cast<Box*>(_item);
+
+    static std::string setitem_str("__setitem__");
+    Box* r = callattrInternal(b, &setitem_str, CLASS_ONLY, NULL, ArgPassSpec(2), key, item, NULL, NULL, NULL);
+
+    RELEASE_ASSERT(r, "");
+    return 0;
+}
+
+extern "C" int PyDict_SetItemString(PyObject* mp, const char* key, PyObject* item) {
+    return PyDict_SetItem(mp, boxStrConstant(key), item);
+}
+
 
 BoxedClass* capifunc_cls;
 class BoxedCApiFunction : public Box {
 private:
+    Box* passthrough;
     const char* name;
     PyCFunction func;
 
 public:
-    BoxedCApiFunction(const char* name, PyCFunction func) : Box(capifunc_cls), name(name), func(func) {}
+    BoxedCApiFunction(Box* passthrough, const char* name, PyCFunction func)
+        : Box(capifunc_cls), passthrough(passthrough), name(name), func(func) {}
 
     static BoxedString* __repr__(BoxedCApiFunction* self) {
         assert(self->cls == capifunc_cls);
@@ -47,26 +114,34 @@ public:
 
         threading::GLPromoteRegion _gil_lock;
 
-        Box* rtn = (Box*)self->func(test_module, varargs);
+        Box* rtn = (Box*)self->func(self->passthrough, varargs);
         assert(rtn);
         return rtn;
     }
 };
 
-extern "C" void* Py_InitModule4(const char* arg0, PyMethodDef* arg1, const char* arg2, PyObject* arg3, int arg4) {
-    test_module = createModule("test", "../test/test_extension/test.so");
+extern "C" void* Py_InitModule4(const char* name, PyMethodDef* methods, const char* doc, PyObject* self, int apiver) {
+    BoxedModule* module = createModule(name, "__builtin__");
 
-    while (arg1->ml_name) {
+    Box* passthrough = static_cast<Box*>(self);
+    if (!passthrough)
+        passthrough = None;
+
+    while (methods->ml_name) {
         if (VERBOSITY())
-            printf("Loading method %s\n", arg1->ml_name);
-        assert(arg1->ml_flags == METH_VARARGS);
+            printf("Loading method %s\n", methods->ml_name);
+        assert(methods->ml_flags == METH_VARARGS);
+
+        module->giveAttr(methods->ml_name, new BoxedCApiFunction(passthrough, methods->ml_name, methods->ml_meth));
 
-        // test_module->giveAttr(arg1->ml_name, boxInt(1));
-        test_module->giveAttr(arg1->ml_name, new BoxedCApiFunction(arg1->ml_name, arg1->ml_meth));
+        methods++;
+    }
 
-        arg1++;
+    if (doc) {
+        module->setattr("__doc__", boxStrConstant(doc), NULL);
     }
-    return test_module;
+
+    return module;
 }
 
 extern "C" void* Py_BuildValue(const char* arg0) {
@@ -94,11 +169,9 @@ extern "C" bool PyArg_ParseTuple(void* tuple, const char* fmt, ...) {
     return true;
 }
 
-BoxedModule* getTestModule() {
-    if (test_module)
-        return test_module;
-
-    void* handle = dlopen("../test/test_extension/test.so", RTLD_NOW);
+BoxedModule* importTestExtension() {
+    const char* pathname = "../test/test_extension/test.so";
+    void* handle = dlopen(pathname, RTLD_NOW);
     if (!handle) {
         fprintf(stderr, "%s\n", dlerror());
         exit(1);
@@ -113,12 +186,19 @@ BoxedModule* getTestModule() {
         exit(1);
     }
 
-    // dlclose(handle);
-
     assert(init);
     (*init)();
-    assert(test_module);
-    return test_module;
+
+    BoxedDict* sys_modules = getSysModulesDict();
+    Box* s = boxStrConstant("test");
+    Box* _m = sys_modules->d[s];
+    RELEASE_ASSERT(_m, "module failed to initialize properly?");
+    assert(_m->cls == module_cls);
+
+    BoxedModule* m = static_cast<BoxedModule*>(_m);
+    m->setattr("__file__", boxStrConstant(pathname), NULL);
+    m->fn = pathname;
+    return m;
 }
 
 void setupCAPI() {
@@ -133,6 +213,11 @@ void setupCAPI() {
         "__call__", new BoxedFunction(boxRTFunction((void*)BoxedCApiFunction::__call__, UNKNOWN, 1, 0, true, false)));
 
     capifunc_cls->freeze();
+
+    attrwrapper_cls = new BoxedClass(object_cls, &AttrWrapper::gcHandler, 0, sizeof(AttrWrapper), false);
+    attrwrapper_cls->giveAttr("__name__", boxStrConstant("attrwrapper"));
+    attrwrapper_cls->giveAttr("__setitem__", new BoxedFunction(boxRTFunction((void*)AttrWrapper::setitem, UNKNOWN, 3)));
+    attrwrapper_cls->freeze();
 }
 
 void teardownCAPI() {

src/runtime/capi.h

@@ -18,7 +18,7 @@
 namespace pyston {
 
 class BoxedModule;
-BoxedModule* getTestModule();
+BoxedModule* importTestExtension();
 }
 
 #endif

src/runtime/objmodel.cpp

@@ -3159,7 +3159,7 @@ extern "C" Box* import(const std::string* name) {
     }
 
     if (*name == "test") {
-        return getTestModule();
+        return importTestExtension();
     }
 
     raiseExcHelper(ImportError, "No module named %s", name->c_str());

src/runtime/types.cpp

@@ -31,6 +31,8 @@
 #include "runtime/objmodel.h"
 #include "runtime/set.h"
 
+extern "C" void initerrno();
+
 namespace pyston {
 
 bool IN_SHUTDOWN = false;
@@ -617,14 +619,17 @@ void setupRuntime() {
     setupMath();
     setupTime();
     setupThread();
-    setupErrno();
     setupPosix();
     setupSre();
 
     setupCAPI();
 
+    initerrno();
+
     setupSysEnd();
 
+    Stats::endOfInit();
+
     TRACK_ALLOCATIONS = true;
 }
 

src/runtime/types.h

@@ -62,7 +62,6 @@ void setupBuiltins();
 void setupMath();
 void setupTime();
 void setupThread();
-void setupErrno();
 void setupPosix();
 void setupSre();
 void setupSysEnd();

test/tests/26.py

 # run_args: -n
-# statcheck: stats["slowpath_runtimecall"] <= 5
+# statcheck: noninit_count("slowpath_runtimecall") <= 5
 # statcheck: stats.get("slowpath_callclfunc", 0) <= 5
 # Simple patchpoint test:
 

test/tests/binop_ics.py

 # run_args: -n
-# statcheck: stats['slowpath_binop'] <= 10
-# statcheck: stats['slowpath_runtimecall'] <= 10
+# statcheck: noninit_count('slowpath_binop') <= 10
+# statcheck: noninit_count('slowpath_runtimecall') <= 10
 
 i = 1
 f = 1.0

test/tests/call_patches.py

 # run_args: -n
 # statcheck: stats.get('slowpath_callattr', 0) <= 20
-# statcheck: stats['slowpath_runtimecall'] <= 20
+# statcheck: noninit_count('slowpath_runtimecall') <= 20
 # statcheck: stats.get("slowpath_callclfunc", 0) <= 20
-# statcheck: stats['rewriter_nopatch'] <= 20
+# statcheck: noninit_count('rewriter_nopatch') <= 20
 
 def outer():
     def f():

test/tests/callattr.py

@@ -2,7 +2,7 @@
 # callattr: c.foo(i) should theoretically create an instancemethod object and then call that,
 # but calling an attribute is such a common case that I special case it as a callattr(),
 # which avoids the allocation/immediate deallocation of the instancemethod object.
-# statcheck: stats.get('num_instancemethods', 0) <= 10
+# statcheck: noninit_count('num_instancemethods') <= 10
 # statcheck: stats['slowpath_callattr'] <= 20
 class C(object):
     def foo(self, a0, a1, a2, a3, a4, a5, a6, a7):

test/tests/ctor_ics.py

 # run_args: -n
-# statcheck: stats['slowpath_runtimecall'] < 40
-# statcheck: stats['slowpath_typecall'] < 30
+# statcheck: noninit_count('slowpath_runtimecall') < 40
+# statcheck: noninit_count('slowpath_typecall') < 30
 def f():
     class B(object):
         def __init__(self):

test/tests/errno_test.py

+import errno
+print errno
+
+print errno.ENOENT
+print errno.__doc__

test/tests/im_ics.py

 # run_args: -n
-# statcheck: stats['slowpath_runtimecall'] < 10
+# statcheck: noninit_count('slowpath_runtimecall') < 10
 
 class C(object):
     def foo(self, a, b, c, d, e):

test/tests/int_ics.py

 # run_args: -n
-# statcheck: stats['slowpath_runtimecall'] < 10
+# statcheck: noninit_count('slowpath_runtimecall') < 10
 
 for i in xrange(1000):
     print int(i * 1.1)

tools/lint.py

@@ -42,6 +42,8 @@ def verify_include_order(_, dir, files):
             for l in f:
                 l = l.strip()
                 if l.startswith("//"):
+                    if "lint: allow-unsorted-includes" in l:
+                        break
                     continue
                 if not l:
                     if section:

tools/tester.py

@@ -213,6 +213,9 @@ def run_test(fn, check_stats, run_memcheck):
     r += "    Correct output (%5.1fms)" % (elapsed * 1000,)
 
     if check_stats:
+        def noninit_count(s):
+            return stats[s] - stats.get("_init_" + s, 0)
+
         for l in statchecks:
             test = eval(l)
             if not test: