add cpython buffer + sqlite3 module, and a couple tests.

.travis.yml

@@ -10,7 +10,7 @@ install:
   - sudo add-apt-repository --yes ppa:ubuntu-toolchain-r/test
   - sudo add-apt-repository --yes ppa:kubuntu-ppa/backports
   - sudo apt-get -qq update
-  - sudo apt-get install -yqq git cmake ninja-build libncurses5-dev liblzma-dev libreadline-dev libgmp3-dev autoconf libtool python-dev texlive-extra-utils
+  - sudo apt-get install -yqq git cmake ninja-build libsqlite3-dev libncurses5-dev liblzma-dev libreadline-dev libgmp3-dev autoconf libtool python-dev texlive-extra-utils
   - if [ "$CXX" = "clang++" ]; then sudo apt-get install -qq clang-3.4 libstdc++-4.8-dev; fi
   - if [ "$CXX" = "g++" ]; then sudo apt-get install -qq g++-4.8; fi
   - if [ "$CXX" = "g++" ]; then export CC="gcc-4.8" CXX="g++-4.8"; fi

CMakeLists.txt

@@ -169,7 +169,7 @@ add_subdirectory(tools)
 
 add_executable(pyston $<TARGET_OBJECTS:PYSTON_MAIN_OBJECT> $<TARGET_OBJECTS:PYSTON_OBJECTS> $<TARGET_OBJECTS:FROM_CPYTHON>)
 # Wrap the stdlib in --whole-archive to force all the symbols to be included and eventually exported
-target_link_libraries(pyston -Wl,--whole-archive stdlib -Wl,--no-whole-archive pthread m readline gmp ssl crypto unwind pypa double-conversion ${LLVM_LIBS} ${LIBLZMA_LIBRARIES} ${OPTIONAL_LIBRARIES})
+target_link_libraries(pyston -Wl,--whole-archive stdlib -Wl,--no-whole-archive pthread m readline sqlite3 gmp ssl crypto unwind pypa double-conversion ${LLVM_LIBS} ${LIBLZMA_LIBRARIES} ${OPTIONAL_LIBRARIES})
 
 # copy src/codegen/parse_ast.py to the build directory
 add_custom_command(TARGET pyston POST_BUILD COMMAND ${CMAKE_COMMAND} -E copy_if_different ${CMAKE_SOURCE_DIR}/src/codegen/parse_ast.py ${CMAKE_BINARY_DIR}/src/codegen/parse_ast.py)

Makefile

@@ -165,7 +165,7 @@ COMMON_CXXFLAGS += -DGITREV=$(shell git rev-parse HEAD | head -c 12) -DLLVMREV=$
 COMMON_CXXFLAGS += -DDEFAULT_PYTHON_MAJOR_VERSION=$(PYTHON_MAJOR_VERSION) -DDEFAULT_PYTHON_MINOR_VERSION=$(PYTHON_MINOR_VERSION) -DDEFAULT_PYTHON_MICRO_VERSION=$(PYTHON_MICRO_VERSION)
 
 # Use our "custom linker" that calls gold if available
-COMMON_LDFLAGS := -B$(TOOLS_DIR)/build_system -L/usr/local/lib -lpthread -lm -lunwind -llzma -L$(DEPS_DIR)/gcc-4.8.2-install/lib64 -lreadline -lgmp -lssl -lcrypto
+COMMON_LDFLAGS := -B$(TOOLS_DIR)/build_system -L/usr/local/lib -lpthread -lm -lunwind -llzma -L$(DEPS_DIR)/gcc-4.8.2-install/lib64 -lreadline -lgmp -lssl -lcrypto -lsqlite3
 COMMON_LDFLAGS += $(DEPS_DIR)/pypa-install/lib/libpypa.a
 
 # Conditionally add libtinfo if available - otherwise nothing will be added
@@ -335,6 +335,15 @@ STDMODULE_SRCS := \
 	_csv.c \
 	_ssl.c \
 	getpath.c \
+	_sqlite/cache.c \
+	_sqlite/connection.c \
+	_sqlite/cursor.c \
+	_sqlite/microprotocols.c \
+	_sqlite/module.c \
+	_sqlite/prepare_protocol.c \
+	_sqlite/row.c \
+	_sqlite/statement.c \
+	_sqlite/util.c \
 	$(EXTRA_STDMODULE_SRCS)
 
 STDOBJECT_SRCS := \
@@ -349,6 +358,7 @@ STDOBJECT_SRCS := \
 	weakrefobject.c \
 	memoryobject.c \
 	iterobject.c \
+	bufferobject.c \
 	$(EXTRA_STDOBJECT_SRCS)
 
 STDPYTHON_SRCS := \

docs/INSTALLING.md

@@ -19,7 +19,7 @@ git clone --recursive https://github.com/dropbox/pyston.git ~/pyston
 clang requires a fairly modern [host compiler](http://llvm.org/docs/GettingStarted.html#host-c-toolchain-both-compiler-and-standard-library), so typically you will have to install a new one.  The easiest thing to do is to just create a fresh build of GCC:
 
 ```
-sudo apt-get install libgmp-dev libmpfr-dev libmpc-dev make build-essential libtool zip gcc-multilib autogen
+sudo apt-get install libsqlite3-dev libgmp-dev libmpfr-dev libmpc-dev make build-essential libtool zip gcc-multilib autogen
 cd ~/pyston_deps
 wget http://ftpmirror.gnu.org/gcc/gcc-4.8.2/gcc-4.8.2.tar.bz2
 tar xvf gcc-4.8.2.tar.bz2

from_cpython/CMakeLists.txt

@@ -58,6 +58,15 @@ file(GLOB_RECURSE STDMODULE_SRCS Modules
     _csv.c
     _ssl.c
     getpath.c
+    cache.c
+    connection.c
+    cursor.c
+    microprotocols.c
+    module.c
+    prepare_protocol.c
+    row.c
+    statement.c
+    util.c
     )
 
 # compile specified files in from_cpython/Objects

from_cpython/Modules/_sqlite/cache.c

@@ -25,6 +25,9 @@
 #include "cache.h"
 #include <limits.h>
 
+// Pyston change: cpython supplies this as a cpp flag
+#define MODULE_NAME "sqlite3"
+
 /* only used internally */
 pysqlite_Node* pysqlite_new_node(PyObject* key, PyObject* data)
 {

from_cpython/Modules/_sqlite/connection.c

@@ -30,8 +30,12 @@
 #include "util.h"
 #include "sqlitecompat.h"
 
+#include "Python.h"
 #include "pythread.h"
 
+// Pyston change: cpython supplies this as a cpp flag
+#define MODULE_NAME "sqlite3"
+
 #define ACTION_FINALIZE 1
 #define ACTION_RESET 2
 

from_cpython/Modules/_sqlite/cursor.c

@@ -26,6 +26,9 @@
 #include "util.h"
 #include "sqlitecompat.h"
 
+// Pyston change: cpython supplies this as a cpp flag
+#define MODULE_NAME "sqlite3"
+
 PyObject* pysqlite_cursor_iternext(pysqlite_Cursor* self);
 
 static char* errmsg_fetch_across_rollback = "Cursor needed to be reset because of commit/rollback and can no longer be fetched from.";

from_cpython/Modules/_sqlite/module.c

@@ -29,6 +29,9 @@
 #include "microprotocols.h"
 #include "row.h"
 
+// Pyston change: cpython supplies this as a cpp flag
+#define MODULE_NAME "sqlite3"
+
 #if SQLITE_VERSION_NUMBER >= 3003003
 #define HAVE_SHARED_CACHE
 #endif
@@ -396,8 +399,11 @@ PyMODINIT_FUNC init_sqlite3(void)
      * need to be a string subclass. Just anything that can act as a special
      * marker for us. So I pulled PyCell_Type out of my magic hat.
      */
-    Py_INCREF((PyObject*)&PyCell_Type);
-    pysqlite_OptimizedUnicode = (PyObject*)&PyCell_Type;
+    // Pyston change: Pyston doesn't expose PyCell_Type, so let's use PyBuffer_Type instead.
+    // Py_INCREF((PyObject*)&PyCell_Type);
+    // pysqlite_OptimizedUnicode = (PyObject*)&PyCell_Type;
+    // PyDict_SetItemString(dict, "OptimizedUnicode", pysqlite_OptimizedUnicode);
+    pysqlite_OptimizedUnicode = (PyObject*)&PyBuffer_Type;
     PyDict_SetItemString(dict, "OptimizedUnicode", pysqlite_OptimizedUnicode);
 
     /* Set integer constants */

from_cpython/Modules/_sqlite/prepare_protocol.c

@@ -24,6 +24,9 @@
 #include "sqlitecompat.h"
 #include "prepare_protocol.h"
 
+// Pyston change: cpython supplies this as a cpp flag
+#define MODULE_NAME "sqlite3"
+
 int pysqlite_prepare_protocol_init(pysqlite_PrepareProtocol* self, PyObject* args, PyObject* kwargs)
 {
     return 0;

from_cpython/Modules/_sqlite/row.c

@@ -25,6 +25,9 @@
 #include "cursor.h"
 #include "sqlitecompat.h"
 
+// Pyston change: cpython supplies this as a cpp flag
+#define MODULE_NAME "sqlite3"
+
 void pysqlite_row_dealloc(pysqlite_Row* self)
 {
     Py_XDECREF(self->data);

from_cpython/Modules/_sqlite/statement.c

@@ -29,6 +29,9 @@
 #include "util.h"
 #include "sqlitecompat.h"
 
+// Pyston change: cpython supplies this as a cpp flag
+#define MODULE_NAME "sqlite3"
+
 /* prototypes */
 static int pysqlite_check_remaining_sql(const char* tail);
 

from_cpython/Objects/bufferobject.c

+
+/* Buffer object implementation */
+
+#include "Python.h"
+
+
+typedef struct {
+    PyObject_HEAD
+    PyObject *b_base;
+    void *b_ptr;
+    Py_ssize_t b_size;
+    Py_ssize_t b_offset;
+    int b_readonly;
+    long b_hash;
+} PyBufferObject;
+
+
+enum buffer_t {
+    READ_BUFFER,
+    WRITE_BUFFER,
+    CHAR_BUFFER,
+    ANY_BUFFER
+};
+
+static int
+get_buf(PyBufferObject *self, void **ptr, Py_ssize_t *size,
+    enum buffer_t buffer_type)
+{
+    if (self->b_base == NULL) {
+        assert (ptr != NULL);
+        *ptr = self->b_ptr;
+        *size = self->b_size;
+    }
+    else {
+        Py_ssize_t count, offset;
+        readbufferproc proc = 0;
+        PyBufferProcs *bp = self->b_base->ob_type->tp_as_buffer;
+        if ((*bp->bf_getsegcount)(self->b_base, NULL) != 1) {
+            PyErr_SetString(PyExc_TypeError,
+                "single-segment buffer object expected");
+            return 0;
+        }
+        if ((buffer_type == READ_BUFFER) ||
+            ((buffer_type == ANY_BUFFER) && self->b_readonly))
+            proc = bp->bf_getreadbuffer;
+        else if ((buffer_type == WRITE_BUFFER) ||
+            (buffer_type == ANY_BUFFER))
+            proc = (readbufferproc)bp->bf_getwritebuffer;
+        else if (buffer_type == CHAR_BUFFER) {
+            if (!PyType_HasFeature(self->ob_type,
+                        Py_TPFLAGS_HAVE_GETCHARBUFFER)) {
+            PyErr_SetString(PyExc_TypeError,
+                "Py_TPFLAGS_HAVE_GETCHARBUFFER needed");
+            return 0;
+            }
+            proc = (readbufferproc)bp->bf_getcharbuffer;
+        }
+        if (!proc) {
+            char *buffer_type_name;
+            switch (buffer_type) {
+            case READ_BUFFER:
+                buffer_type_name = "read";
+                break;
+            case WRITE_BUFFER:
+                buffer_type_name = "write";
+                break;
+            case CHAR_BUFFER:
+                buffer_type_name = "char";
+                break;
+            default:
+                buffer_type_name = "no";
+                break;
+            }
+            PyErr_Format(PyExc_TypeError,
+                "%s buffer type not available",
+                buffer_type_name);
+            return 0;
+        }
+        if ((count = (*proc)(self->b_base, 0, ptr)) < 0)
+            return 0;
+        /* apply constraints to the start/end */
+        if (self->b_offset > count)
+            offset = count;
+        else
+            offset = self->b_offset;
+        *(char **)ptr = *(char **)ptr + offset;
+        if (self->b_size == Py_END_OF_BUFFER)
+            *size = count;
+        else
+            *size = self->b_size;
+        if (*size > count - offset)
+            *size = count - offset;
+    }
+    return 1;
+}
+
+
+static PyObject *
+buffer_from_memory(PyObject *base, Py_ssize_t size, Py_ssize_t offset, void *ptr,
+                   int readonly)
+{
+    PyBufferObject * b;
+
+    if (size < 0 && size != Py_END_OF_BUFFER) {
+        PyErr_SetString(PyExc_ValueError,
+                        "size must be zero or positive");
+        return NULL;
+    }
+    if (offset < 0) {
+        PyErr_SetString(PyExc_ValueError,
+                        "offset must be zero or positive");
+        return NULL;
+    }
+
+    b = PyObject_NEW(PyBufferObject, &PyBuffer_Type);
+    if ( b == NULL )
+        return NULL;
+
+    Py_XINCREF(base);
+    b->b_base = base;
+    b->b_ptr = ptr;
+    b->b_size = size;
+    b->b_offset = offset;
+    b->b_readonly = readonly;
+    b->b_hash = -1;
+
+    return (PyObject *) b;
+}
+
+static PyObject *
+buffer_from_object(PyObject *base, Py_ssize_t size, Py_ssize_t offset, int readonly)
+{
+    if (offset < 0) {
+        PyErr_SetString(PyExc_ValueError,
+                        "offset must be zero or positive");
+        return NULL;
+    }
+    if ( PyBuffer_Check(base) && (((PyBufferObject *)base)->b_base) ) {
+        /* another buffer, refer to the base object */
+        PyBufferObject *b = (PyBufferObject *)base;
+        if (b->b_size != Py_END_OF_BUFFER) {
+            Py_ssize_t base_size = b->b_size - offset;
+            if (base_size < 0)
+                base_size = 0;
+            if (size == Py_END_OF_BUFFER || size > base_size)
+                size = base_size;
+        }
+        offset += b->b_offset;
+        base = b->b_base;
+    }
+    return buffer_from_memory(base, size, offset, NULL, readonly);
+}
+
+
+PyObject *
+PyBuffer_FromObject(PyObject *base, Py_ssize_t offset, Py_ssize_t size)
+{
+    PyBufferProcs *pb = base->ob_type->tp_as_buffer;
+
+    if ( pb == NULL ||
+         pb->bf_getreadbuffer == NULL ||
+         pb->bf_getsegcount == NULL )
+    {
+        PyErr_SetString(PyExc_TypeError, "buffer object expected");
+        return NULL;
+    }
+
+    return buffer_from_object(base, size, offset, 1);
+}
+
+PyObject *
+PyBuffer_FromReadWriteObject(PyObject *base, Py_ssize_t offset, Py_ssize_t size)
+{
+    PyBufferProcs *pb = base->ob_type->tp_as_buffer;
+
+    if ( pb == NULL ||
+         pb->bf_getwritebuffer == NULL ||
+         pb->bf_getsegcount == NULL )
+    {
+        PyErr_SetString(PyExc_TypeError, "buffer object expected");
+        return NULL;
+    }
+
+    return buffer_from_object(base, size,  offset, 0);
+}
+
+PyObject *
+PyBuffer_FromMemory(void *ptr, Py_ssize_t size)
+{
+    return buffer_from_memory(NULL, size, 0, ptr, 1);
+}
+
+PyObject *
+PyBuffer_FromReadWriteMemory(void *ptr, Py_ssize_t size)
+{
+    return buffer_from_memory(NULL, size, 0, ptr, 0);
+}
+
+PyObject *
+PyBuffer_New(Py_ssize_t size)
+{
+    PyObject *o;
+    PyBufferObject * b;
+
+    if (size < 0) {
+        PyErr_SetString(PyExc_ValueError,
+                        "size must be zero or positive");
+        return NULL;
+    }
+    if (sizeof(*b) > PY_SSIZE_T_MAX - size) {
+        /* unlikely */
+        return PyErr_NoMemory();
+    }
+    /* Inline PyObject_New */
+    o = (PyObject *)PyObject_MALLOC(sizeof(*b) + size);
+    if ( o == NULL )
+        return PyErr_NoMemory();
+    b = (PyBufferObject *) PyObject_INIT(o, &PyBuffer_Type);
+
+    b->b_base = NULL;
+    b->b_ptr = (void *)(b + 1);
+    b->b_size = size;
+    b->b_offset = 0;
+    b->b_readonly = 0;
+    b->b_hash = -1;
+
+    return o;
+}
+
+/* Methods */
+
+static PyObject *
+buffer_new(PyTypeObject *type, PyObject *args, PyObject *kw)
+{
+    PyObject *ob;
+    Py_ssize_t offset = 0;
+    Py_ssize_t size = Py_END_OF_BUFFER;
+
+    if (PyErr_WarnPy3k("buffer() not supported in 3.x", 1) < 0)
+        return NULL;
+
+    if (!_PyArg_NoKeywords("buffer()", kw))
+        return NULL;
+
+    if (!PyArg_ParseTuple(args, "O|nn:buffer", &ob, &offset, &size))
+        return NULL;
+    return PyBuffer_FromObject(ob, offset, size);
+}
+
+PyDoc_STRVAR(buffer_doc,
+"buffer(object [, offset[, size]])\n\
+\n\
+Create a new buffer object which references the given object.\n\
+The buffer will reference a slice of the target object from the\n\
+start of the object (or at the specified offset). The slice will\n\
+extend to the end of the target object (or with the specified size).");
+
+
+static void
+buffer_dealloc(PyBufferObject *self)
+{
+    Py_XDECREF(self->b_base);
+    PyObject_DEL(self);
+}
+
+static int
+buffer_compare(PyBufferObject *self, PyBufferObject *other)
+{
+    void *p1, *p2;
+    Py_ssize_t len_self, len_other, min_len;
+    int cmp;
+
+    if (!get_buf(self, &p1, &len_self, ANY_BUFFER))
+        return -1;
+    if (!get_buf(other, &p2, &len_other, ANY_BUFFER))
+        return -1;
+    min_len = (len_self < len_other) ? len_self : len_other;
+    if (min_len > 0) {
+        cmp = memcmp(p1, p2, min_len);
+        if (cmp != 0)
+            return cmp < 0 ? -1 : 1;
+    }
+    return (len_self < len_other) ? -1 : (len_self > len_other) ? 1 : 0;
+}
+
+static PyObject *
+buffer_repr(PyBufferObject *self)
+{
+    const char *status = self->b_readonly ? "read-only" : "read-write";
+
+    if ( self->b_base == NULL )
+        return PyString_FromFormat("<%s buffer ptr %p, size %zd at %p>",
+                                   status,
+                                   self->b_ptr,
+                                   self->b_size,
+                                   self);
+    else
+        return PyString_FromFormat(
+            "<%s buffer for %p, size %zd, offset %zd at %p>",
+            status,
+            self->b_base,
+            self->b_size,
+            self->b_offset,
+            self);
+}
+
+static long
+buffer_hash(PyBufferObject *self)
+{
+    void *ptr;
+    Py_ssize_t size;
+    register Py_ssize_t len;
+    register unsigned char *p;
+    register long x;
+
+    if ( self->b_hash != -1 )
+        return self->b_hash;
+
+    /* XXX potential bugs here, a readonly buffer does not imply that the
+     * underlying memory is immutable.  b_readonly is a necessary but not
+     * sufficient condition for a buffer to be hashable.  Perhaps it would
+     * be better to only allow hashing if the underlying object is known to
+     * be immutable (e.g. PyString_Check() is true).  Another idea would
+     * be to call tp_hash on the underlying object and see if it raises
+     * an error. */
+    if ( !self->b_readonly )
+    {
+        PyErr_SetString(PyExc_TypeError,
+                        "writable buffers are not hashable");
+        return -1;
+    }
+
+    if (!get_buf(self, &ptr, &size, ANY_BUFFER))
+        return -1;
+    p = (unsigned char *) ptr;
+    len = size;
+    /*
+      We make the hash of the empty buffer be 0, rather than using
+      (prefix ^ suffix), since this slightly obfuscates the hash secret
+    */
+    if (len == 0) {
+        self->b_hash = 0;
+        return 0;
+    }
+    x = _Py_HashSecret.prefix;
+    x ^= *p << 7;
+    while (--len >= 0)
+        x = (1000003*x) ^ *p++;
+    x ^= size;
+    x ^= _Py_HashSecret.suffix;
+    if (x == -1)
+        x = -2;
+    self->b_hash = x;
+    return x;
+}
+
+static PyObject *
+buffer_str(PyBufferObject *self)
+{
+    void *ptr;
+    Py_ssize_t size;
+    if (!get_buf(self, &ptr, &size, ANY_BUFFER))
+        return NULL;
+    return PyString_FromStringAndSize((const char *)ptr, size);
+}
+
+/* Sequence methods */
+
+static Py_ssize_t
+buffer_length(PyBufferObject *self)
+{
+    void *ptr;
+    Py_ssize_t size;
+    if (!get_buf(self, &ptr, &size, ANY_BUFFER))
+        return -1;
+    return size;
+}
+
+static PyObject *
+buffer_concat(PyBufferObject *self, PyObject *other)
+{
+    PyBufferProcs *pb = other->ob_type->tp_as_buffer;
+    void *ptr1, *ptr2;
+    char *p;
+    PyObject *ob;
+    Py_ssize_t size, count;
+
+    if ( pb == NULL ||
+         pb->bf_getreadbuffer == NULL ||
+         pb->bf_getsegcount == NULL )
+    {
+        PyErr_BadArgument();
+        return NULL;
+    }
+    if ( (*pb->bf_getsegcount)(other, NULL) != 1 )
+    {
+        /* ### use a different exception type/message? */
+        PyErr_SetString(PyExc_TypeError,
+                        "single-segment buffer object expected");
+        return NULL;
+    }
+
+    if (!get_buf(self, &ptr1, &size, ANY_BUFFER))
+        return NULL;
+
+    /* optimize special case */
+    if ( size == 0 )
+    {
+        Py_INCREF(other);
+        return other;
+    }
+
+    if ( (count = (*pb->bf_getreadbuffer)(other, 0, &ptr2)) < 0 )
+        return NULL;
+
+    // Pyston change: no PY_SIZE_MAX
+    // assert(count <= PY_SIZE_MAX - size);
+
+    ob = PyString_FromStringAndSize(NULL, size + count);
+    if ( ob == NULL )
+        return NULL;
+    p = PyString_AS_STRING(ob);
+    memcpy(p, ptr1, size);
+    memcpy(p + size, ptr2, count);
+
+    /* there is an extra byte in the string object, so this is safe */
+    p[size + count] = '\0';
+
+    return ob;
+}
+
+static PyObject *
+buffer_repeat(PyBufferObject *self, Py_ssize_t count)
+{
+    PyObject *ob;
+    register char *p;
+    void *ptr;
+    Py_ssize_t size;
+
+    if ( count < 0 )
+        count = 0;
+    if (!get_buf(self, &ptr, &size, ANY_BUFFER))
+        return NULL;
+    if (count > PY_SSIZE_T_MAX / size) {
+        PyErr_SetString(PyExc_MemoryError, "result too large");
+        return NULL;
+    }
+    ob = PyString_FromStringAndSize(NULL, size * count);
+    if ( ob == NULL )
+        return NULL;
+
+    p = PyString_AS_STRING(ob);
+    while ( count-- )
+    {
+        memcpy(p, ptr, size);
+        p += size;
+    }
+
+    /* there is an extra byte in the string object, so this is safe */
+    *p = '\0';
+
+    return ob;
+}
+
+static PyObject *
+buffer_item(PyBufferObject *self, Py_ssize_t idx)
+{
+    void *ptr;
+    Py_ssize_t size;
+    if (!get_buf(self, &ptr, &size, ANY_BUFFER))
+        return NULL;
+    if ( idx < 0 || idx >= size ) {
+        PyErr_SetString(PyExc_IndexError, "buffer index out of range");
+        return NULL;
+    }
+    return PyString_FromStringAndSize((char *)ptr + idx, 1);
+}
+
+static PyObject *
+buffer_slice(PyBufferObject *self, Py_ssize_t left, Py_ssize_t right)
+{
+    void *ptr;
+    Py_ssize_t size;
+    if (!get_buf(self, &ptr, &size, ANY_BUFFER))
+        return NULL;
+    if ( left < 0 )
+        left = 0;
+    if ( right < 0 )
+        right = 0;
+    if ( right > size )
+        right = size;
+    if ( right < left )
+        right = left;
+    return PyString_FromStringAndSize((char *)ptr + left,
+                                      right - left);
+}
+
+static PyObject *
+buffer_subscript(PyBufferObject *self, PyObject *item)
+{
+    void *p;
+    Py_ssize_t size;
+
+    if (!get_buf(self, &p, &size, ANY_BUFFER))
+        return NULL;
+    if (PyIndex_Check(item)) {
+        Py_ssize_t i = PyNumber_AsSsize_t(item, PyExc_IndexError);
+        if (i == -1 && PyErr_Occurred())
+            return NULL;
+        if (i < 0)
+            i += size;
+        return buffer_item(self, i);
+    }
+    else if (PySlice_Check(item)) {
+        Py_ssize_t start, stop, step, slicelength, cur, i;
+
+        if (PySlice_GetIndicesEx((PySliceObject*)item, size,
+                         &start, &stop, &step, &slicelength) < 0) {
+            return NULL;
+        }
+
+        if (slicelength <= 0)
+            return PyString_FromStringAndSize("", 0);
+        else if (step == 1)
+            return PyString_FromStringAndSize((char *)p + start,
+                                              stop - start);
+        else {
+            PyObject *result;
+            char *source_buf = (char *)p;
+            char *result_buf = (char *)PyMem_Malloc(slicelength);
+
+            if (result_buf == NULL)
+                return PyErr_NoMemory();
+
+            for (cur = start, i = 0; i < slicelength;
+                 cur += step, i++) {
+                result_buf[i] = source_buf[cur];
+            }
+
+            result = PyString_FromStringAndSize(result_buf,
+                                                slicelength);
+            PyMem_Free(result_buf);
+            return result;
+        }
+    }
+    else {
+        PyErr_SetString(PyExc_TypeError,
+                        "sequence index must be integer");
+        return NULL;
+    }
+}
+
+static int
+buffer_ass_item(PyBufferObject *self, Py_ssize_t idx, PyObject *other)
+{
+    PyBufferProcs *pb;
+    void *ptr1, *ptr2;
+    Py_ssize_t size;
+    Py_ssize_t count;
+
+    if ( self->b_readonly ) {
+        PyErr_SetString(PyExc_TypeError,
+                        "buffer is read-only");
+        return -1;
+    }
+
+    if (!get_buf(self, &ptr1, &size, ANY_BUFFER))
+        return -1;
+
+    if (idx < 0 || idx >= size) {
+        PyErr_SetString(PyExc_IndexError,
+                        "buffer assignment index out of range");
+        return -1;
+    }
+
+    pb = other ? other->ob_type->tp_as_buffer : NULL;
+    if ( pb == NULL ||
+         pb->bf_getreadbuffer == NULL ||
+         pb->bf_getsegcount == NULL )
+    {
+        PyErr_BadArgument();
+        return -1;
+    }
+    if ( (*pb->bf_getsegcount)(other, NULL) != 1 )
+    {
+        /* ### use a different exception type/message? */
+        PyErr_SetString(PyExc_TypeError,
+                        "single-segment buffer object expected");
+        return -1;
+    }
+
+    if ( (count = (*pb->bf_getreadbuffer)(other, 0, &ptr2)) < 0 )
+        return -1;
+    if ( count != 1 ) {
+        PyErr_SetString(PyExc_TypeError,
+                        "right operand must be a single byte");
+        return -1;
+    }
+
+    ((char *)ptr1)[idx] = *(char *)ptr2;
+    return 0;
+}
+
+static int
+buffer_ass_slice(PyBufferObject *self, Py_ssize_t left, Py_ssize_t right, PyObject *other)
+{
+    PyBufferProcs *pb;
+    void *ptr1, *ptr2;
+    Py_ssize_t size;
+    Py_ssize_t slice_len;
+    Py_ssize_t count;
+
+    if ( self->b_readonly ) {
+        PyErr_SetString(PyExc_TypeError,
+                        "buffer is read-only");
+        return -1;
+    }
+
+    pb = other ? other->ob_type->tp_as_buffer : NULL;
+    if ( pb == NULL ||
+         pb->bf_getreadbuffer == NULL ||
+         pb->bf_getsegcount == NULL )
+    {
+        PyErr_BadArgument();
+        return -1;
+    }
+    if ( (*pb->bf_getsegcount)(other, NULL) != 1 )
+    {
+        /* ### use a different exception type/message? */
+        PyErr_SetString(PyExc_TypeError,
+                        "single-segment buffer object expected");
+        return -1;
+    }
+    if (!get_buf(self, &ptr1, &size, ANY_BUFFER))
+        return -1;
+    if ( (count = (*pb->bf_getreadbuffer)(other, 0, &ptr2)) < 0 )
+        return -1;
+
+    if ( left < 0 )
+        left = 0;
+    else if ( left > size )
+        left = size;
+    if ( right < left )
+        right = left;
+    else if ( right > size )
+        right = size;
+    slice_len = right - left;
+
+    if ( count != slice_len ) {
+        PyErr_SetString(
+            PyExc_TypeError,
+            "right operand length must match slice length");
+        return -1;
+    }
+
+    if ( slice_len )
+        memcpy((char *)ptr1 + left, ptr2, slice_len);
+
+    return 0;
+}
+
+static int
+buffer_ass_subscript(PyBufferObject *self, PyObject *item, PyObject *value)
+{
+    PyBufferProcs *pb;
+    void *ptr1, *ptr2;
+    Py_ssize_t selfsize;
+    Py_ssize_t othersize;
+
+    if ( self->b_readonly ) {
+        PyErr_SetString(PyExc_TypeError,
+                        "buffer is read-only");
+        return -1;
+    }
+
+    pb = value ? value->ob_type->tp_as_buffer : NULL;
+    if ( pb == NULL ||
+         pb->bf_getreadbuffer == NULL ||
+         pb->bf_getsegcount == NULL )
+    {
+        PyErr_BadArgument();
+        return -1;
+    }
+    if ( (*pb->bf_getsegcount)(value, NULL) != 1 )
+    {
+        /* ### use a different exception type/message? */
+        PyErr_SetString(PyExc_TypeError,
+                        "single-segment buffer object expected");
+        return -1;
+    }
+    if (!get_buf(self, &ptr1, &selfsize, ANY_BUFFER))
+        return -1;
+    if (PyIndex_Check(item)) {
+        Py_ssize_t i = PyNumber_AsSsize_t(item, PyExc_IndexError);
+        if (i == -1 && PyErr_Occurred())
+            return -1;
+        if (i < 0)
+            i += selfsize;
+        return buffer_ass_item(self, i, value);
+    }
+    else if (PySlice_Check(item)) {
+        Py_ssize_t start, stop, step, slicelength;
+
+        if (PySlice_GetIndicesEx((PySliceObject *)item, selfsize,
+                        &start, &stop, &step, &slicelength) < 0)
+            return -1;
+
+        if ((othersize = (*pb->bf_getreadbuffer)(value, 0, &ptr2)) < 0)
+            return -1;
+
+        if (othersize != slicelength) {
+            PyErr_SetString(
+                PyExc_TypeError,
+                "right operand length must match slice length");
+            return -1;
+        }
+
+        if (slicelength == 0)
+            return 0;
+        else if (step == 1) {
+            memcpy((char *)ptr1 + start, ptr2, slicelength);
+            return 0;
+        }
+        else {
+            Py_ssize_t cur, i;
+
+            for (cur = start, i = 0; i < slicelength;
+                 cur += step, i++) {
+                ((char *)ptr1)[cur] = ((char *)ptr2)[i];
+            }
+
+            return 0;
+        }
+    } else {
+        PyErr_SetString(PyExc_TypeError,
+                        "buffer indices must be integers");
+        return -1;
+    }
+}
+
+/* Buffer methods */
+
+static Py_ssize_t
+buffer_getreadbuf(PyBufferObject *self, Py_ssize_t idx, void **pp)
+{
+    Py_ssize_t size;
+    if ( idx != 0 ) {
+        PyErr_SetString(PyExc_SystemError,
+                        "accessing non-existent buffer segment");
+        return -1;
+    }
+    if (!get_buf(self, pp, &size, READ_BUFFER))
+        return -1;
+    return size;
+}
+
+static Py_ssize_t
+buffer_getwritebuf(PyBufferObject *self, Py_ssize_t idx, void **pp)
+{
+    Py_ssize_t size;
+
+    if ( self->b_readonly )
+    {
+        PyErr_SetString(PyExc_TypeError, "buffer is read-only");
+        return -1;
+    }
+
+    if ( idx != 0 ) {
+        PyErr_SetString(PyExc_SystemError,
+                        "accessing non-existent buffer segment");
+        return -1;
+    }
+    if (!get_buf(self, pp, &size, WRITE_BUFFER))
+        return -1;
+    return size;
+}
+
+static Py_ssize_t
+buffer_getsegcount(PyBufferObject *self, Py_ssize_t *lenp)
+{
+    void *ptr;
+    Py_ssize_t size;
+    if (!get_buf(self, &ptr, &size, ANY_BUFFER))
+        return -1;
+    if (lenp)
+        *lenp = size;
+    return 1;
+}
+
+static Py_ssize_t
+buffer_getcharbuf(PyBufferObject *self, Py_ssize_t idx, const char **pp)
+{
+    void *ptr;
+    Py_ssize_t size;
+    if ( idx != 0 ) {
+        PyErr_SetString(PyExc_SystemError,
+                        "accessing non-existent buffer segment");
+        return -1;
+    }
+    if (!get_buf(self, &ptr, &size, CHAR_BUFFER))
+        return -1;
+    *pp = (const char *)ptr;
+    return size;
+}
+
+static int buffer_getbuffer(PyBufferObject *self, Py_buffer *buf, int flags)
+{
+    void *ptr;
+    Py_ssize_t size;
+    if (!get_buf(self, &ptr, &size, ANY_BUFFER))
+        return -1;
+    return PyBuffer_FillInfo(buf, (PyObject*)self, ptr, size,
+                             self->b_readonly, flags);
+}
+
+static PySequenceMethods buffer_as_sequence = {
+    (lenfunc)buffer_length, /*sq_length*/
+    (binaryfunc)buffer_concat, /*sq_concat*/
+    (ssizeargfunc)buffer_repeat, /*sq_repeat*/
+    (ssizeargfunc)buffer_item, /*sq_item*/
+    (ssizessizeargfunc)buffer_slice, /*sq_slice*/
+    (ssizeobjargproc)buffer_ass_item, /*sq_ass_item*/
+    (ssizessizeobjargproc)buffer_ass_slice, /*sq_ass_slice*/
+};
+
+static PyMappingMethods buffer_as_mapping = {
+    (lenfunc)buffer_length,
+    (binaryfunc)buffer_subscript,
+    (objobjargproc)buffer_ass_subscript,
+};
+
+static PyBufferProcs buffer_as_buffer = {
+    (readbufferproc)buffer_getreadbuf,
+    (writebufferproc)buffer_getwritebuf,
+    (segcountproc)buffer_getsegcount,
+    (charbufferproc)buffer_getcharbuf,
+    (getbufferproc)buffer_getbuffer,
+};
+
+PyTypeObject PyBuffer_Type = {
+    // Pyston change:
+    // PyVarObject_HEAD_INIT(&PyType_Type, 0)
+    PyVarObject_HEAD_INIT(NULL, 0)
+    "buffer",
+    sizeof(PyBufferObject),
+    0,
+    (destructor)buffer_dealloc,                 /* tp_dealloc */
+    0,                                          /* tp_print */
+    0,                                          /* tp_getattr */
+    0,                                          /* tp_setattr */
+    // Pyston change:
+    //(cmpfunc)buffer_compare,                    /* tp_compare */
+    NULL,                                       /* tp_compare */
+    (reprfunc)buffer_repr,                      /* tp_repr */
+    0,                                          /* tp_as_number */
+    &buffer_as_sequence,                        /* tp_as_sequence */
+    &buffer_as_mapping,                         /* tp_as_mapping */
+    (hashfunc)buffer_hash,                      /* tp_hash */
+    0,                                          /* tp_call */
+    (reprfunc)buffer_str,                       /* tp_str */
+    PyObject_GenericGetAttr,                    /* tp_getattro */
+    0,                                          /* tp_setattro */
+    &buffer_as_buffer,                          /* tp_as_buffer */
+    Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GETCHARBUFFER | Py_TPFLAGS_HAVE_NEWBUFFER, /* tp_flags */
+    buffer_doc,                                 /* tp_doc */
+    0,                                          /* tp_traverse */
+    0,                                          /* tp_clear */
+    0,                                          /* tp_richcompare */
+    0,                                          /* tp_weaklistoffset */
+    0,                                          /* tp_iter */
+    0,                                          /* tp_iternext */
+    0,                                          /* tp_methods */
+    0,                                          /* tp_members */
+    0,                                          /* tp_getset */
+    0,                                          /* tp_base */
+    0,                                          /* tp_dict */
+    0,                                          /* tp_descr_get */
+    0,                                          /* tp_descr_set */
+    0,                                          /* tp_dictoffset */
+    0,                                          /* tp_init */
+    0,                                          /* tp_alloc */
+    buffer_new,                                 /* tp_new */
+};

src/runtime/builtin_modules/builtins.cpp

@@ -1176,13 +1176,13 @@ void setupBuiltins() {
     builtins_module->giveAttr("memoryview", memoryview_cls);
     PyType_Ready(&PyByteArray_Type);
     builtins_module->giveAttr("bytearray", &PyByteArray_Type);
+    Py_TYPE(&PyBuffer_Type) = &PyType_Type;
+    PyType_Ready(&PyBuffer_Type);
+    builtins_module->giveAttr("buffer", &PyBuffer_Type);
 
     builtins_module->giveAttr(
         "eval", new BoxedBuiltinFunctionOrMethod(boxRTFunction((void*)eval, UNKNOWN, 1, 0, false, false), "eval"));
     builtins_module->giveAttr("callable",
                               new BoxedBuiltinFunctionOrMethod(boxRTFunction((void*)callable, UNKNOWN, 1), "callable"));
-
-    BoxedClass* buffer_cls = BoxedHeapClass::create(type_cls, object_cls, NULL, 0, 0, sizeof(Box), false, "buffer");
-    builtins_module->giveAttr("buffer", buffer_cls);
 }
 }

src/runtime/capi.cpp

@@ -593,6 +593,18 @@ finally:
     --tstate->recursion_depth;
 }
 
+extern "C" PyGILState_STATE PyGILState_Ensure(void) noexcept {
+    Py_FatalError("unimplemented");
+}
+
+extern "C" void PyGILState_Release(PyGILState_STATE) noexcept {
+    Py_FatalError("unimplemented");
+}
+
+extern "C" PyThreadState* PyGILState_GetThisThreadState(void) noexcept {
+    Py_FatalError("unimplemented");
+}
+
 void setCAPIException(const ExcInfo& e) {
     cur_thread_state.curexc_type = e.type;
     cur_thread_state.curexc_value = e.value;
@@ -1173,14 +1185,14 @@ extern "C" int _PyEval_SliceIndex(PyObject* v, Py_ssize_t* pi) noexcept {
     return 1;
 }
 
-extern "C" PyObject* PyBuffer_FromMemory(void* ptr, Py_ssize_t size) noexcept {
-    Py_FatalError("unimplemented");
-}
-
 extern "C" int PyEval_GetRestricted(void) noexcept {
     return 0; // We don't support restricted mode
 }
 
+extern "C" void PyEval_InitThreads(void) noexcept {
+    // nothing to do here
+}
+
 BoxedModule* importTestExtension(const std::string& name) {
     std::string pathname_name = "test/test_extension/" + name + ".pyston.so";
     const char* pathname = pathname_name.c_str();

src/runtime/long.cpp

@@ -97,6 +97,10 @@ extern "C" PY_LONG_LONG PyLong_AsLongLong(PyObject* vv) noexcept {
     Py_FatalError("unimplemented");
 }
 
+extern "C" PY_LONG_LONG PyLong_AsLongLongAndOverflow(PyObject* obj, int* overflow) noexcept {
+    Py_FatalError("unimplemented");
+}
+
 extern "C" PyObject* PyLong_FromString(const char* str, char** pend, int base) noexcept {
     Py_FatalError("unimplemented");
 }

src/runtime/types.cpp

@@ -74,6 +74,7 @@ extern "C" void init_io();
 extern "C" void initzipimport();
 extern "C" void init_csv();
 extern "C" void init_ssl();
+extern "C" void init_sqlite3();
 
 namespace pyston {
 
@@ -1985,6 +1986,7 @@ void setupRuntime() {
     initzipimport();
     init_csv();
     init_ssl();
+    init_sqlite3();
 
     // some additional setup to ensure weakrefs participate in our GC
     BoxedClass* weakref_ref_cls = &_PyWeakref_RefType;

test/tests/buffer_test.py

+s = 'Hello world'
+t = buffer(s, 6, 5)
+print t

test/tests/sqlite_test.py

+import os
+try:
+    os.unlink('example.db')
+except Exception:
+    pass
+
+import sqlite3
+conn = sqlite3.connect('example.db')
+
+c = conn.cursor()
+
+# Create table
+c.execute('''CREATE TABLE stocks
+             (date text, trans text, symbol text, qty real, price real)''')
+
+# Insert a row of data
+c.execute("INSERT INTO stocks VALUES ('2006-01-05','BUY','RHAT',100,35.14)")
+
+# Save (commit) the changes
+conn.commit()
+
+# We can also close the connection if we are done with it.
+# Just be sure any changes have been committed or they will be lost.
+conn.close()
+
+
+
+conn = sqlite3.connect('example.db')
+c = conn.cursor()
+
+# Never do this -- insecure!
+symbol = 'RHAT'
+c.execute("SELECT * FROM stocks WHERE symbol = '%s'" % symbol)
+
+# Do this instead
+t = ('RHAT',)
+c.execute('SELECT * FROM stocks WHERE symbol=?', t)
+print c.fetchone()
+
+# Larger example that inserts many records at a time
+purchases = [('2006-03-28', 'BUY', 'IBM', 1000, 45.00),
+             ('2006-04-05', 'BUY', 'MSFT', 1000, 72.00),
+             ('2006-04-06', 'SELL', 'IBM', 500, 53.00),
+            ]
+c.executemany('INSERT INTO stocks VALUES (?,?,?,?,?)', purchases)
+
+for row in c.execute('SELECT * FROM stocks ORDER BY price'):
+    print row