Merge pull request #379 from toshok/sqlite3

add cpython buffer + sqlite3 module, and a couple tests.

.travis.yml

@@ -10,7 +10,7 @@ install:
   - sudo add-apt-repository --yes ppa:ubuntu-toolchain-r/test
   - sudo add-apt-repository --yes ppa:kubuntu-ppa/backports
   - sudo apt-get -qq update
-  - sudo apt-get install -yqq git cmake ninja-build libncurses5-dev liblzma-dev libreadline-dev libgmp3-dev autoconf libtool python-dev texlive-extra-utils
+  - sudo apt-get install -yqq git cmake ninja-build libsqlite3-dev libncurses5-dev liblzma-dev libreadline-dev libgmp3-dev autoconf libtool python-dev texlive-extra-utils
   - if [ "$CXX" = "clang++" ]; then sudo apt-get install -qq clang-3.4 libstdc++-4.8-dev; fi
   - if [ "$CXX" = "g++" ]; then sudo apt-get install -qq g++-4.8; fi
   - if [ "$CXX" = "g++" ]; then export CC="gcc-4.8" CXX="g++-4.8"; fi

CMakeLists.txt

@@ -169,7 +169,7 @@ add_subdirectory(tools)
 
 add_executable(pyston $<TARGET_OBJECTS:PYSTON_MAIN_OBJECT> $<TARGET_OBJECTS:PYSTON_OBJECTS> $<TARGET_OBJECTS:FROM_CPYTHON>)
 # Wrap the stdlib in --whole-archive to force all the symbols to be included and eventually exported
-target_link_libraries(pyston -Wl,--whole-archive stdlib -Wl,--no-whole-archive pthread m readline gmp ssl crypto unwind pypa double-conversion ${LLVM_LIBS} ${LIBLZMA_LIBRARIES} ${OPTIONAL_LIBRARIES})
+target_link_libraries(pyston -Wl,--whole-archive stdlib -Wl,--no-whole-archive pthread m readline sqlite3 gmp ssl crypto unwind pypa double-conversion ${LLVM_LIBS} ${LIBLZMA_LIBRARIES} ${OPTIONAL_LIBRARIES})
 
 # copy src/codegen/parse_ast.py to the build directory
 add_custom_command(TARGET pyston POST_BUILD COMMAND ${CMAKE_COMMAND} -E copy_if_different ${CMAKE_SOURCE_DIR}/src/codegen/parse_ast.py ${CMAKE_BINARY_DIR}/src/codegen/parse_ast.py)

Makefile

@@ -165,7 +165,7 @@ COMMON_CXXFLAGS += -DGITREV=$(shell git rev-parse HEAD | head -c 12) -DLLVMREV=$
 COMMON_CXXFLAGS += -DDEFAULT_PYTHON_MAJOR_VERSION=$(PYTHON_MAJOR_VERSION) -DDEFAULT_PYTHON_MINOR_VERSION=$(PYTHON_MINOR_VERSION) -DDEFAULT_PYTHON_MICRO_VERSION=$(PYTHON_MICRO_VERSION)
 
 # Use our "custom linker" that calls gold if available
-COMMON_LDFLAGS := -B$(TOOLS_DIR)/build_system -L/usr/local/lib -lpthread -lm -lunwind -llzma -L$(DEPS_DIR)/gcc-4.8.2-install/lib64 -lreadline -lgmp -lssl -lcrypto
+COMMON_LDFLAGS := -B$(TOOLS_DIR)/build_system -L/usr/local/lib -lpthread -lm -lunwind -llzma -L$(DEPS_DIR)/gcc-4.8.2-install/lib64 -lreadline -lgmp -lssl -lcrypto -lsqlite3
 COMMON_LDFLAGS += $(DEPS_DIR)/pypa-install/lib/libpypa.a
 
 # Conditionally add libtinfo if available - otherwise nothing will be added
@@ -335,6 +335,15 @@ STDMODULE_SRCS := \
 	_csv.c \
 	_ssl.c \
 	getpath.c \
+	_sqlite/cache.c \
+	_sqlite/connection.c \
+	_sqlite/cursor.c \
+	_sqlite/microprotocols.c \
+	_sqlite/module.c \
+	_sqlite/prepare_protocol.c \
+	_sqlite/row.c \
+	_sqlite/statement.c \
+	_sqlite/util.c \
 	$(EXTRA_STDMODULE_SRCS)
 
 STDOBJECT_SRCS := \
@@ -349,6 +358,7 @@ STDOBJECT_SRCS := \
 	weakrefobject.c \
 	memoryobject.c \
 	iterobject.c \
+	bufferobject.c \
 	$(EXTRA_STDOBJECT_SRCS)
 
 STDPYTHON_SRCS := \

docs/INSTALLING.md

@@ -19,7 +19,7 @@ git clone --recursive https://github.com/dropbox/pyston.git ~/pyston
 clang requires a fairly modern [host compiler](http://llvm.org/docs/GettingStarted.html#host-c-toolchain-both-compiler-and-standard-library), so typically you will have to install a new one.  The easiest thing to do is to just create a fresh build of GCC:
 
 ```
-sudo apt-get install libgmp-dev libmpfr-dev libmpc-dev make build-essential libtool zip gcc-multilib autogen
+sudo apt-get install libsqlite3-dev libgmp-dev libmpfr-dev libmpc-dev make build-essential libtool zip gcc-multilib autogen
 cd ~/pyston_deps
 wget http://ftpmirror.gnu.org/gcc/gcc-4.8.2/gcc-4.8.2.tar.bz2
 tar xvf gcc-4.8.2.tar.bz2

from_cpython/CMakeLists.txt

@@ -58,6 +58,15 @@ file(GLOB_RECURSE STDMODULE_SRCS Modules
     _csv.c
     _ssl.c
     getpath.c
+    cache.c
+    connection.c
+    cursor.c
+    microprotocols.c
+    module.c
+    prepare_protocol.c
+    row.c
+    statement.c
+    util.c
     )
 
 # compile specified files in from_cpython/Objects
@@ -73,6 +82,7 @@ file(GLOB_RECURSE STDOBJECT_SRCS Objects
     weakrefobject.c
     memoryobject.c
     iterobject.c
+    bufferobject.c
     )
 
 # compile specified files in from_cpython/Python

from_cpython/Modules/_sqlite/cache.c

@@ -25,6 +25,9 @@
 #include "cache.h"
 #include <limits.h>
 
+// Pyston change: cpython supplies this as a cpp flag
+#define MODULE_NAME "sqlite3"
+
 /* only used internally */
 pysqlite_Node* pysqlite_new_node(PyObject* key, PyObject* data)
 {

from_cpython/Modules/_sqlite/connection.c

@@ -30,8 +30,12 @@
 #include "util.h"
 #include "sqlitecompat.h"
 
+#include "Python.h"
 #include "pythread.h"
 
+// Pyston change: cpython supplies this as a cpp flag
+#define MODULE_NAME "sqlite3"
+
 #define ACTION_FINALIZE 1
 #define ACTION_RESET 2
 

from_cpython/Modules/_sqlite/cursor.c

@@ -26,6 +26,9 @@
 #include "util.h"
 #include "sqlitecompat.h"
 
+// Pyston change: cpython supplies this as a cpp flag
+#define MODULE_NAME "sqlite3"
+
 PyObject* pysqlite_cursor_iternext(pysqlite_Cursor* self);
 
 static char* errmsg_fetch_across_rollback = "Cursor needed to be reset because of commit/rollback and can no longer be fetched from.";

from_cpython/Modules/_sqlite/module.c

@@ -29,6 +29,9 @@
 #include "microprotocols.h"
 #include "row.h"
 
+// Pyston change: cpython supplies this as a cpp flag
+#define MODULE_NAME "sqlite3"
+
 #if SQLITE_VERSION_NUMBER >= 3003003
 #define HAVE_SHARED_CACHE
 #endif
@@ -396,8 +399,11 @@ PyMODINIT_FUNC init_sqlite3(void)
      * need to be a string subclass. Just anything that can act as a special
      * marker for us. So I pulled PyCell_Type out of my magic hat.
      */
-    Py_INCREF((PyObject*)&PyCell_Type);
-    pysqlite_OptimizedUnicode = (PyObject*)&PyCell_Type;
+    // Pyston change: Pyston doesn't expose PyCell_Type, so let's use PyBuffer_Type instead.
+    // Py_INCREF((PyObject*)&PyCell_Type);
+    // pysqlite_OptimizedUnicode = (PyObject*)&PyCell_Type;
+    // PyDict_SetItemString(dict, "OptimizedUnicode", pysqlite_OptimizedUnicode);
+    pysqlite_OptimizedUnicode = (PyObject*)&PyBuffer_Type;
     PyDict_SetItemString(dict, "OptimizedUnicode", pysqlite_OptimizedUnicode);
 
     /* Set integer constants */

from_cpython/Modules/_sqlite/prepare_protocol.c

@@ -24,6 +24,9 @@
 #include "sqlitecompat.h"
 #include "prepare_protocol.h"
 
+// Pyston change: cpython supplies this as a cpp flag
+#define MODULE_NAME "sqlite3"
+
 int pysqlite_prepare_protocol_init(pysqlite_PrepareProtocol* self, PyObject* args, PyObject* kwargs)
 {
     return 0;

from_cpython/Modules/_sqlite/row.c

@@ -25,6 +25,9 @@
 #include "cursor.h"
 #include "sqlitecompat.h"
 
+// Pyston change: cpython supplies this as a cpp flag
+#define MODULE_NAME "sqlite3"
+
 void pysqlite_row_dealloc(pysqlite_Row* self)
 {
     Py_XDECREF(self->data);

from_cpython/Modules/_sqlite/statement.c

@@ -29,6 +29,9 @@
 #include "util.h"
 #include "sqlitecompat.h"
 
+// Pyston change: cpython supplies this as a cpp flag
+#define MODULE_NAME "sqlite3"
+
 /* prototypes */
 static int pysqlite_check_remaining_sql(const char* tail);
 

src/runtime/builtin_modules/builtins.cpp

@@ -1176,13 +1176,13 @@ void setupBuiltins() {
     builtins_module->giveAttr("memoryview", memoryview_cls);
     PyType_Ready(&PyByteArray_Type);
     builtins_module->giveAttr("bytearray", &PyByteArray_Type);
+    Py_TYPE(&PyBuffer_Type) = &PyType_Type;
+    PyType_Ready(&PyBuffer_Type);
+    builtins_module->giveAttr("buffer", &PyBuffer_Type);
 
     builtins_module->giveAttr(
         "eval", new BoxedBuiltinFunctionOrMethod(boxRTFunction((void*)eval, UNKNOWN, 1, 0, false, false), "eval"));
     builtins_module->giveAttr("callable",
                               new BoxedBuiltinFunctionOrMethod(boxRTFunction((void*)callable, UNKNOWN, 1), "callable"));
-
-    BoxedClass* buffer_cls = BoxedHeapClass::create(type_cls, object_cls, NULL, 0, 0, sizeof(Box), false, "buffer");
-    builtins_module->giveAttr("buffer", buffer_cls);
 }
 }

src/runtime/capi.cpp

@@ -593,6 +593,18 @@ finally:
     --tstate->recursion_depth;
 }
 
+extern "C" PyGILState_STATE PyGILState_Ensure(void) noexcept {
+    Py_FatalError("unimplemented");
+}
+
+extern "C" void PyGILState_Release(PyGILState_STATE) noexcept {
+    Py_FatalError("unimplemented");
+}
+
+extern "C" PyThreadState* PyGILState_GetThisThreadState(void) noexcept {
+    Py_FatalError("unimplemented");
+}
+
 void setCAPIException(const ExcInfo& e) {
     cur_thread_state.curexc_type = e.type;
     cur_thread_state.curexc_value = e.value;
@@ -1173,14 +1185,14 @@ extern "C" int _PyEval_SliceIndex(PyObject* v, Py_ssize_t* pi) noexcept {
     return 1;
 }
 
-extern "C" PyObject* PyBuffer_FromMemory(void* ptr, Py_ssize_t size) noexcept {
-    Py_FatalError("unimplemented");
-}
-
 extern "C" int PyEval_GetRestricted(void) noexcept {
     return 0; // We don't support restricted mode
 }
 
+extern "C" void PyEval_InitThreads(void) noexcept {
+    // nothing to do here
+}
+
 BoxedModule* importTestExtension(const std::string& name) {
     std::string pathname_name = "test/test_extension/" + name + ".pyston.so";
     const char* pathname = pathname_name.c_str();

src/runtime/long.cpp

@@ -97,6 +97,10 @@ extern "C" PY_LONG_LONG PyLong_AsLongLong(PyObject* vv) noexcept {
     Py_FatalError("unimplemented");
 }
 
+extern "C" PY_LONG_LONG PyLong_AsLongLongAndOverflow(PyObject* obj, int* overflow) noexcept {
+    Py_FatalError("unimplemented");
+}
+
 extern "C" PyObject* PyLong_FromString(const char* str, char** pend, int base) noexcept {
     Py_FatalError("unimplemented");
 }

src/runtime/types.cpp

@@ -74,6 +74,7 @@ extern "C" void init_io();
 extern "C" void initzipimport();
 extern "C" void init_csv();
 extern "C" void init_ssl();
+extern "C" void init_sqlite3();
 
 namespace pyston {
 
@@ -1985,6 +1986,7 @@ void setupRuntime() {
     initzipimport();
     init_csv();
     init_ssl();
+    init_sqlite3();
 
     // some additional setup to ensure weakrefs participate in our GC
     BoxedClass* weakref_ref_cls = &_PyWeakref_RefType;

test/tests/buffer_compare.py

+# expected: fail
+# we don't support tp_compare yet
+
+s = 'Hello world'
+t = buffer(s, 6, 5)
+
+s2 = "Goodbye world"
+t2 = buffer(s2, 8, 5)
+
+print t == t2

test/tests/buffer_test.py

+s = 'Hello world'
+t = buffer(s, 6, 5)
+print t

test/tests/sqlite_test.py

+import os
+try:
+    os.unlink('example.db')
+except Exception:
+    pass
+
+import sqlite3
+conn = sqlite3.connect('example.db')
+
+c = conn.cursor()
+
+# Create table
+c.execute('''CREATE TABLE stocks
+             (date text, trans text, symbol text, qty real, price real)''')
+
+# Insert a row of data
+c.execute("INSERT INTO stocks VALUES ('2006-01-05','BUY','RHAT',100,35.14)")
+
+# Save (commit) the changes
+conn.commit()
+
+# We can also close the connection if we are done with it.
+# Just be sure any changes have been committed or they will be lost.
+conn.close()
+
+
+
+conn = sqlite3.connect('example.db')
+c = conn.cursor()
+
+# Never do this -- insecure!
+symbol = 'RHAT'
+c.execute("SELECT * FROM stocks WHERE symbol = '%s'" % symbol)
+
+# Do this instead
+t = ('RHAT',)
+c.execute('SELECT * FROM stocks WHERE symbol=?', t)
+print c.fetchone()
+
+# Larger example that inserts many records at a time
+purchases = [('2006-03-28', 'BUY', 'IBM', 1000, 45.00),
+             ('2006-04-05', 'BUY', 'MSFT', 1000, 72.00),
+             ('2006-04-06', 'SELL', 'IBM', 500, 53.00),
+            ]
+c.executemany('INSERT INTO stocks VALUES (?,?,?,?,?)', purchases)
+
+for row in c.execute('SELECT * FROM stocks ORDER BY price'):
+    print row

test/unittests/CMakeLists.txt

@@ -9,7 +9,7 @@ add_custom_target(unittests)
 
 macro(add_unittest unittest)
   add_executable(${unittest}_unittest EXCLUDE_FROM_ALL ${unittest}.cpp $<TARGET_OBJECTS:PYSTON_OBJECTS> $<TARGET_OBJECTS:FROM_CPYTHON>)
-  target_link_libraries(${unittest}_unittest stdlib gmp ssl crypto readline pypa double-conversion unwind gtest gtest_main ${LLVM_LIBS} ${LIBLZMA_LIBRARIES})
+  target_link_libraries(${unittest}_unittest stdlib sqlite3 gmp ssl crypto readline pypa double-conversion unwind gtest gtest_main ${LLVM_LIBS} ${LIBLZMA_LIBRARIES})
   add_dependencies(${unittest}_unittest gtest gtest_main)
   add_dependencies(unittests ${unittest}_unittest)
 endmacro()