Merge pull request #1286 from Daetalus/scipy_fixing5_nexedi

Some SciPy fixing.

from_cpython/Lib/distutils/ccompiler.py

@@ -371,7 +371,9 @@ class CCompiler:
             cc_args[:0] = ['-g']
         if before:
             cc_args[:0] = before
-        cc_args = cc_args + ["-Werror=implicit-function-declaration"]
+
+        if not any ('scipy' in s for s in pp_opts):
+            cc_args = cc_args + ["-Werror=implicit-function-declaration"]
         return cc_args
 
     def _fix_compile_args(self, output_dir, macros, include_dirs):

src/runtime/code.cpp

@@ -137,8 +137,10 @@ extern "C" PyCodeObject* PyCode_New(int argcount, int nlocals, int stacksize, in
     is_dummy = is_dummy && code == EmptyString && lnotab == EmptyString;
     for (auto&& var : { consts, names, varnames, freevars, cellvars })
         is_dummy = is_dummy && var == EmptyTuple;
-    RELEASE_ASSERT(is_dummy, "not implemented");
-    // ok this is an empty/dummy code object
+    // The follwing variables are not implemented but we allow them because there is currently
+    // no way for code to retrieve them.
+    auto temp_allowed = argcount || argcount || flags || varnames != EmptyTuple;
+    RELEASE_ASSERT(is_dummy || temp_allowed, "not implemented");
 
     RELEASE_ASSERT(PyString_Check(filename), "");
     RELEASE_ASSERT(PyString_Check(name), "");

src/runtime/dict.cpp

@@ -425,8 +425,10 @@ extern "C" int PyDict_Next(PyObject* op, Py_ssize_t* ppos, PyObject** pkey, PyOb
         return 0;
     }
 
-    *pkey = (*it)->first.value;
-    *pvalue = (*it)->second;
+    if (pkey)
+        *pkey = (*it)->first.value;
+    if (pvalue)
+        *pvalue = (*it)->second;
 
     ++(*it);
 

test/extra/lxml_test.py

@@ -5,20 +5,20 @@ from test_helper import create_virtenv, run_test
 
 ENV_NAME = "lxml_test_env_" + os.path.basename(sys.executable)
 SRC_DIR = os.path.abspath(os.path.join(ENV_NAME, "src"))
+CYTHON_DIR = os.path.abspath(os.path.join(SRC_DIR, "cython"))
 PYTHON_EXE = os.path.abspath(os.path.join(ENV_NAME, "bin", "python"))
 
 def install_and_test_lxml():
     shutil.rmtree(SRC_DIR, ignore_errors=True)
     os.makedirs(SRC_DIR)
 
-    url = "https://github.com/cython/cython/archive/0.22.tar.gz"
-    subprocess.check_call(["wget", url, "-O", "Cython-0.22.tar.gz"], cwd=SRC_DIR)
-    subprocess.check_call(["tar", "-zxf", "Cython-0.22.tar.gz"], cwd=SRC_DIR)
+    url = "https://github.com/cython/cython"
+    subprocess.check_call(["git", "clone", "--depth", "1", "--branch", "0.24", url], cwd=SRC_DIR)
 
-    CYTHON_DIR = os.path.abspath(os.path.join(SRC_DIR, "cython-0.22"))
-    PATCH_FILE = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "integration", "Cython-0.22.patch"))
+    PATCH_FILE = os.path.abspath(os.path.join(os.path.dirname(__file__), "../integration/Cython-0.24.patch"))
     subprocess.check_call(["patch", "-p1", "--input=" + PATCH_FILE], cwd=CYTHON_DIR)
-    print "Applied Cython patch"
+    print ">>> Applied Cython patch"
+
     subprocess.check_call([PYTHON_EXE, "setup.py", "install"], cwd=CYTHON_DIR)
     subprocess.check_call([PYTHON_EXE, "-c", "import Cython"], cwd=CYTHON_DIR)
 
@@ -33,7 +33,7 @@ def install_and_test_lxml():
 
     subprocess.check_call([PYTHON_EXE, "setup.py", "build_ext", "-i", "--with-cython"], cwd=LXML_DIR)
  
-    expected = [{'ran': 1381, 'failures': 1}]
+    expected = [{'ran': 1381}]
     run_test([PYTHON_EXE, "test.py"], cwd=LXML_DIR, expected=expected)
     
 create_virtenv(ENV_NAME, None, force_create = True)

test/extra/numpy_fulltest.py

@@ -22,18 +22,17 @@ test_helper.create_virtenv(ENV_NAME, DEPENDENCIES)
 SRC_DIR = ENV_NAME
 ENV_DIR = os.path.abspath(ENV_NAME)
 PYTHON_EXE = os.path.abspath(ENV_NAME + "/bin/python")
-CYTHON_DIR = os.path.abspath(os.path.join(SRC_DIR, "cython-0.22"))
+CYTHON_DIR = os.path.abspath(os.path.join(SRC_DIR, "cython"))
 NUMPY_DIR = os.path.abspath(os.path.join(SRC_DIR, "numpy"))
 
 print_progress_header("Setting up Cython...")
 if not os.path.exists(CYTHON_DIR):
 
-    url = "https://github.com/cython/cython/archive/0.22.tar.gz"
-    subprocess.check_call(["wget", url, "-O", "Cython-0.22.tar.gz"], cwd=SRC_DIR)
-    subprocess.check_call(["tar", "-zxf", "Cython-0.22.tar.gz"], cwd=SRC_DIR)
+    url = "https://github.com/cython/cython"
+    subprocess.check_call(["git", "clone", "--depth", "1", "--branch", "0.24", url], cwd=SRC_DIR)
 
     if USE_CUSTOM_PATCHES:
-        PATCH_FILE = os.path.abspath(os.path.join(os.path.dirname(__file__), "../integration/Cython-0.22.patch"))
+        PATCH_FILE = os.path.abspath(os.path.join(os.path.dirname(__file__), "../integration/Cython-0.24.patch"))
         subprocess.check_call(["patch", "-p1", "--input=" + PATCH_FILE], cwd=CYTHON_DIR)
         print ">>> Applied Cython patch"
 

test/integration/Cython-0.22.patch

-From eaf300b61a56af151e93aa51803e6a61a0f8afee Mon Sep 17 00:00:00 2001
-From: Marius Wachtler <undingen@gmail.com>
-Date: Fri, 6 May 2016 16:19:50 +0100
-Subject: [PATCH] [PATCH] Pyston change: make cython work with pyston
-
----
- Cython/Compiler/ExprNodes.py     | 9 +++++++++
- Cython/Utility/CythonFunction.c  | 4 +++-
- Cython/Utility/Exceptions.c      | 4 +++-
- Cython/Utility/ModuleSetupCode.c | 2 +-
- 4 files changed, 16 insertions(+), 3 deletions(-)
-
-diff --git a/Cython/Compiler/ExprNodes.py b/Cython/Compiler/ExprNodes.py
-index f99ec6e..7ab41f3 100644
---- a/Cython/Compiler/ExprNodes.py
-+++ b/Cython/Compiler/ExprNodes.py
-@@ -7784,12 +7784,21 @@ class PyCFunctionNode(ExprNode, ModuleNameMixin):
- 
-     @classmethod
-     def from_defnode(cls, node, binding):
-+        """
-+        # Pyston change: dont't generate custom code objects because we don't support them currently
-         return cls(node.pos,
-                    def_node=node,
-                    pymethdef_cname=node.entry.pymethdef_cname,
-                    binding=binding or node.specialized_cpdefs,
-                    specialized_cpdefs=node.specialized_cpdefs,
-                    code_object=CodeObjectNode(node))
-+        """
-+        return cls(node.pos,
-+                   def_node=node,
-+                   pymethdef_cname=node.entry.pymethdef_cname,
-+                   binding=binding or node.specialized_cpdefs,
-+                   specialized_cpdefs=node.specialized_cpdefs,
-+                   code_object=None)
- 
-     def analyse_types(self, env):
-         if self.binding:
-diff --git a/Cython/Utility/CythonFunction.c b/Cython/Utility/CythonFunction.c
-index 9cc38f0..ab05ad1 100644
---- a/Cython/Utility/CythonFunction.c
-+++ b/Cython/Utility/CythonFunction.c
-@@ -561,7 +561,9 @@ __Pyx_CyFunction_repr(__pyx_CyFunctionObject *op)
- #endif
- }
- 
--#if CYTHON_COMPILING_IN_PYPY
-+// Pyston change:
-+// #if CYTHON_COMPILING_IN_PYPY
-+#if 0 && CYTHON_COMPILING_IN_PYPY
- // originally copied from PyCFunction_Call() in CPython's Objects/methodobject.c
- // PyPy does not have this function
- static PyObject * __Pyx_CyFunction_Call(PyObject *func, PyObject *arg, PyObject *kw) {
-diff --git a/Cython/Utility/Exceptions.c b/Cython/Utility/Exceptions.c
-index 354a776..567970d 100644
---- a/Cython/Utility/Exceptions.c
-+++ b/Cython/Utility/Exceptions.c
-@@ -528,7 +528,9 @@ static void __Pyx_AddTraceback(const char *funcname, int c_line,
-         0                    /*PyObject *locals*/
-     );
-     if (!py_frame) goto bad;
--    py_frame->f_lineno = py_line;
-+    // Pyston change:
-+    // py_frame->f_lineno = py_line;
-+    PyFrame_SetLineNumber(py_frame, py_line);
-     PyTraceBack_Here(py_frame);
- bad:
-     Py_XDECREF(py_code);
-diff --git a/Cython/Utility/ModuleSetupCode.c b/Cython/Utility/ModuleSetupCode.c
-index 6477fb2..75dcdda 100644
---- a/Cython/Utility/ModuleSetupCode.c
-+++ b/Cython/Utility/ModuleSetupCode.c
-@@ -32,7 +32,7 @@
-   #define Py_HUGE_VAL HUGE_VAL
- #endif
- 
--#ifdef PYPY_VERSION
-+#if defined(PYPY_VERSION) || defined(PYSTON_VERSION)
- #define CYTHON_COMPILING_IN_PYPY 1
- #define CYTHON_COMPILING_IN_CPYTHON 0
- #else
--- 
-1.9.1
-

test/integration/numpy_test.py → test/integration/scipy_test.py

+# skip-if: True
+# This test will take a long time(51mins).
+# 3100.61s user 490.14s system 118% cpu 50:30.63 total
+# So please test it manually.
+
 import os
 import sys
 import subprocess
+sys.path.append(os.path.dirname(__file__) + "/../lib")
+from test_helper import run_test
 
 """
 Using this test file.
 
-We apply some patches on NumPy for some issues that we can't fix at the moment. The
-patches are applied directly to the NumPy subrepository (test/lib/numpy). If you need,
-you can make modifications in that folder directly for testing purposes. Just make sure
-that everytime you do, run this script again, so that the contents of numpy_test_env_*
-are updated (this is where the code and binaries that get tested are located).
-
-Note that sometimes it can be a pain to run this script everytime you make a change,
-as it does take a while. You can cd to the numpy_test_env_* directory and run the
-binaries inside of that folder, which will be able to "see" the NumPy module. For example:
+Note that sometimes it can be a pain to run this script everytime you make a
+change, as it does take a while. You can cd to the scipy_test_env_* directory
+and run the binaries inside of that folder, which will be able to "see" the
+SciPy module. For example:
 
-~/pyston/numpy_test_env_pyston_dbg$ gdb --args ./bin/python -c "import numpy as np; np.test()"
+~/pyston/scipy_test_env_pyston_dbg$ gdb --args ./bin/python -c "import scipy; scipy.test()"
 
 The /bin/python is the pyston executable so if you recompile pyston, you need to run this
 script again to update it.
 
-Currently this script is not running the NumPy tests since there are still crashes
-happening. If you want to run the test, go to the bottom of the file and uncomment
-the subprocess call to the test suite.
-
-Some test cases in test/lib/numpy are commented out by the patch since they caused
-a crash where the cause was not immediately obvious and I wanted to make progress. Those
-will need to be uncommented at some point.
 """
 
 def print_progress_header(text):
@@ -34,7 +29,7 @@ def print_progress_header(text):
     print ">>> " + text + "..."
     print ">>>"
 
-ENV_NAME = "numpy_test_env_" + os.path.basename(sys.executable)
+ENV_NAME = "scipy_test_env_" + os.path.basename(sys.executable)
 DEPENDENCIES = ["nose==1.3.7"]
 
 import platform
@@ -61,19 +56,18 @@ if not os.path.exists(ENV_NAME) or os.stat(sys.executable).st_mtime > os.stat(EN
 
 SRC_DIR = ENV_NAME
 PYTHON_EXE = os.path.abspath(ENV_NAME + "/bin/python")
-CYTHON_DIR = os.path.abspath(os.path.join(SRC_DIR, "cython-0.22"))
+CYTHON_DIR = os.path.abspath(os.path.join(SRC_DIR, "cython"))
 NUMPY_DIR = os.path.abspath(os.path.join(SRC_DIR, "numpy"))
 SCIPY_DIR = os.path.abspath(os.path.join(SRC_DIR, "scipy"))
 
 print_progress_header("Setting up Cython...")
 if not os.path.exists(CYTHON_DIR):
 
-    url = "https://github.com/cython/cython/archive/0.22.tar.gz"
-    subprocess.check_call(["wget", url, "-O", "Cython-0.22.tar.gz"], cwd=SRC_DIR)
-    subprocess.check_call(["tar", "-zxf", "Cython-0.22.tar.gz"], cwd=SRC_DIR)
+    url = "https://github.com/cython/cython"
+    subprocess.check_call(["git", "clone", "--depth", "1", "--branch", "0.24", url], cwd=SRC_DIR)
 
     if USE_CUSTOM_PATCHES:
-        PATCH_FILE = os.path.abspath(os.path.join(os.path.dirname(__file__), "Cython-0.22.patch"))
+        PATCH_FILE = os.path.abspath(os.path.join(os.path.dirname(__file__), "Cython-0.24.patch"))
         subprocess.check_call(["patch", "-p1", "--input=" + PATCH_FILE], cwd=CYTHON_DIR)
         print ">>> Applied Cython patch"
 
@@ -108,91 +102,29 @@ except:
 
     raise
 
-print_progress_header("Cloning up SciPy...")
 if not os.path.exists(SCIPY_DIR):
+    print_progress_header("Cloning up SciPy...")
     url = "https://github.com/scipy/scipy"
-    # subprocess.check_call(["git", "clone", "--depth", "1", "--branch", "v0.17.1", url], cwd=SRC_DIR)
+    subprocess.check_call(["git", "clone", "--depth", "1", "--branch", "v0.17.1", url], cwd=SRC_DIR)
 else:
     print ">>> SciPy already installed."
 
 try:
     print_progress_header("Setting up SciPy...")
-    # subprocess.check_call([PYTHON_EXE, "setup.py", "build"], cwd=SCIPY_DIR, env=env)
+    subprocess.check_call([PYTHON_EXE, "setup.py", "build"], cwd=SCIPY_DIR, env=env)
 
     print_progress_header("Installing SciPy...")
-    # subprocess.check_call([PYTHON_EXE, "setup.py", "install"], cwd=SCIPY_DIR, env=env)
+    subprocess.check_call([PYTHON_EXE, "setup.py", "install"], cwd=SCIPY_DIR, env=env)
 except:
     subprocess.check_call(["rm", "-rf", SCIPY_DIR + "/build"])
     subprocess.check_call(["rm", "-rf", SCIPY_DIR + "/dist"])
-
     raise
 
-# From Wikipedia
-script = """
-import numpy as np
-from numpy.linalg import solve, inv
-a = np.array([[1, 2, 3], [3, 4, 6.7], [5, 9.0, 5]])
-b = np.array([3, 2, 1])
-a.transpose()
-inv(a)
-solve(a, b)
-print a, b
-"""
-
-# http://wiki.scipy.org/Tentative_NumPy_Tutorial/Mandelbrot_Set_Example
-mandelbrot = """
-from numpy import *
-
-def mandelbrot( h,w, maxit=20 ):
-        '''Returns an image of the Mandelbrot fractal of size (h,w).
-        '''
-        y,x = ogrid[ -1.4:1.4:h*1j, -2:0.8:w*1j ]
-        c = x+y*1j
-        z = c
-        divtime = maxit + zeros(z.shape, dtype=int)
-
-        for i in xrange(maxit):
-                z  = z**2 + c
-                diverge = z*conj(z) > 2**2
-                div_now = diverge & (divtime==maxit)
-                divtime[div_now] = i
-                z[diverge] = 2
-
-        return divtime
-
-m = mandelbrot(complex(400),complex(400))
-print m
-"""
-
-# this is just a small test which checks if numpy is able to manually create a pyston string
-string_test = """
-import numpy as np
-a = np.array([1, 2**16, 2**32], dtype=int)
-s = a.astype(str)
-print s
-assert repr(s) == "array(['1', '65536', '4294967296'], \\n      dtype='|S21')"
-"""
-
-numpy_test = "import numpy as np; np.test(verbose=2)"
-
-print_progress_header("Running NumPy linear algebra test...")
-subprocess.check_call([PYTHON_EXE, "-c", script], cwd=CYTHON_DIR)
-
-print_progress_header("Running NumPy mandelbrot test...")
-subprocess.check_call([PYTHON_EXE, "-c", mandelbrot], cwd=CYTHON_DIR)
-
-
-print_progress_header("Running NumPy str test...")
-subprocess.check_call([PYTHON_EXE, "-c", string_test], cwd=CYTHON_DIR)
-
-# fails in release mode
-#print_progress_header("Running NumPy test_abc.py test...")
-#subprocess.check_call([PYTHON_EXE, os.path.join(NUMPY_DIR, "numpy/core/tests/test_abc.py")], cwd=CYTHON_DIR)
+scip_test = "import scipy; scipy.test(verbose=2)"
 
-print_progress_header("Running NumPy test suite...")
-# Currently we crash running the test suite. Uncomment for testing or
-# when all the crashes are fixed.
-# subprocess.check_call([PYTHON_EXE, "-c", numpy_test], cwd=CYTHON_DIR)
+print_progress_header("Running SciPy test suite...")
+expected = [{'ran': 20391}]
+run_test([PYTHON_EXE, "-c", scip_test], cwd=CYTHON_DIR, expected=expected)
 
 print
 print "PASSED"