Enable pypy as a required Travis test (GH-3392)

Reasoning being that this make it easier to catch pypy3
regressions as they happen.

* Fixed some very simple pypy3 failures (largely to do with
  different exception strings)
* Splits pypy3_bugs.txt into three files
  - one for bugs that cause hard crashes (which we don't want to
    run in Travis at all);
  - one for bugs that are probably unfixable because they're just
    due to implementation details (e.g. when destructors are
    called).
  - all other bugs remain in pypy3_bugs.txt
  (The categorization has been done fairly quickly, so some bugs
  may be in the wrong place)
* Made sure (hopefully) all bugs are now categorized, so a basic
  runtests.py with pypy3 should hopefully pass
* Changed pypy3 to be required in Travis
* Added an extra (optional) test that runs through pypy3_bugs.txt.
  The majority of this is expected to fail. This requires an
  extra option to runtest.py "--listfile", which just runs through
  the tests listed in the file.

I haven't made pypy2 a required test in this commit - since Python2 support is deprecated soon, there seemed limited value in putting much effort into pypy2.

Added faulthandler to runtests in the hope of being able to pin-down segmentation faults better on Travis

FileListExcluder matches regexes, not just name. Uses the same mechanism as is used for processing string passed on commandline

.travis.yml

@@ -71,10 +71,16 @@ matrix:
       env: BACKEND=c
     - python: pypy3
       env: BACKEND=c
-  allow_failures:
-    - python: 3.9-dev
+# a secondary pypy tests which is allowed to fail and which specifically
+# tests known bugs
     - python: pypy
+      env: BACKEND=c EXCLUDE="--listfile=tests/pypy_bugs.txt --listfile=tests/pypy2_bugs.txt bugs"
     - python: pypy3
+      env: BACKEND=c EXCLUDE="--listfile=tests/pypy_bugs.txt bugs"
+  allow_failures:
+    - python: 3.9-dev
+    - env: BACKEND=c EXCLUDE="--listfile=tests/pypy_bugs.txt bugs"
+    - env: BACKEND=c EXCLUDE="--listfile=tests/pypy_bugs.txt --listfile=tests/pypy2_bugs.txt bugs"
 
 branches:
   only:

Cython/Utility/ModuleSetupCode.c

@@ -631,7 +631,7 @@ static CYTHON_INLINE PyObject * __Pyx_PyDict_GetItemStr(PyObject *dict, PyObject
     if (res == NULL) PyErr_Clear();
     return res;
 }
-#elif PY_MAJOR_VERSION >= 3
+#elif PY_MAJOR_VERSION >= 3 && (!CYTHON_COMPILING_IN_PYPY || PYPY_VERSION_NUM >= 0x07020000)
 #define __Pyx_PyDict_GetItemStrWithError  PyDict_GetItemWithError
 #define __Pyx_PyDict_GetItemStr           PyDict_GetItem
 #else

Cython/Utility/Optimize.c

@@ -190,7 +190,7 @@ static PyObject* __Pyx_PyDict_GetItemDefault(PyObject* d, PyObject* key, PyObjec
 
 static PyObject* __Pyx_PyDict_GetItemDefault(PyObject* d, PyObject* key, PyObject* default_value) {
     PyObject* value;
-#if PY_MAJOR_VERSION >= 3 && !CYTHON_COMPILING_IN_PYPY
+#if PY_MAJOR_VERSION >= 3 && (!CYTHON_COMPILING_IN_PYPY || PYPY_VERSION_NUM >= 0x07020000)
     value = PyDict_GetItemWithError(d, key);
     if (unlikely(!value)) {
         if (unlikely(PyErr_Occurred()))
@@ -238,7 +238,7 @@ static CYTHON_INLINE PyObject *__Pyx_PyDict_SetDefault(PyObject *d, PyObject *ke
 #else
     if (is_safe_type == 1 || (is_safe_type == -1 &&
         /* the following builtins presumably have repeatably safe and fast hash functions */
-#if PY_MAJOR_VERSION >= 3 && !CYTHON_COMPILING_IN_PYPY
+#if PY_MAJOR_VERSION >= 3 && (!CYTHON_COMPILING_IN_PYPY || PYPY_VERSION_NUM >= 0x07020000)
             (PyUnicode_CheckExact(key) || PyString_CheckExact(key) || PyLong_CheckExact(key)))) {
         value = PyDict_GetItemWithError(d, key);
         if (unlikely(!value)) {

runtests.py

@@ -1826,6 +1826,10 @@ class EmbedTest(unittest.TestCase):
         except OSError:
             pass
 
+def load_listfile(filename):
+    # just re-use the FileListExclude implementation
+    fle = FileListExcluder(filename)
+    return list(fle.excludes)
 
 class MissingDependencyExcluder(object):
     def __init__(self, deps):
@@ -1874,8 +1878,7 @@ class FileListExcluder(object):
                     self.excludes[line.split()[0]] = True
 
     def __call__(self, testname, tags=None):
-        exclude = (testname in self.excludes
-                   or testname.split('.')[-1] in self.excludes)
+        exclude = any(string_selector(ex)(testname) for ex in self.excludes)
         if exclude and self.verbose:
             print("Excluding %s because it's listed in %s"
                   % (testname, self._list_file))
@@ -2068,6 +2071,9 @@ def main():
     parser.add_option("-x", "--exclude", dest="exclude",
                       action="append", metavar="PATTERN",
                       help="exclude tests matching the PATTERN")
+    parser.add_option("--listfile", dest="listfile",
+                      action="append",
+                      help="specify a file containing a list of tests to run")
     parser.add_option("-j", "--shard_count", dest="shard_count", metavar="N",
                       type=int, default=1,
                       help="shard this run into several parallel runs")
@@ -2161,6 +2167,10 @@ def main():
     if options.xml_output_dir:
         shutil.rmtree(options.xml_output_dir, ignore_errors=True)
 
+    if options.listfile:
+        for listfile in options.listfile:
+            cmd_args.extend(load_listfile(listfile))
+
     if options.capture:
         keep_alive_interval = 10
     else:
@@ -2299,6 +2309,16 @@ def runtests_callback(args):
 
 
 def runtests(options, cmd_args, coverage=None):
+    # faulthandler should be able to provide a limited traceback
+    # in the event of a segmentation fault. Hopefully better than Travis
+    # just keeping running until timeout. Only available on Python 3.3+
+    try:
+        import faulthandler
+    except ImportError:
+        pass  # OK - not essential
+    else:
+        faulthandler.enable()
+
 
     WITH_CYTHON = options.with_cython
     ROOTDIR = os.path.abspath(options.root_dir)
@@ -2416,6 +2436,9 @@ def runtests(options, cmd_args, coverage=None):
         bug_files = [
             ('bugs.txt', True),
             ('pypy_bugs.txt', IS_PYPY),
+            ('pypy2_bugs.txt', IS_PYPY and IS_PY2),
+            ('pypy_crash_bugs.txt', IS_PYPY),
+            ('pypy_implementation_detail_bugs.txt', IS_PYPY),
             ('limited_api_bugs.txt', options.limited_api),
             ('windows_bugs.txt', sys.platform == 'win32'),
             ('cygwin_bugs.txt', sys.platform == 'cygwin')

tests/pypy2_bugs.txt

+# Specific bugs that only apply to pypy2
+
+build.cythonize_script
+build.cythonize_script_package
+run.initial_file_path
+run.reduce_pickle
+run.final_in_pxd
+run.cdef_multiple_inheritance
+run.cdef_multiple_inheritance_nodict
+run.extstarargs
+run.cpython_capi
+run.isnot
+
+# pypy 2 seems to be preferring .py files to .so files
+# https://foss.heptapod.net/pypy/pypy/issues/3185
+run.language_level
+run.pure_pxd
+
+# Silly error with doctest matching slightly different string outputs rather than 
+# an actual bug but one I can't easily resolve
+run.with_gil
+
+
+# looks like a "when does the GC run?" issue - slightly surprised it's OK on pypy3
+memoryview.numpy_memoryview

tests/pypy_bugs.txt

@@ -4,15 +4,58 @@
 
 broken_exception
 bufaccess
-memoryview
-memslice
+memoryview.memoryview$
 sequential_parallel
 
 yield_from_pep380
 memoryview_inplace_division
 
+run.unicodemethods
+run.unicode_imports
+run.tp_new
+run.test_fstring
+run.test_exceptions
+run.test_dictviews
+run.str_subclass_kwargs
+run.special_method_docstrings
+run.slice_ptr
+compile.min_async
+run.cython_includes
+run.pyarray
+run.test_unicode
+run.__getattribute__
+run.__getattribute_subclasses__
+run.__debug__
+run.array_cimport
+run.builtin_abs
+run.builtincomplex
+run.cdef_multiple_inheritance_errors
+run.cdivision_CEP_516
+run.cyfunction
+run.final_cdef_class
+run.index
+run.pyclass_special_methods
+run.reimport_from_package
+run.reimport_from_subinterpreter
+pkg.cimportfrom
+embedded
+TestCyCache
+run.ext_auto_richcmp
+run.coverage_cmd
+run.coverage_cmd_src_layout
+run.coverage_installed_pkg
+run.coverage_api
+run.coverage_nogil
+
+# very little coroutine-related seems to work
+run.test_asyncgen
+run.test_coroutines_pep492
+run.async_iter_pep492
+run.embedsignatures
+run.py35_asyncio_async_def
+run.asyncio_generators
+
 # gc issue?
-memoryview_in_subclasses
 external_ref_reassignment
 run.exttype_dealloc
 
@@ -20,20 +63,13 @@ run.exttype_dealloc
 run.special_methods_T561
 run.special_methods_T561_py2
 
-# tests for things that don't exist in cpyext
-compile.pylong
-run.datetime_pxd
-run.datetime_cimport
-run.datetime_members
-run.extern_builtins_T258
-run.line_trace
-run.line_profile_test
-run.pstats_profile_test
-run.longintrepr
-
-# refcounting-specific tests
-double_dealloc_T796
-run.exceptionrefcount
-run.capiimpl
-run.refcount_in_meth
-
+# looks to be fixed in PyPy 7.3.0
+# TODO - remove when Travis updates
+run.py_unicode_strings
+run.unicodeliterals
+run.unicode_identifiers
+run.unicode_identifiers_import
+errors.unicode_identifiers_e4
+run.tracebacks
+run.fstring
+run.unicode_identifiers_normalization

tests/pypy_crash_bugs.txt

+# Bugs that causes hard crashes that we certainly don't 
+# want to run because it will break the testsuite
+
+# segfault
+run.fastcall
+memslice
+
+# """Fatal RPython error: NotImplementedError
+# Aborted (core dumped)"""
+run.py35_pep492_interop
+
+# gc issue?
+memoryview_in_subclasses

tests/pypy_implementation_detail_bugs.txt

+# PyPy "bugs" that are probably implementation differences from
+# CPython rather than actual bugs. Therefore they aren't targets
+# to be fixed (but there *may* be other details in the testfile
+# that should be tested on PyPy?)
+
+run.starargs
+
+# refcounting-specific tests
+double_dealloc_T796
+run.exceptionrefcount
+run.capiimpl
+run.refcount_in_meth
+# Ideally just disable the reference-counting tests on PyPy?
+run.fused_types
+run.generator_frame_cycle
+run.generators_in_refcycles
+run.generators_py
+run.parallel
+
+# "sys.getsizeof(object, default) will always return default on PyPy, and
+# raise a TypeError if default is not provided."
+buildenv
+
+# tests for things that don't exist in cpyext
+compile.pylong
+run.datetime_pxd
+run.datetime_cimport
+run.datetime_members
+run.extern_builtins_T258
+run.line_trace
+run.line_profile_test
+run.pstats_profile_test
+run.longintrepr
+
+# tests probably rely on immediate GC (although maybe the tests could be tweaked so
+# only these bits don't run in PyPy?)
+buffers.buffer
+buffers.userbuffer
+memoryview.cythonarray
+memoryview.memoryview_pep489_typing
+run.cpp_classes
+run.cpp_classes_def
+
+# missing pypy feature?
+matrix_multiplier

tests/run/callargs.pyx

@@ -168,15 +168,15 @@ def test_int_kwargs(f):
     """
     >>> test_int_kwargs(e)     # doctest: +ELLIPSIS
     Traceback (most recent call last):
-    TypeError: ...keywords must be strings
+    TypeError: ...keywords must be strings...
     >>> test_int_kwargs(f)     # doctest: +ELLIPSIS
     Traceback (most recent call last):
-    TypeError: ...keywords must be strings
+    TypeError: ...keywords must be strings...
     >>> test_int_kwargs(g)     # doctest: +ELLIPSIS
     Traceback (most recent call last):
-    TypeError: ...keywords must be strings
+    TypeError: ...keywords must be strings...
     >>> test_int_kwargs(h)     # doctest: +ELLIPSIS
     Traceback (most recent call last):
-    TypeError: ...keywords must be strings
+    TypeError: ...keywords must be strings...
     """
     f(a=1,b=2,c=3, **{10:20,30:40})

tests/run/cpp_stl_conversion.pyx

@@ -143,10 +143,12 @@ def test_typedef_vector(o):
     Traceback (most recent call last):
     ...
     OverflowError: ...
+
+    "TypeError: an integer is required" on CPython
     >>> test_typedef_vector([1, 2, None])       #doctest: +ELLIPSIS
     Traceback (most recent call last):
     ...
-    TypeError: an integer is required
+    TypeError: ...int...
     """
     cdef vector[my_int] v = o
     return v

tests/run/importfrom.pyx

@@ -68,6 +68,10 @@ def typed_imports():
     try:
         from sys import version_info as maxunicode
     except TypeError, e:
+        if getattr(sys, "pypy_version_info", None):
+            # translate message
+            if e.args[0].startswith("int() argument must be"):
+                e = "an integer is required"
         print(e)
 
     try:

tests/run/kwargproblems.pyx

@@ -4,7 +4,7 @@ def test(**kw):
     >>> d = {1 : 2}
     >>> test(**d)       # doctest: +ELLIPSIS
     Traceback (most recent call last):
-    TypeError: ...keywords must be strings
+    TypeError: ...keywords must be strings...
     >>> d
     {1: 2}
     >>> d = {}

tests/run/list_pop.pyx

@@ -206,7 +206,7 @@ def crazy_pop(L):
     """
     >>> crazy_pop(list(range(10)))    # doctest: +ELLIPSIS
     Traceback (most recent call last):
-    TypeError: pop... at most ... argument...
+    TypeError: pop... argument...
     >>> crazy_pop(A())
     (1, 2, 3)
     """

tests/run/modop.pyx

@@ -9,7 +9,7 @@ def modobj(obj2, obj3):
     '5'
     >>> modobj(1, 0)  # doctest: +ELLIPSIS
     Traceback (most recent call last):
-    ZeroDivisionError: integer division or modulo by zero
+    ZeroDivisionError: integer... modulo by zero
     """
     obj1 = obj2 % obj3
     return obj1
@@ -17,9 +17,9 @@ def modobj(obj2, obj3):
 
 def mod_10_obj(int2):
     """
-    >>> mod_10_obj(0)
+    >>> mod_10_obj(0)  # doctest: +ELLIPSIS
     Traceback (most recent call last):
-    ZeroDivisionError: integer division or modulo by zero
+    ZeroDivisionError: ... modulo by zero
     >>> 10 % 1
     0
     >>> mod_10_obj(1)

tests/run/pep448_test_extcall.pyx

@@ -326,7 +326,7 @@ def errors_non_string_kwarg():
     """
     >>> errors_non_string_kwarg()  # doctest: +ELLIPSIS
     Traceback (most recent call last):
-    TypeError: ...keywords must be strings
+    TypeError: ...keywords must be strings...
     """
     f(**{1:2})
 
@@ -463,10 +463,10 @@ def call_builtin_empty_dict():
 
 def call_builtin_nonempty_dict():
     """
-    >>> call_builtin_nonempty_dict()
+    >>> call_builtin_nonempty_dict() # doctest: +ELLIPSIS
     Traceback (most recent call last):
       ...
-    TypeError: id() takes no keyword arguments
+    TypeError: id() ... keyword argument...
     """
     return id(1, **{'foo': 1})
 

tests/run/powop.pyx

@@ -123,9 +123,9 @@ def optimised_pow2(n):
     0.5
     >>> optimised_pow2(0.5) == 2 ** 0.5
     True
-    >>> optimised_pow2('test')
+    >>> optimised_pow2('test') # doctest: +ELLIPSIS
     Traceback (most recent call last):
-    TypeError: unsupported operand type(s) for ** or pow(): 'int' and 'str'
+    TypeError: ...operand... **...
     """
     if isinstance(n, (int, long)) and 0 <= n < 1000:
         assert isinstance(2.0 ** n, float), 'float %s' % n
@@ -153,9 +153,9 @@ def optimised_pow2_inplace(n):
     0.5
     >>> optimised_pow2_inplace(0.5) == 2 ** 0.5
     True
-    >>> optimised_pow2_inplace('test')
+    >>> optimised_pow2_inplace('test') # doctest: +ELLIPSIS
     Traceback (most recent call last):
-    TypeError: unsupported operand type(s) for ** or pow(): 'int' and 'str'
+    TypeError: ...operand... **...
     """
     x = 2
     x **= n

tests/run/pure.pyx

@@ -25,10 +25,12 @@ def test_declare(n):
     (100, 100)
     >>> test_declare(100.5)
     (100, 100)
-    >>> test_declare(None)
+
+    # CPython: "TypeError: an integer is required"
+    >>> test_declare(None) # doctest: +ELLIPSIS
     Traceback (most recent call last):
     ...
-    TypeError: an integer is required
+    TypeError: ...int...
     """
     x = cython.declare(cython.int)
     y = cython.declare(cython.int, n)

tests/run/py_hash_t.pyx

@@ -25,7 +25,7 @@ def assign_py_hash_t(x):
     >>> assign_py_hash_t(IntLike(1.5))  # doctest: +ELLIPSIS
     Traceback (most recent call last):
     ...
-    TypeError: __index__ ... (type float)
+    TypeError: __index__ ... (type ...float...)
     """
     cdef Py_hash_t h = x
     return h