Merge pull request #1315 from gevent/win-pypy6

Appveyor updates: Test PyPy6

.travis.yml

@@ -8,7 +8,7 @@ python:
 env:
     global:
         - BUILD_RUNTIMES=$HOME/.runtimes
-        - PYTHONHASHSEED=random
+        - PYTHONHASHSEED=8675309
         - CC="ccache gcc"
         - CCACHE_NOCPP2=true
         - CCACHE_SLOPPINESS=file_macro,time_macros,include_file_ctime,include_file_mtime

CHANGES.rst

@@ -9,6 +9,8 @@
 
 - Build with Cython 0.29 in '3str' mode.
 
+- Test with PyPy 6.0 on Windows.
+
 - Add support for application-wide callbacks when ``Greenlet`` objects
   are started. See :pr:`1289`, provided by Yury Selivanov.
 
@@ -53,6 +55,9 @@
 - Make gevent's pywsgi server set the non-standard environment value
   ``wsgi.input_terminated`` to True. See :issue:`1308`.
 
+- Make `gevent.util.assert_switches` produce more informative messages
+  when the assertion fails.
+
 1.3.7 (2018-10-12)
 ==================
 

appveyor.yml

+clone_depth: 50
 environment:
   global:
     # SDK v7.0 MSVC Express 2008's SetEnv.cmd script will fail if the
     # /E:ON and /V:ON options are not enabled in the batch script interpreter
     # See: http://stackoverflow.com/a/13751649/163740
     CMD_IN_ENV: "cmd /E:ON /V:ON /C .\\appveyor\\run_with_env.cmd"
+    # Use a fixed hash seed for reproducability
+    PYTHONHASHSEED: 8675309
 
   matrix:
 
     # Pre-installed Python versions, which Appveyor may upgrade to
     # a later point release.
 
+    # 64-bit
+
     - PYTHON: "C:\\Python37-x64"
       PYTHON_VERSION: "3.7.x"
       PYTHON_ARCH: "64"
@@ -25,42 +30,54 @@ environment:
       PYTHON_ARCH: "64"
       PYTHON_EXE: python
 
-    - PYTHON: "C:\\pypy2-v5.10.0-win32"
-      PYTHON_ID: "pypy"
-      PYTHON_EXE: pypy
-      PYTHON_VERSION: "2.7.x"
-      PYTHON_ARCH: "32"
+    - PYTHON: "C:\\Python35-x64"
+      PYTHON_VERSION: "3.5.x" # currently 3.5.2
+      PYTHON_ARCH: "64"
+      PYTHON_EXE: python
 
     - PYTHON: "C:\\Python34-x64"
       PYTHON_VERSION: "3.4.x" # currently 3.4.4
       PYTHON_ARCH: "64"
       PYTHON_EXE: python
 
-    - PYTHON: "C:\\Python35-x64"
-      PYTHON_VERSION: "3.5.x" # currently 3.5.2
-      PYTHON_ARCH: "64"
+    # 32-bit
+    - PYTHON: "C:\\pypy2-v6.0.0-win32"
+      PYTHON_ID: "pypy"
+      PYTHON_EXE: pypy
+      PYTHON_VERSION: "2.7.x"
+      PYTHON_ARCH: "32"
+
+    # 32-bit, wheel only (no testing)
+
+    - PYTHON: "C:\\Python37"
+      PYTHON_VERSION: "3.7.x"
+      PYTHON_ARCH: "32"
       PYTHON_EXE: python
+      GWHEEL_ONLY: true
 
-    - PYTHON: "C:\\Python35"
-      PYTHON_VERSION: "3.5.x" # currently 3.5.2
+    - PYTHON: "C:\\Python36"
+      PYTHON_VERSION: "3.6.x" # currently 3.6.3
       PYTHON_ARCH: "32"
       PYTHON_EXE: python
+      GWHEEL_ONLY: true
 
-    - PYTHON: "C:\\Python27"
-      PYTHON_VERSION: "2.7.x" # currently 2.7.13
+    - PYTHON: "C:\\Python35"
+      PYTHON_VERSION: "3.5.x" # currently 3.5.2
       PYTHON_ARCH: "32"
       PYTHON_EXE: python
+      GWHEEL_ONLY: true
 
     - PYTHON: "C:\\Python34"
       PYTHON_VERSION: "3.4.x" # currently 3.4.3
       PYTHON_ARCH: "32"
       PYTHON_EXE: python
+      GWHEEL_ONLY: true
 
-    - PYTHON: "C:\\Python36"
-      PYTHON_VERSION: "3.6.x" # currently 3.6.3
+    - PYTHON: "C:\\Python27"
+      PYTHON_VERSION: "2.7.x" # currently 2.7.13
       PYTHON_ARCH: "32"
       PYTHON_EXE: python
-
+      GWHEEL_ONLY: true
 
     # Also test a Python version not pre-installed
     # See: https://github.com/ogrisel/python-appveyor-demo/issues/10
@@ -99,10 +116,10 @@ install:
         New-Item -ItemType directory -Path "$env:PYTMP" | Out-Null;
       }
       if ("${env:PYTHON_ID}" -eq "pypy") {
-        if (!(Test-Path "${env:PYTMP}\pypy2-v5.10.0-win32.zip")) {
-          (New-Object Net.WebClient).DownloadFile('https://bitbucket.org/pypy/pypy/downloads/pypy2-v5.10.0-win32.zip', "${env:PYTMP}\pypy2-v5.10.0-win32.zip");
+        if (!(Test-Path "${env:PYTMP}\pypy2-v6.0.0-win32.zip")) {
+          (New-Object Net.WebClient).DownloadFile('https://bitbucket.org/pypy/pypy/downloads/pypy2-v6.0.0-win32.zip', "${env:PYTMP}\pypy2-v6.0.0-win32.zip");
         }
-        7z x -y "${env:PYTMP}\pypy2-v5.10.0-win32.zip" -oC:\ | Out-Null;
+        7z x -y "${env:PYTMP}\pypy2-v6.0.0-win32.zip" -oC:\ | Out-Null;
         & "${env:PYTHON}\pypy.exe" "-mensurepip";
 
       }
@@ -138,21 +155,32 @@ cache:
   - '%LOCALAPPDATA%\pip\Cache'
 
 build_script:
-  - "%PYEXE% -m pip install -U --upgrade-strategy=eager .[test,events,dnspython]"
+  # Build the compiled extension
+  - "%CMD_IN_ENV% %PYEXE% -m pip wheel . -w dist"
+  - ps: "ls dist"
+  # Now install the wheel.
+  # I couldn't get wildcards to work for pip install, so stuff it
+  # into a variable, using python to glob.
+  - "%PYEXE% -c \"import glob; print(glob.glob('dist/gevent*whl')[0])\" > whl.txt"
+  - set /p PYWHL=<whl.txt
+  - if not "%GWHEEL_ONLY%"=="true" %PYEXE% -m pip install -U --upgrade-strategy=eager %PYWHL%[test,events,dnspython]
 
 test_script:
   # Run the project tests
-  - "%PYEXE% -c \"import gevent.core; print(gevent.core.loop)\""
-  - "%PYEXE% -c \"import gevent; print(gevent.config.settings['resolver'].get_options())\""
-  - "%PYEXE% -mgevent.tests --config known_failures.py --quiet"
+  - if not "%GWHEEL_ONLY%"=="true" %PYEXE% -c "import gevent.core; print(gevent.core.loop)"
+  - if not "%GWHEEL_ONLY%"=="true" %PYEXE% -c "import gevent; print(gevent.config.settings['resolver'].get_options())"
+  - if not "%GWHEEL_ONLY%"=="true" %PYEXE% -mgevent.tests --config known_failures.py --quiet
 
 after_test:
-  - "%CMD_IN_ENV% %PYEXE% setup.py bdist_wheel"
+  # We already built the wheel during build_script, because it's
+  # much faster to do that and install from the wheel than to
+  # rebuild it here
+  #- "%CMD_IN_ENV% %PYEXE% setup.py bdist_wheel bdist_wininst"
   - ps: "ls dist"
 
 artifacts:
   # Archive the generated wheel package in the ci.appveyor.com build report.
-  - path: dist\*whl
+  - path: dist\gevent*whl
 
 #on_success:
 #  - TODO: upload the content of dist/*.whl to a public wheelhouse

src/gevent/greenlet.py

@@ -302,11 +302,16 @@ class Greenlet(greenlet):
         """
         The greenlet name. By default, a unique name is constructed using
         the :attr:`minimal_ident`. You can assign a string to this
-        value to change it. It is shown in the `repr` of this object.
+        value to change it. It is shown in the `repr` of this object if it
+        has been assigned to or if the `minimal_ident` has already been generated.
 
         .. versionadded:: 1.3a2
+        .. versionchanged:: 1.4
+           Stop showing generated names in the `repr` when the ``minimal_ident``
+           hasn't been requested. This reduces overhead and may be less confusing,
+           since ``minimal_ident`` can get reused.
         """
-        return 'Greenlet-%d' % (self.minimal_ident)
+        return 'Greenlet-%d' % (self.minimal_ident,)
 
     def _raise_exception(self):
         reraise(*self.exc_info)
@@ -426,7 +431,14 @@ class Greenlet(greenlet):
 
     def __repr__(self):
         classname = self.__class__.__name__
-        result = '<%s "%s" at %s' % (classname, self.name, hex(id(self)))
+        # If no name has been assigned, don't generate one, including a minimal_ident,
+        # if not necessary. This reduces the use of weak references and associated
+        # overhead.
+        if 'name' not in self.__dict__ and self._ident is None:
+            name = ' '
+        else:
+            name = ' "%s" ' % (self.name,)
+        result = '<%s%sat %s' % (classname, name, hex(id(self)))
         formatted = self._formatinfo()
         if formatted:
             result += ': ' + formatted

src/gevent/hub.py

@@ -339,8 +339,6 @@ def reinit(hub=None):
     #sleep(0.00001)
 
 
-hub_ident_registry = IdentRegistry()
-
 class Hub(WaitOperationsGreenlet):
     """
     A greenlet that runs the event loop.
@@ -386,6 +384,15 @@ class Hub(WaitOperationsGreenlet):
     # because that conflicts with the slot we inherit from the
     # Cythonized-bases.
 
+    # This is the source for our 'minimal_ident' property. We don't use a
+    # IdentRegistry because we've seen some crashes having to do with
+    # clearing weak references on shutdown in Windows (see known_failures.py).
+    # This gives us slightly different semantics than a greenlet's minimal_ident
+    # (notably, there can be holes) but we never documented this object's minimal_ident,
+    # and there should be few enough hub's over the lifetime of a process so as not
+    # to matter much.
+    _hub_counter = 0
+
     def __init__(self, loop=None, default=None):
         WaitOperationsGreenlet.__init__(self, None, None)
         self.thread_ident = get_thread_ident()
@@ -408,7 +415,9 @@ class Hub(WaitOperationsGreenlet):
         self._resolver = None
         self._threadpool = None
         self.format_context = GEVENT_CONFIG.format_context
-        self.minimal_ident = hub_ident_registry.get_ident(self)
+
+        Hub._hub_counter += 1
+        self.minimal_ident = Hub._hub_counter
 
     @Lazy
     def ident_registry(self):

src/gevent/testing/patched_tests_setup.py

@@ -461,9 +461,11 @@ if LIBUV:
             # Inserting GCs doesn't fix it.
             'test_ssl.ThreadedTests.test_handshake_timeout',
 
-            # These sometimes raise LoopExit, for no apparent reason.
+            # These sometimes raise LoopExit, for no apparent reason,
+            # mostly but not exclusively on Python 2.
             'test_socket.BufferIOTest.testRecvFromIntoBytearray',
             'test_socket.BufferIOTest.testRecvFromIntoArray',
+            'test_socket.BufferIOTest.testRecvFromIntoEmptyBuffer',
         ]
 
         if PY3:
@@ -593,6 +595,7 @@ if WIN:
         'test_socket.SendfileUsingSendTest.testWithTimeout': _flaky_socket_timeout,
         'test_socket.SendfileUsingSendTest.testOffset': _flaky_socket_timeout,
         'test_socket.SendfileUsingSendTest.testRegularFile': _flaky_socket_timeout,
+        'test_socket.SendfileUsingSendTest.testCount': _flaky_socket_timeout,
     })
 
 if PYPY:

src/gevent/tests/known_failures.py

@@ -110,7 +110,73 @@ if sys.platform == 'win32':
                 #     File "C:\Python37-x64\lib\weakref.py", line 356 in remove
 
                 # ! C:\Python37-x64\python.exe -u -mgevent.tests.test__greenness [code 3221225477] [took 1.3s]
-                'FLAKY test__greenness.py',
+                # We have also seen this for Python 3.6.6 Nov 13 2018:
+                # | C:\Python36-x64\python.exe -u -mgevent.tests.test__backdoor
+                #   ss.s.s
+                #   ----------------------------------------------------------------------
+                #   Ran 6 tests in 0.953s
+
+                #   OK (skipped=4)
+                #   Windows fatal exception: access violation
+
+                #   Thread 0x00000aec (most recent call first):
+                #     File "C:\Python36-x64\lib\site-packages\gevent\_threading.py", line 84 in wait
+                #     File "C:\Python36-x64\lib\site-packages\gevent\_threading.py", line 166 in get
+                #     File "C:\Python36-x64\lib\site-packages\gevent\threadpool.py", line 270 in _worker
+
+                #   Thread 0x00000548 (most recent call first):
+
+                #   Thread 0x000003d0 (most recent call first):
+                #     File "C:\Python36-x64\lib\site-packages\gevent\_threading.py", line 84 in wait
+                #     File "C:\Python36-x64\lib\site-packages\gevent\_threading.py", line 166 in get
+                #     File "C:\Python36-x64\lib\site-packages\gevent\threadpool.py", line 270 in _worker
+
+                #   Thread 0x00000ad0 (most recent call first):
+
+                #   Thread 0x00000588 (most recent call first):
+                #     File "C:\Python36-x64\lib\site-packages\gevent\_threading.py", line 84 in wait
+                #     File "C:\Python36-x64\lib\site-packages\gevent\_threading.py", line 166 in get
+                #     File "C:\Python36-x64\lib\site-packages\gevent\threadpool.py", line 270 in _worker
+
+                #   Thread 0x00000a54 (most recent call first):
+
+                #   Thread 0x00000768 (most recent call first):
+                #     File "C:\Python36-x64\lib\site-packages\gevent\_threading.py", line 84 in wait
+                #     File "C:\Python36-x64\lib\site-packages\gevent\_threading.py", line 166 in get
+                #     File "C:\Python36-x64\lib\site-packages\gevent\threadpool.py", line 270 in _worker
+
+                #   Current thread 0x00000894 (most recent call first):
+                #     File "C:\Python36-x64\lib\site-packages\gevent\threadpool.py", line 261 in _worker
+
+                #   Thread 0x00000634 (most recent call first):
+                #     File "C:\Python36-x64\lib\site-packages\gevent\_threading.py", line 84 in wait
+                #     File "C:\Python36-x64\lib\site-packages\gevent\_threading.py", line 166 in get
+                #     File "C:\Python36-x64\lib\site-packages\gevent\threadpool.py", line 270 in _worker
+
+                #   Thread 0x00000538 (most recent call first):
+
+                #   Thread 0x0000049c (most recent call first):
+                #     File "C:\Python36-x64\lib\weakref.py", line 356 in remove
+
+                # ! C:\Python36-x64\python.exe -u -mgevent.tests.test__backdoor [code 3221225477] [Ran 6 tests in 2.1s]
+
+                # Note the common factors:
+                # - The test is finished (successfully) and we're apparently exiting the VM,
+                #   doing GC
+                # - A weakref is being cleaned up
+
+                # weakref.py line 356 remove() is in WeakKeyDictionary. We only use WeakKeyDictionary
+                # in gevent._ident.IdentRegistry, which is only used in two places:
+                # gevent.hub.hub_ident_registry, which has weak references to Hub objects,
+                # and gevent.greenlet.Greenlet.minimal_ident, which uses its parent Hub's
+                # IdentRegistry to get its own identifier. So basically they have weak references
+                # to Hub and arbitrary Greenlets.
+
+                # Our attempted solution: stop using a module-level IdentRegistry to get
+                # Hub idents, and reduce how often we auto-generate one for greenlets.
+                # Commenting out the tests, lets see if it works.
+                #'FLAKY test__greenness.py',
+                #'FLAKY test__backdoor.py',
             ]
 
     if not PY35:

src/gevent/tests/test__greenlet.py

@@ -424,13 +424,24 @@ class TestStr(greentest.TestCase):
         assert_not_ready(g)
         g.join()
         assert_ready(g)
-        self.assertTrue(hexobj.sub('X', str(g)).endswith(' at X: dummy_test_func>'))
+        self.assertTrue(hexobj.sub('X', str(g)).endswith(' at X: dummy_test_func>'), str(g))
 
     def test_method(self):
         g = gevent.Greenlet.spawn(A().method)
         str_g = hexobj.sub('X', str(g))
         str_g = str_g.replace(__name__, 'module')
-        self.assertTrue(str_g.startswith('<Greenlet "Greenlet-'))
+        self.assertTrue(str_g.startswith('<Greenlet at X:'), str_g)
+        # Accessing the name to generate a minimal_ident will cause it to be included.
+        getattr(g, 'name')
+        str_g = hexobj.sub('X', str(g))
+        str_g = str_g.replace(__name__, 'module')
+        self.assertTrue(str_g.startswith('<Greenlet "Greenlet-'), str_g)
+        # Assigning to the name changes it
+        g.name = 'Foo'
+        str_g = hexobj.sub('X', str(g))
+        str_g = str_g.replace(__name__, 'module')
+        self.assertTrue(str_g.startswith('<Greenlet "Foo"'), str_g)
+
         self.assertTrue(str_g.endswith('at X: <bound method A.method of <module.A object at X>>>'))
         assert_not_ready(g)
         g.join()
@@ -443,7 +454,7 @@ class TestStr(greentest.TestCase):
         g = Subclass()
         str_g = hexobj.sub('X', str(g))
         str_g = str_g.replace(__name__, 'module')
-        self.assertTrue(str_g.startswith('<Subclass '))
+        self.assertTrue(str_g.startswith('<Subclass '), str_g)
         self.assertTrue(str_g.endswith('at X: _run>'))
 
         g = Subclass(None, 'question', answer=42)

src/gevent/tests/test__hub.py

@@ -115,8 +115,11 @@ class TestWaiter(greentest.TestCase):
 
         waiter = Waiter()
         g = gevent.spawn(waiter.get)
+        g.name = 'AName'
         gevent.sleep(0)
-        self.assertTrue(str(waiter).startswith('<Waiter greenlet=<Greenlet "Greenlet-'))
+        str_waiter = str(waiter)
+        self.assertTrue(str_waiter.startswith('<Waiter greenlet=<Greenlet "AName'),
+                        str_waiter)
 
         g.kill()
 

src/gevent/tests/test__util.py

@@ -221,23 +221,37 @@ class TestAssertSwitches(unittest.TestCase):
     def test_time_sleep(self):
         # A real blocking function
         from time import sleep
-        with self.assertRaises(util._FailedToSwitch):
+
+        # No time given, we detect the failure to switch immediately
+        with self.assertRaises(util._FailedToSwitch) as exc:
             with util.assert_switches():
                 sleep(0.001)
 
-        # Supply a max allowed and exceed it
+        message = str(exc.exception)
+        self.assertIn('To any greenlet in', message)
+
+        # Supply a max blocking allowed and exceed it
         with self.assertRaises(util._FailedToSwitch):
             with util.assert_switches(0.001):
                 sleep(0.1)
 
 
-        # Stay within it, but don't switch to the hub
-        with self.assertRaises(util._FailedToSwitch):
+        # Supply a max blocking allowed, and exit before that happens,
+        # but don't switch to the hub as requested
+        with self.assertRaises(util._FailedToSwitch) as exc:
             with util.assert_switches(0.001, hub_only=True):
                 sleep(0)
 
-        # Stay within it, and we only watch for any switch
-        with util.assert_switches(0.001, hub_only=False):
+        message = str(exc.exception)
+        self.assertIn('To the hub in', message)
+        self.assertIn('(max allowed 0.0010 seconds)', message)
+
+        # Supply a max blocking allowed, and exit before that happens,
+        # and allow any switch (or no switch).
+        # Note that we need to use a relatively long duration;
+        # sleep(0) on Windows can actually take a substantial amount of time
+        # sometimes (more than 0.001s)
+        with util.assert_switches(1.0, hub_only=False):
             sleep(0)
 
 

src/gevent/util.py

@@ -12,6 +12,7 @@ import traceback
 
 from greenlet import getcurrent
 
+from gevent._compat import perf_counter
 from gevent._compat import PYPY
 from gevent._compat import thread_mod_name
 from gevent._util import _NONE
@@ -541,10 +542,15 @@ class assert_switches(object):
             pass
 
     .. versionadded:: 1.3
+
+    .. versionchanged:: 1.4
+        If an exception is raised, it now includes information about
+        the duration of blocking and the parameters of this object.
     """
 
     hub = None
     tracer = None
+    _entered = None
 
 
     def __init__(self, max_blocking_time=None, hub_only=False):
@@ -567,6 +573,7 @@ class assert_switches(object):
         else:
             self.tracer = _tracer.MaxSwitchTracer(hub, self.max_blocking_time)
 
+        self._entered = perf_counter()
         self.tracer.monitor_current_greenlet_blocking()
         return self
 
@@ -583,6 +590,14 @@ class assert_switches(object):
 
         did_block = tracer.did_block_hub(hub)
         if did_block:
+            execution_time_s = perf_counter() - self._entered
             active_greenlet = did_block[1]
             report_lines = tracer.did_block_hub_report(hub, active_greenlet, {})
-            raise _FailedToSwitch('\n'.join(report_lines))
+
+            message = 'To the hub' if self.hub_only else 'To any greenlet'
+            message += ' in %.4f seconds' % (execution_time_s,)
+            max_block = self.max_blocking_time
+            message += ' (max allowed %.4f seconds)' % (max_block,) if max_block else ''
+            message += '\n'
+            message += '\n'.join(report_lines)
+            raise _FailedToSwitch(message)