Use PollSelector as DefaultSelector after monkey-patching if available.

Fixes #1466

Add extra tests around selectors.

CHANGES.rst

@@ -63,6 +63,13 @@ Other
   where `contextvars` is a standard library module, it is
   monkey-patched by default. See :issue:`1407`.
 
+- Use `selectors.PollSelector` as the `selectors.DefaultSelector`
+  after monkey-patching if `select.poll` was defined. Previously,
+  gevent replaced it with `selectors.SelectSelector`, which has a
+  different set of limitations (e.g., on certain platforms such as
+  glibc Linux, it has a hardcoded limitation of only working with file
+  descriptors < 1024). See :issue:`1466` reported by Sam Wong.
+
 1.5a3 (2020-01-01)
 ==================
 

src/gevent/monkey.py

@@ -286,8 +286,37 @@ def __call_module_hook(gevent_module, name, module, items, _warnings):
     func(module, items, warn)
 
 
+class _GeventDoPatchRequest(object):
+
+    PY3 = PY3
+    get_original = staticmethod(get_original)
+
+    def __init__(self,
+                 target_module,
+                 source_module,
+                 items,
+                 patch_kwargs):
+        self.target_module = target_module
+        self.source_module = source_module
+        self.items = items
+        self.patch_kwargs = patch_kwargs or {}
+
+    def default_patch_items(self):
+        for attr in self.items:
+            patch_item(self.target_module, attr, getattr(self.source_module, attr))
+
+    def remove_item(self, target_module, *items):
+        if isinstance(target_module, str):
+            items = (target_module,) + items
+            target_module = self.target_module
+
+        for item in items:
+            remove_item(target_module, item)
+
+
 def patch_module(target_module, source_module, items=None,
                  _warnings=None,
+                 _patch_kwargs=None,
                  _notify_will_subscribers=True,
                  _notify_did_subscribers=True,
                  _call_hooks=True):
@@ -335,8 +364,16 @@ def patch_module(target_module, source_module, items=None,
     except events.DoNotPatch:
         return False
 
-    for attr in items:
-        patch_item(target_module, attr, getattr(source_module, attr))
+    # Undocumented, internal use: If the module defines
+    # `_gevent_do_monkey_patch(patch_request: _GeventDoPatchRequest)` call that;
+    # the module is responsible for its own patching.
+    do_patch = getattr(
+        source_module,
+        '_gevent_do_monkey_patch',
+        _GeventDoPatchRequest.default_patch_items
+    )
+    request = _GeventDoPatchRequest(target_module, source_module, items, _patch_kwargs)
+    do_patch(request)
 
     if _call_hooks:
         __call_module_hook(source_module, 'did', target_module, items, _warnings)
@@ -355,6 +392,7 @@ def patch_module(target_module, source_module, items=None,
 def _patch_module(name,
                   items=None,
                   _warnings=None,
+                  _patch_kwargs=None,
                   _notify_will_subscribers=True,
                   _notify_did_subscribers=True,
                   _call_hooks=True):
@@ -364,7 +402,7 @@ def _patch_module(name,
     target_module = __import__(module_name)
 
     patch_module(target_module, gevent_module, items=items,
-                 _warnings=_warnings,
+                 _warnings=_warnings, _patch_kwargs=_patch_kwargs,
                  _notify_will_subscribers=_notify_will_subscribers,
                  _notify_did_subscribers=_notify_did_subscribers,
                  _call_hooks=_call_hooks)
@@ -946,56 +984,8 @@ def patch_select(aggressive=True):
     - :class:`selectors.KqueueSelector`
     - :class:`selectors.DevpollSelector` (Python 3.5+)
     """
-
-    source_mod, target_mod = _patch_module('select', _notify_did_subscribers=False)
-    if aggressive:
-        select = target_mod
-        # since these are blocking we're removing them here. This makes some other
-        # modules (e.g. asyncore)  non-blocking, as they use select that we provide
-        # when none of these are available.
-        remove_item(select, 'epoll')
-        remove_item(select, 'kqueue')
-        remove_item(select, 'kevent')
-        remove_item(select, 'devpoll')
-
-    if PY3:
-        # Python 3 wants to use `select.select` as a member function,
-        # leading to this error in selectors.py (because gevent.select.select is
-        # not a builtin and doesn't get the magic auto-static that they do)
-        #    r, w, _ = self._select(self._readers, self._writers, [], timeout)
-        #    TypeError: select() takes from 3 to 4 positional arguments but 5 were given
-        # Note that this obviously only happens if selectors was imported after we had patched
-        # select; but there is a code path that leads to it being imported first (but now we've
-        # patched select---so we can't compare them identically)
-        select = target_mod # Should be gevent-patched now
-        orig_select_select = get_original('select', 'select')
-        assert select.select is not orig_select_select
-        selectors = __import__('selectors')
-        if selectors.SelectSelector._select in (select.select, orig_select_select):
-            def _select(self, *args, **kwargs): # pylint:disable=unused-argument
-                return select.select(*args, **kwargs)
-            selectors.SelectSelector._select = _select
-            _select._gevent_monkey = True
-
-        # Python 3.7 refactors the poll-like selectors to use a common
-        # base class and capture a reference to select.poll, etc, at
-        # import time. selectors tends to get imported early
-        # (importing 'platform' does it: platform -> subprocess -> selectors),
-        # so we need to clean that up.
-        if hasattr(selectors, 'PollSelector') and hasattr(selectors.PollSelector, '_selector_cls'):
-            selectors.PollSelector._selector_cls = select.poll
-
-        if aggressive:
-            # If `selectors` had already been imported before we removed
-            # select.epoll|kqueue|devpoll, these may have been defined in terms
-            # of those functions. They'll fail at runtime.
-            remove_item(selectors, 'EpollSelector')
-            remove_item(selectors, 'KqueueSelector')
-            remove_item(selectors, 'DevpollSelector')
-            selectors.DefaultSelector = selectors.SelectSelector
-
-    from gevent import events
-    _notify_patch(events.GeventDidPatchModuleEvent('select', source_mod, target_mod))
+    _patch_module('select',
+                  _patch_kwargs={'aggressive': aggressive})
 
 @_ignores_DoNotPatch
 def patch_subprocess():

src/gevent/select.py

@@ -299,3 +299,67 @@ if original_poll is not None:
             del self.fds[fileno]
 
 del original_poll
+
+
+def _gevent_do_monkey_patch(patch_request):
+    aggressive = patch_request.patch_kwargs['aggressive']
+    target_mod = patch_request.target_module
+
+    patch_request.default_patch_items()
+
+    if aggressive:
+        # since these are blocking we're removing them here. This makes some other
+        # modules (e.g. asyncore)  non-blocking, as they use select that we provide
+        # when none of these are available.
+        patch_request.remove_item(
+            'epoll'
+            'kqueue',
+            'kevent',
+            'devpoll',
+        )
+
+    if patch_request.PY3:
+        # TODO: Do we need to broadcast events about patching the selectors
+        # package? If so, must be careful to deal with DoNotPatch exceptions.
+
+        # Python 3 wants to use `select.select` as a member function,
+        # leading to this error in selectors.py (because gevent.select.select is
+        # not a builtin and doesn't get the magic auto-static that they do)
+        #    r, w, _ = self._select(self._readers, self._writers, [], timeout)
+        #    TypeError: select() takes from 3 to 4 positional arguments but 5 were given
+        # Note that this obviously only happens if selectors was imported after we had patched
+        # select; but there is a code path that leads to it being imported first (but now we've
+        # patched select---so we can't compare them identically)
+
+        orig_select_select = patch_request.get_original('select', 'select')
+        assert target_mod.select is not orig_select_select
+        selectors = __import__('selectors')
+        if selectors.SelectSelector._select in (target_mod.select, orig_select_select):
+            def _select(self, *args, **kwargs): # pylint:disable=unused-argument
+                return select(*args, **kwargs)
+            selectors.SelectSelector._select = _select
+            _select._gevent_monkey = True # prove for test cases
+
+        # Python 3.7 refactors the poll-like selectors to use a common
+        # base class and capture a reference to select.poll, etc, at
+        # import time. selectors tends to get imported early
+        # (importing 'platform' does it: platform -> subprocess -> selectors),
+        # so we need to clean that up.
+        if hasattr(selectors, 'PollSelector') and hasattr(selectors.PollSelector, '_selector_cls'):
+            selectors.PollSelector._selector_cls = poll
+
+        if aggressive:
+            # If `selectors` had already been imported before we removed
+            # select.epoll|kqueue|devpoll, these may have been defined in terms
+            # of those functions. They'll fail at runtime.
+            patch_request.remove_item(
+                selectors,
+                'EpollSelector',
+                'KqueueSelector',
+                'DevpollSelector',
+            )
+            selectors.DefaultSelector = getattr(
+                selectors,
+                'PollSelector',
+                selectors.SelectSelector
+            )

src/gevent/tests/test__monkey_selectors.py

 
-import gevent.testing as greentest
 try:
-    import selectors # Do this before the patch, just to force it
+    # Do this before the patch to be sure we clean
+    # things up properly if the order is wrong.
+    import selectors
 except ImportError:
-    pass
+    selectors = None
+import socket
+
+import gevent
+from gevent import select
+
 from gevent.monkey import patch_all
+import gevent.testing as greentest
+
 patch_all()
 
-@greentest.skipUnless(
-    not greentest.WIN and greentest.PY3,
-    "selectors only guaranteed on Python 3 and not windows"
+@greentest.skipIf(
+    selectors is None,
+    "selectors module not present"
 )
 class TestSelectors(greentest.TestCase):
 
     def test_selectors_select_is_patched(self):
         # https://github.com/gevent/gevent/issues/835
         _select = selectors.SelectSelector._select
-        self.assertTrue(hasattr(_select, '_gevent_monkey'), dir(_select))
+        self.assertIn('_gevent_monkey', dir(_select))
+
+    @greentest.skipUnless(
+        hasattr(select, 'poll'),
+        "Needs gevent.select.poll"
+    )
+    def test_poll_is_default(self):
+        # https://github.com/gevent/gevent/issues/1466
+        self.assertIs(selectors.DefaultSelector, selectors.PollSelector)
+
+    def _check_selector(self, sel):
+        def read(conn, _mask):
+            data = conn.recv(100)  # Should be ready
+            if data:
+                conn.send(data)  # Hope it won't block
+
+            sel.unregister(conn)
+            conn.close()
+
+        def run_selector_once():
+            events = sel.select()
+            for key, mask in events:
+                key.data(key.fileobj, mask)
+
+        sock1, sock2 = socket.socketpair()
+        try:
+            sel.register(sock1, selectors.EVENT_READ, read)
+            glet = gevent.spawn(run_selector_once)
+            DATA = b'abcdef'
+            sock2.send(DATA)
+            data = sock2.recv(50)
+            self.assertEqual(data, DATA)
+        finally:
+            sel.close()
+            sock1.close()
+            sock2.close()
+            glet.join(10)
+        self.assertTrue(glet.ready())
+
+
+    def _make_test(name, kind): # pylint:disable=no-self-argument
+        if kind is None:
+            def m(self):
+                self.skipTest(name + ' is not defined')
+        else:
+            def m(self, k=kind):
+                sel = k()
+                self._check_selector(sel)
+        m.__name__ = 'test_selector_' + name
+        return m
+
+    SelKind = SelKindName = None
+    for SelKindName in (
+            # The subclass hierarchy changes between versions, and is
+            # complex (e.g, BaseSelector <- BaseSelectorImpl <-
+            # _PollLikSelector <- PollSelector) so its easier to check against
+            # names.
+            'KqueueSelector',
+            'EpollSelector',
+            'DevpollSelector',
+            'PollSelector',
+            'SelectSelector',
+    ):
+        SelKind = getattr(selectors, SelKindName, None)
+        m = _make_test(SelKindName, SelKind)
+        locals()[m.__name__] = m
+
+    del SelKind
+    del SelKindName
+    del _make_test
 
 
 if __name__ == '__main__':