Merge pull request #1418 from gevent/issue1410

Make dnspython optional for testing.

src/gevent/resolver/_addresses.py

+# -*- coding: utf-8 -*-
+# Copyright (c) 2019  gevent contributors. See LICENSE for details.
+#
+# Portions of this code taken from dnspython
+#   https://github.com/rthalley/dnspython
+#
+# Copyright (C) Dnspython Contributors, see LICENSE for text of ISC license
+
+# Copyright (C) 2003-2017 Nominum, Inc.
+#
+# Permission to use, copy, modify, and distribute this software and its
+# documentation for any purpose with or without fee is hereby granted,
+# provided that the above copyright notice and this permission notice
+# appear in all copies.
+#
+# THE SOFTWARE IS PROVIDED "AS IS" AND NOMINUM DISCLAIMS ALL WARRANTIES
+# WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
+# MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL NOMINUM BE LIABLE FOR
+# ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+# WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
+# ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT
+# OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+"""
+Private support for parsing textual addresses.
+
+"""
+from __future__ import absolute_import, division, print_function
+
+import binascii
+import re
+
+from gevent.resolver import hostname_types
+
+
+class AddressSyntaxError(ValueError):
+    pass
+
+
+def _ipv4_inet_aton(text):
+    """
+    Convert an IPv4 address in text form to binary struct.
+
+    *text*, a ``text``, the IPv4 address in textual form.
+
+    Returns a ``binary``.
+    """
+
+    if not isinstance(text, bytes):
+        text = text.encode()
+    parts = text.split(b'.')
+    if len(parts) != 4:
+        raise AddressSyntaxError(text)
+    for part in parts:
+        if not part.isdigit():
+            raise AddressSyntaxError
+        if len(part) > 1 and part[0] == '0':
+            # No leading zeros
+            raise AddressSyntaxError(text)
+    try:
+        ints = [int(part) for part in parts]
+        return struct.pack('BBBB', *ints)
+    except:
+        raise AddressSyntaxError(text)
+
+
+def _ipv6_inet_aton(text,
+                    _v4_ending=re.compile(br'(.*):(\d+\.\d+\.\d+\.\d+)$'),
+                    _colon_colon_start=re.compile(br'::.*'),
+                    _colon_colon_end=re.compile(br'.*::$')):
+    """
+    Convert an IPv6 address in text form to binary form.
+
+    *text*, a ``text``, the IPv6 address in textual form.
+
+    Returns a ``binary``.
+    """
+    # pylint:disable=too-many-branches
+
+    #
+    # Our aim here is not something fast; we just want something that works.
+    #
+    if not isinstance(text, bytes):
+        text = text.encode()
+
+    if text == b'::':
+        text = b'0::'
+    #
+    # Get rid of the icky dot-quad syntax if we have it.
+    #
+    m = _v4_ending.match(text)
+    if not m is None:
+        b = bytearray(_ipv4_inet_aton(m.group(2)))
+        text = (u"{}:{:02x}{:02x}:{:02x}{:02x}".format(m.group(1).decode(),
+                                                       b[0], b[1], b[2],
+                                                       b[3])).encode()
+    #
+    # Try to turn '::<whatever>' into ':<whatever>'; if no match try to
+    # turn '<whatever>::' into '<whatever>:'
+    #
+    m = _colon_colon_start.match(text)
+    if not m is None:
+        text = text[1:]
+    else:
+        m = _colon_colon_end.match(text)
+        if not m is None:
+            text = text[:-1]
+    #
+    # Now canonicalize into 8 chunks of 4 hex digits each
+    #
+    chunks = text.split(b':')
+    l = len(chunks)
+    if l > 8:
+        raise SyntaxError
+    seen_empty = False
+    canonical = []
+    for c in chunks:
+        if c == b'':
+            if seen_empty:
+                raise AddressSyntaxError(text)
+            seen_empty = True
+            for _ in range(0, 8 - l + 1):
+                canonical.append(b'0000')
+        else:
+            lc = len(c)
+            if lc > 4:
+                raise AddressSyntaxError(text)
+            if lc != 4:
+                c = (b'0' * (4 - lc)) + c
+            canonical.append(c)
+    if l < 8 and not seen_empty:
+        raise AddressSyntaxError(text)
+    text = b''.join(canonical)
+
+    #
+    # Finally we can go to binary.
+    #
+    try:
+        return binascii.unhexlify(text)
+    except (binascii.Error, TypeError):
+        raise AddressSyntaxError(text)
+
+
+def _is_addr(host, parse=_ipv4_inet_aton):
+    if not host:
+        return False
+    assert isinstance(host, hostname_types), repr(host)
+    try:
+        parse(host)
+    except AddressSyntaxError:
+        return False
+    else:
+        return True
+
+# Return True if host is a valid IPv4 address
+is_ipv4_addr = _is_addr
+
+
+def is_ipv6_addr(host):
+    # Return True if host is a valid IPv6 address
+    if host:
+        s = '%' if isinstance(host, str) else b'%'
+        host = host.split(s, 1)[0]
+    return _is_addr(host, _ipv6_inet_aton)

src/gevent/resolver/_hostsfile.py

+# -*- coding: utf-8 -*-
+# Copyright (c) 2019  gevent contributors. See LICENSE for details.
+#
+# Portions of this code taken from dnspython
+#   https://github.com/rthalley/dnspython
+#
+# Copyright (C) Dnspython Contributors, see LICENSE for text of ISC license
+
+# Copyright (C) 2003-2017 Nominum, Inc.
+#
+# Permission to use, copy, modify, and distribute this software and its
+# documentation for any purpose with or without fee is hereby granted,
+# provided that the above copyright notice and this permission notice
+# appear in all copies.
+#
+# THE SOFTWARE IS PROVIDED "AS IS" AND NOMINUM DISCLAIMS ALL WARRANTIES
+# WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
+# MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL NOMINUM BE LIABLE FOR
+# ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+# WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
+# ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT
+# OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+"""
+Private support for parsing /etc/hosts.
+
+"""
+from __future__ import absolute_import, division, print_function
+
+import sys
+import os
+import re
+
+from gevent.resolver._addresses import is_ipv4_addr
+from gevent.resolver._addresses import is_ipv6_addr
+
+from gevent._compat import iteritems
+
+
+class HostsFile(object):
+    """
+    A class to read the contents of a hosts file (/etc/hosts).
+    """
+
+    LINES_RE = re.compile(r"""
+        \s*  # Leading space
+        ([^\r\n#]+?)  # The actual match, non-greedy so as not to include trailing space
+        \s*  # Trailing space
+        (?:[#][^\r\n]+)?  # Comments
+        (?:$|[\r\n]+)  # EOF or newline
+    """, re.VERBOSE)
+
+    def __init__(self, fname=None):
+        self.v4 = {} # name -> ipv4
+        self.v6 = {} # name -> ipv6
+        self.aliases = {} # name -> canonical_name
+        self.reverse = {} # ip addr -> some name
+        if fname is None:
+            if os.name == 'posix':
+                fname = '/etc/hosts'
+            elif os.name == 'nt': # pragma: no cover
+                fname = os.path.expandvars(
+                    r'%SystemRoot%\system32\drivers\etc\hosts')
+        self.fname = fname
+        assert self.fname
+        self._last_load = 0
+
+
+    def _readlines(self):
+        # Read the contents of the hosts file.
+        #
+        # Return list of lines, comment lines and empty lines are
+        # excluded. Note that this performs disk I/O so can be
+        # blocking.
+        with open(self.fname, 'rb') as fp:
+            fdata = fp.read()
+
+
+        # XXX: Using default decoding. Is that correct?
+        udata = fdata.decode(errors='ignore') if not isinstance(fdata, str) else fdata
+
+        return self.LINES_RE.findall(udata)
+
+    def load(self): # pylint:disable=too-many-locals
+        # Load hosts file
+
+        # This will (re)load the data from the hosts
+        # file if it has changed.
+
+        try:
+            load_time = os.stat(self.fname).st_mtime
+            needs_load = load_time > self._last_load
+        except (IOError, OSError):
+            from gevent import get_hub
+            get_hub().handle_error(self, *sys.exc_info())
+            needs_load = False
+
+        if not needs_load:
+            return
+
+        v4 = {}
+        v6 = {}
+        aliases = {}
+        reverse = {}
+
+        for line in self._readlines():
+            parts = line.split()
+            if len(parts) < 2:
+                continue
+            ip = parts.pop(0)
+            if is_ipv4_addr(ip):
+                ipmap = v4
+            elif is_ipv6_addr(ip):
+                if ip.startswith('fe80'):
+                    # Do not use link-local addresses, OSX stores these here
+                    continue
+                ipmap = v6
+            else:
+                continue
+            cname = parts.pop(0).lower()
+            ipmap[cname] = ip
+            for alias in parts:
+                alias = alias.lower()
+                ipmap[alias] = ip
+                aliases[alias] = cname
+
+            # XXX: This is wrong for ipv6
+            if ipmap is v4:
+                ptr = '.'.join(reversed(ip.split('.'))) + '.in-addr.arpa'
+            else:
+                ptr = ip + '.ip6.arpa.'
+            if ptr not in reverse:
+                reverse[ptr] = cname
+
+        self._last_load = load_time
+        self.v4 = v4
+        self.v6 = v6
+        self.aliases = aliases
+        self.reverse = reverse
+
+    def iter_all_host_addr_pairs(self):
+        self.load()
+        for name, addr in iteritems(self.v4):
+            yield name, addr
+        for name, addr in iteritems(self.v6):
+            yield name, addr

src/gevent/resolver/dnspython.py

@@ -60,9 +60,6 @@
 # THE SOFTWARE.
 from __future__ import absolute_import, print_function, division
 
-import os
-
-import re
 import sys
 import time
 
@@ -78,6 +75,8 @@ import socket
 
 from gevent.resolver import AbstractResolver
 from gevent.resolver import hostname_types
+from gevent.resolver._hostsfile import HostsFile
+from gevent.resolver._addresses import is_ipv6_addr
 
 from gevent._compat import string_types
 from gevent._compat import iteritems
@@ -172,135 +171,7 @@ resolver._getaddrinfo = _getaddrinfo
 
 HOSTS_TTL = 300.0
 
-def _is_addr(host, parse=dns.ipv4.inet_aton):
-    if not host:
-        return False
-    assert isinstance(host, hostname_types), repr(host)
-    try:
-        parse(host)
-    except dns.exception.SyntaxError:
-        return False
-    else:
-        return True
-
-# Return True if host is a valid IPv4 address
-_is_ipv4_addr = _is_addr
-
-
-def _is_ipv6_addr(host):
-    # Return True if host is a valid IPv6 address
-    if host:
-        s = '%' if isinstance(host, str) else b'%'
-        host = host.split(s, 1)[0]
-    return _is_addr(host, dns.ipv6.inet_aton)
-
-class HostsFile(object):
-    """
-    A class to read the contents of a hosts file (/etc/hosts).
-    """
-
-    LINES_RE = re.compile(r"""
-        \s*  # Leading space
-        ([^\r\n#]+?)  # The actual match, non-greedy so as not to include trailing space
-        \s*  # Trailing space
-        (?:[#][^\r\n]+)?  # Comments
-        (?:$|[\r\n]+)  # EOF or newline
-    """, re.VERBOSE)
-
-    def __init__(self, fname=None):
-        self.v4 = {} # name -> ipv4
-        self.v6 = {} # name -> ipv6
-        self.aliases = {} # name -> canonical_name
-        self.reverse = {} # ip addr -> some name
-        if fname is None:
-            if os.name == 'posix':
-                fname = '/etc/hosts'
-            elif os.name == 'nt': # pragma: no cover
-                fname = os.path.expandvars(
-                    r'%SystemRoot%\system32\drivers\etc\hosts')
-        self.fname = fname
-        assert self.fname
-        self._last_load = 0
-
-
-    def _readlines(self):
-        # Read the contents of the hosts file.
-        #
-        # Return list of lines, comment lines and empty lines are
-        # excluded. Note that this performs disk I/O so can be
-        # blocking.
-        with open(self.fname, 'rb') as fp:
-            fdata = fp.read()
-
-
-        # XXX: Using default decoding. Is that correct?
-        udata = fdata.decode(errors='ignore') if not isinstance(fdata, str) else fdata
-
-        return self.LINES_RE.findall(udata)
 
-    def load(self): # pylint:disable=too-many-locals
-        # Load hosts file
-
-        # This will (re)load the data from the hosts
-        # file if it has changed.
-
-        try:
-            load_time = os.stat(self.fname).st_mtime
-            needs_load = load_time > self._last_load
-        except (IOError, OSError):
-            from gevent import get_hub
-            get_hub().handle_error(self, *sys.exc_info())
-            needs_load = False
-
-        if not needs_load:
-            return
-
-        v4 = {}
-        v6 = {}
-        aliases = {}
-        reverse = {}
-
-        for line in self._readlines():
-            parts = line.split()
-            if len(parts) < 2:
-                continue
-            ip = parts.pop(0)
-            if _is_ipv4_addr(ip):
-                ipmap = v4
-            elif _is_ipv6_addr(ip):
-                if ip.startswith('fe80'):
-                    # Do not use link-local addresses, OSX stores these here
-                    continue
-                ipmap = v6
-            else:
-                continue
-            cname = parts.pop(0).lower()
-            ipmap[cname] = ip
-            for alias in parts:
-                alias = alias.lower()
-                ipmap[alias] = ip
-                aliases[alias] = cname
-
-            # XXX: This is wrong for ipv6
-            if ipmap is v4:
-                ptr = '.'.join(reversed(ip.split('.'))) + '.in-addr.arpa'
-            else:
-                ptr = ip + '.ip6.arpa.'
-            if ptr not in reverse:
-                reverse[ptr] = cname
-
-        self._last_load = load_time
-        self.v4 = v4
-        self.v6 = v6
-        self.aliases = aliases
-        self.reverse = reverse
-
-    def iter_all_host_addr_pairs(self):
-        self.load()
-        for name, addr in iteritems(self.v4):
-            yield name, addr
-        for name, addr in iteritems(self.v6):
-            yield name, addr
 
 class _HostsAnswer(dns.resolver.Answer):
     # Answer class for HostsResolver object
@@ -536,7 +407,7 @@ class Resolver(AbstractResolver):
 
     def getaddrinfo(self, host, port, family=0, socktype=0, proto=0, flags=0):
         if ((host in (u'localhost', b'localhost')
-             or (_is_ipv6_addr(host) and host.startswith('fe80')))
+             or (is_ipv6_addr(host) and host.startswith('fe80')))
                 or not isinstance(host, str) or (flags & AI_NUMERICHOST)):
             # this handles cases which do not require network access
             # 1) host is None

src/gevent/testing/modules.py

@@ -29,14 +29,19 @@ from . import sysinfo
 from . import util
 
 
-OPTIONAL_MODULES = [
+OPTIONAL_MODULES = frozenset({
+    ## Resolvers.
+    # ares might not be built
     'gevent.resolver_ares',
     'gevent.resolver.ares',
+    # dnspython might not be installed
+    'gevent.resolver.dnspython',
+    ## Backends
     'gevent.libev',
     'gevent.libev.watcher',
     'gevent.libuv.loop',
     'gevent.libuv.watcher',
-]
+})
 
 
 def walk_modules(
@@ -45,6 +50,7 @@ def walk_modules(
         include_so=False,
         recursive=False,
         check_optional=True,
+        optional_modules=OPTIONAL_MODULES,
 ):
     """
     Find gevent modules, yielding tuples of ``(path, importable_module_name)``.
@@ -53,7 +59,7 @@ def walk_modules(
        module that is known to be optional on this system (such as a backend),
        we will attempt to import it; if the import fails, it will not be returned.
        If false, then we will not make such an attempt, the caller will need to be prepared
-       for an `ImportError`; the caller can examine *OPTIONAL_MODULES* against
+       for an `ImportError`; the caller can examine *optional_modules* against
        the yielded *importable_module_name*.
     """
     # pylint:disable=too-many-branches
@@ -78,7 +84,8 @@ def walk_modules(
             if os.path.exists(pkg_init):
                 yield pkg_init, modpath + fn
                 for p, m in walk_modules(path, modpath + fn + ".",
-                                         check_optional=check_optional):
+                                         check_optional=check_optional,
+                                         optional_modules=optional_modules):
                     yield p, m
             continue
 
@@ -90,7 +97,7 @@ def walk_modules(
                      'corecffi', '_corecffi', '_corecffi_build']:
                 continue
             modname = modpath + x
-            if check_optional and modname in OPTIONAL_MODULES:
+            if check_optional and modname in optional_modules:
                 try:
                     with warnings.catch_warnings():
                         warnings.simplefilter('ignore', DeprecationWarning)

src/gevent/tests/test___config.py

@@ -42,9 +42,16 @@ class TestResolver(unittest.TestCase):
         self.assertEqual(conf.get(), Resolver)
 
         # A new object reflects it
-        conf = _config.Resolver()
-        from gevent.resolver.dnspython import Resolver as DResolver
-        self.assertEqual(conf.get(), DResolver)
+        try:
+            from gevent.resolver.dnspython import Resolver as DResolver
+        except ImportError: # pragma: no cover
+            # dnspython is optional; skip it.
+            import warnings
+            warnings.warn('dnspython not installed')
+        else:
+            conf = _config.Resolver()
+
+            self.assertEqual(conf.get(), DResolver)
 
     def test_set_str_long(self):
         from gevent.resolver.blocking import Resolver

src/gevent/tests/test__all__.py

@@ -40,15 +40,17 @@ COULD_BE_MISSING = {
 # helpers
 NO_ALL = {
     'gevent.threading',
-    'gevent._util',
     'gevent._compat',
-    'gevent._socketcommon',
+    'gevent._corecffi',
+    'gevent._ffi',
     'gevent._fileobjectcommon',
     'gevent._fileobjectposix',
-    'gevent._tblib',
-    'gevent._corecffi',
     'gevent._patcher',
-    'gevent._ffi',
+    'gevent._socketcommon',
+    'gevent._tblib',
+    'gevent._util',
+    'gevent.resolver._addresses',
+    'gevent.resolver._hostsfile',
 }
 
 ALLOW_IMPLEMENTS = [
@@ -229,7 +231,9 @@ are missing from %r:
                 self.module = importlib.import_module(modname)
             except ImportError:
                 if modname in modules.OPTIONAL_MODULES:
-                    raise unittest.SkipTest("Unable to import %s" % modname)
+                    msg = "Unable to import %s" % modname
+                    warnings.warn(msg) # make the testrunner print it
+                    raise unittest.SkipTest(msg)
                 raise
 
         self.check_all()

src/gevent/tests/test__socket_dns.py

@@ -467,7 +467,7 @@ class TestBroadcast(TestCase):
 add(TestBroadcast, '<broadcast>')
 
 
-from gevent.resolver.dnspython import HostsFile # XXX: This will move.
+from gevent.resolver._hostsfile import HostsFile
 class SanitizedHostsFile(HostsFile):
     def iter_all_host_addr_pairs(self):
         for name, addr in super(SanitizedHostsFile, self).iter_all_host_addr_pairs():