Add support for /etc/hosts to dnspython resolver.

Based on code from eventlet, but heavily refactored and modified:

- Split host parsing into its own class, re-used from the test case
- Hosts handle PTR lookups for ipv4
- Some bug fixes around what should be strings where.
- Allow strings for the rdtype paramater
- Watch last modified time instead of reloading the file at fixed
  intervals.

More of our DNS tests pass now (notable TestEtcHosts is now enabled
for this resolver), and we tend to generate more conformant results
than the ares resolver (by observation, not tested).

src/gevent/queue.py

@@ -45,6 +45,9 @@ from gevent.hub import InvalidSwitchError
 __implements__ = ['Queue', 'PriorityQueue', 'LifoQueue']
 __extensions__ = ['JoinableQueue', 'Channel']
 __imports__ = ['Empty', 'Full']
+if hasattr(__queue__, 'SimpleQueue'):
+    __imports__.append('SimpleQueue') # New in 3.7
+    SimpleQueue = __queue__.SimpleQueue
 __all__ = __implements__ + __extensions__ + __imports__
 
 

src/gevent/resolver/__init__.py

@@ -63,8 +63,12 @@ def _lookup_port(port, socktype):
         socktypes.append(socktype)
     return port, socktypes
 
+hostname_types = tuple(set(string_types + (bytearray, bytes)))
 
 def _resolve_special(hostname, family):
+    if not isinstance(hostname, hostname_types):
+        raise TypeError("argument 1 must be str, bytes or bytearray, not %s" % (type(hostname),))
+
     if hostname == '':
         result = getaddrinfo(None, 0, family, SOCK_DGRAM, 0, AI_PASSIVE)
         if len(result) != 1:
@@ -80,7 +84,7 @@ class AbstractResolver(object):
         return self.gethostbyname_ex(hostname, family)[-1][0]
 
     def gethostbyname_ex(self, hostname, family=AF_INET):
-        aliases = []
+        aliases = self._getaliases(hostname, family)
         addresses = []
         tuples = self.getaddrinfo(hostname, 0, family,
                                   SOCK_STREAM,
@@ -93,3 +97,7 @@ class AbstractResolver(object):
 
     def getaddrinfo(self, host, port, family=0, socktype=0, proto=0, flags=0):
         raise NotImplementedError()
+
+    def _getaliases(self, hostname, family):
+        # pylint:disable=unused-argument
+        return []

src/greentest/test__socket_dns.py

@@ -50,7 +50,7 @@ def format_call(function, args):
     if args.endswith(',)'):
         args = args[:-2] + ')'
     try:
-        module = function.__module__.replace('gevent.socket', 'gevent').replace('_socket', 'stdlib')
+        module = function.__module__.replace('gevent._socketcommon', 'gevent')
         name = function.__name__
         return '%s:%s%s' % (module, name, args)
     except AttributeError:
@@ -166,13 +166,16 @@ def relaxed_is_equal(a, b):
     return all(relaxed_is_equal(x, y) for (x, y) in zip(a, b))
 
 
-def add(klass, hostname, name=None):
+def add(klass, hostname, name=None,
+        skip=None, skip_reason=None):
 
     call = callable(hostname)
 
-    def _setattr(k, n, v):
+    def _setattr(k, n, func):
+        if skip:
+            func = greentest.skipIf(skip, skip_reason,)(func)
         if not hasattr(k, n):
-            setattr(k, n, v)
+            setattr(k, n, func)
 
     if name is None:
         if call:
@@ -283,7 +286,8 @@ class TestCase(greentest.TestCase):
         ips = result[2]
         if ips == ['127.0.0.1', '127.0.0.1']:
             ips = ['127.0.0.1']
-        return (result[0], [], ips)
+        # On some systems, the hostname can get caps
+        return (result[0].lower(), [], ips)
 
     def _normalize_result_getaddrinfo(self, result):
         if not RESOLVER_NOT_SYSTEM:
@@ -345,8 +349,6 @@ add(TestTypeError, None)
 add(TestTypeError, 25)
 
 
-@unittest.skipIf(RESOLVER_DNSPYTHON,
-                 "This commonly needs /etc/hosts to function, and dnspython doesn't do that.")
 class TestHostname(TestCase):
     pass
 
@@ -423,33 +425,45 @@ class TestBroadcast(TestCase):
 add(TestBroadcast, '<broadcast>')
 
 
-@unittest.skipIf(RESOLVER_DNSPYTHON, "/etc/hosts completely ignored under dnspython")
+from gevent.resolver.dnspython import HostsFile
+class SanitizedHostsFile(HostsFile):
+    def iter_all_host_addr_pairs(self):
+        for name, addr in super(SanitizedHostsFile, self).iter_all_host_addr_pairs():
+            if (RESOLVER_NOT_SYSTEM
+                    and (name.endswith('local') # ignore bonjour, ares can't find them
+                         # ignore common aliases that ares can't find
+                         or addr == '255.255.255.255'
+                         or name == 'broadcasthost'
+                         # We get extra results from some impls, like OS X
+                         # it returns DGRAM results
+                         or name == 'localhost')):
+                continue
+            if name.endswith('local'):
+                # These can only be found if bonjour is running,
+                # and are very slow to do so with the system resolver on OS X
+                continue
+            yield name, addr
+
 class TestEtcHosts(TestCase):
-    pass
 
-try:
-    with open('/etc/hosts') as f:
-        etc_hosts = f.read()
-except IOError:
-    etc_hosts = ''
-
-for ip, host in re.findall(r'^\s*(\d+\.\d+\.\d+\.\d+)\s+([^\s]+)', etc_hosts, re.M)[:10]:
-    if (RESOLVER_NOT_SYSTEM
-        and (host.endswith('local') # ignore bonjour, ares can't find them
-             # ignore common aliases that ares can't find
-             or ip == '255.255.255.255'
-             or host == 'broadcasthost'
-             # We get extra results from some impls, like OS X
-             # it returns DGRAM results
-             or host == 'localhost')):
-        continue
-    if host.endswith('local'):
-        # These can only be found if bonjour is running,
-        # and are very slow to do so with the system resolver on OS X
-        continue
-    add(TestEtcHosts, host)
-    add(TestEtcHosts, ip)
-    del host, ip
+    MAX_HOSTS = os.getenv('GEVENTTEST_MAX_ETC_HOSTS', 10)
+
+    @classmethod
+    def populate_tests(cls):
+        hf = SanitizedHostsFile(os.path.join(os.path.dirname(__file__),
+                                             'hosts_file.txt'))
+        all_etc_hosts = sorted(hf.iter_all_host_addr_pairs())
+        if len(all_etc_hosts) > cls.MAX_HOSTS and not DEBUG:
+            all_etc_hosts = all_etc_hosts[:cls.MAX_HOSTS]
+
+        for host, ip in all_etc_hosts:
+            add(cls, host)
+            add(cls, ip)
+
+
+
+TestEtcHosts.populate_tests()
+
 
 
 class TestGeventOrg(TestCase):
@@ -544,12 +558,12 @@ class Test_getaddrinfo(TestCase):
 class TestInternational(TestCase):
     pass
 
-if not (PY2 and RESOLVER_DNSPYTHON):
-    # dns python can actually resolve these: it uses
-    # the 2008 version of idna encoding, whereas on Python 2,
-    # with the default resolver, it tries to encode to ascii and
-    # raises a UnicodeEncodeError. So we get different results.
-    add(TestInternational, u'президент.рф', 'russian')
+# dns python can actually resolve these: it uses
+# the 2008 version of idna encoding, whereas on Python 2,
+# with the default resolver, it tries to encode to ascii and
+# raises a UnicodeEncodeError. So we get different results.
+add(TestInternational, u'президент.рф', 'russian',
+    skip=(PY2 and RESOLVER_DNSPYTHON), skip_reason="dnspython can actually resolve these")
 add(TestInternational, u'президент.рф'.encode('idna'), 'idna')