Merge pull request #1594 from gevent/issue1012

Use `ares_getaddrinfo`

_setupares.py

@@ -95,7 +95,6 @@ ARES = Extension(
     libraries=list(LIBRARIES),
     define_macros=list(DEFINE_MACROS),
     depends=glob_many(
-        'src/gevent/resolver/dnshelper.c',
         'src/gevent/resolver/cares_*.[ch]')
 )
 

docs/changes/1012.bugfix

+Use ``ares_getaddrinfo`` instead of a manual lookup.
+
+This requires c-ares 1.16.0.
+
+Note that this may change the results, in particular their order.
+
+As part of this, certain parts of the c-ares extension were adapted to
+use modern Cython idioms.
+
+A few minor errors and discrepancies were fixed as well, such as
+``gethostbyaddr(''localhost')`` working on Python 3 and failing on
+Python 2. The DNSpython resolver now raises the expected TypeError in
+more cases instead of an AttributeError.

src/gevent/libev/_corecffi_source.c

@@ -76,7 +76,9 @@ static void gevent_zero_prepare(struct ev_prepare* handle)
 
 static void gevent_set_ev_alloc()
 {
-    ev_set_allocator(gevent_realloc);
+    void* (*ptr)(void*, long);
+    ptr = (void*(*)(void*, long))&gevent_realloc;
+    ev_set_allocator(ptr);
 }
 
 #ifdef __clang__

src/gevent/resolver/_addresses.py

@@ -142,9 +142,9 @@ def _ipv6_inet_aton(text,
 
 
 def _is_addr(host, parse=_ipv4_inet_aton):
-    if not host:
+    if not host or not isinstance(host, hostname_types):
         return False
-    assert isinstance(host, hostname_types), repr(host)
+
     try:
         parse(host)
     except AddressSyntaxError:
@@ -158,7 +158,7 @@ is_ipv4_addr = _is_addr
 
 def is_ipv6_addr(host):
     # Return True if host is a valid IPv6 address
-    if host:
+    if host and isinstance(host, hostname_types):
         s = '%' if isinstance(host, str) else b'%'
         host = host.split(s, 1)[0]
     return _is_addr(host, _ipv6_inet_aton)

src/gevent/resolver/cares_ntop.h

-#ifdef CARES_EMBED
-#include "ares_setup.h"
-#include "ares.h"
-#else
-#include <arpa/inet.h>
-#define ares_inet_ntop(w,x,y,z) inet_ntop(w,x,y,z)
-#endif

src/gevent/resolver/cares_pton.h

-#ifdef CARES_EMBED
-#include "ares_setup.h"
-#include "ares_inet_net_pton.h"
-#else
-#include <arpa/inet.h>
-#define ares_inet_pton(x,y,z) inet_pton(x,y,z)
-#define ares_inet_net_pton(w,x,y,z) inet_net_pton(w,x,y,z)
-#endif

src/gevent/resolver/dnshelper.c

-/* Copyright (c) 2011 Denis Bilenko. See LICENSE for details. */
-#include "Python.h"
-#ifdef CARES_EMBED
-#include "ares_setup.h"
-#endif
-
-#ifdef HAVE_NETDB_H
-#include <netdb.h>
-#endif
-
-#include "ares.h"
-
-#include "cares_ntop.h"
-#include "cares_pton.h"
-
-#if PY_MAJOR_VERSION >= 3
-#define PY3K
-#define GPyNative_FromString PyUnicode_FromString
-#else
-#define GPyNative_FromString PyString_FromString
-#endif
-
-
-static PyObject* _socket_error = 0;
-
-static PyObject*
-get_socket_object(PyObject** pobject, const char* name)
-{
-    if (!*pobject) {
-        PyObject* _socket;
-        _socket = PyImport_ImportModule("_socket");
-        if (_socket) {
-            *pobject = PyObject_GetAttrString(_socket, name);
-            if (!*pobject) {
-                PyErr_WriteUnraisable(Py_None);
-            }
-            Py_DECREF(_socket);
-        }
-        else {
-            PyErr_WriteUnraisable(Py_None);
-        }
-        if (!*pobject) {
-            *pobject = PyExc_IOError;
-        }
-    }
-    return *pobject;
-}
-
-
-static int
-gevent_append_addr(PyObject* list, int family, void* src, char* tmpbuf, size_t tmpsize) {
-    int status = -1;
-    PyObject* tmp;
-    if (ares_inet_ntop(family, src, tmpbuf, tmpsize)) {
-        tmp = GPyNative_FromString(tmpbuf);
-        if (tmp) {
-            status = PyList_Append(list, tmp);
-            Py_DECREF(tmp);
-        }
-    }
-    return status;
-}
-
-
-static PyObject*
-parse_h_name(struct hostent *h)
-{
-    return GPyNative_FromString(h->h_name);
-}
-
-
-static PyObject*
-parse_h_aliases(struct hostent *h)
-{
-    char **pch;
-    PyObject *result = NULL;
-    PyObject *tmp;
-
-    result = PyList_New(0);
-
-    if (result && h->h_aliases) {
-        for (pch = h->h_aliases; *pch != NULL; pch++) {
-            if (*pch != h->h_name && strcmp(*pch, h->h_name)) {
-                int status;
-                tmp = GPyNative_FromString(*pch);
-                if (tmp == NULL) {
-                    break;
-                }
-
-                status = PyList_Append(result, tmp);
-                Py_DECREF(tmp);
-
-                if (status) {
-                    break;
-                }
-            }
-        }
-    }
-
-    return result;
-}
-
-
-static PyObject *
-parse_h_addr_list(struct hostent *h)
-{
-    char **pch;
-    PyObject *result = NULL;
-
-    result = PyList_New(0);
-
-    if (result) {
-        switch (h->h_addrtype) {
-            case AF_INET:
-            {
-                char tmpbuf[sizeof "255.255.255.255"];
-                for (pch = h->h_addr_list; *pch != NULL; pch++) {
-                    if (gevent_append_addr(result, AF_INET, *pch, tmpbuf, sizeof(tmpbuf))) {
-                        break;
-                    }
-                }
-                break;
-            }
-            case AF_INET6:
-            {
-                char tmpbuf[sizeof("ffff:ffff:ffff:ffff:ffff:ffff:255.255.255.255")];
-                for (pch = h->h_addr_list; *pch != NULL; pch++) {
-                    if (gevent_append_addr(result, AF_INET6, *pch, tmpbuf, sizeof(tmpbuf))) {
-                        break;
-                    }
-                }
-                break;
-            }
-            default:
-                PyErr_SetString(get_socket_object(&_socket_error, "error"), "unsupported address family");
-                Py_DECREF(result);
-                result = NULL;
-        }
-    }
-
-    return result;
-}
-
-
-static int
-gevent_make_sockaddr(char* hostp, int port, int flowinfo, int scope_id, struct sockaddr_in6* sa6) {
-    if ( ares_inet_pton(AF_INET, hostp, &((struct sockaddr_in*)sa6)->sin_addr.s_addr) > 0 ) {
-        ((struct sockaddr_in*)sa6)->sin_family = AF_INET;
-        ((struct sockaddr_in*)sa6)->sin_port = htons(port);
-        return sizeof(struct sockaddr_in);
-    }
-    else if ( ares_inet_pton(AF_INET6, hostp, &sa6->sin6_addr.s6_addr) > 0 ) {
-        sa6->sin6_family = AF_INET6;
-        sa6->sin6_port = htons(port);
-        sa6->sin6_flowinfo = flowinfo;
-        sa6->sin6_scope_id = scope_id;
-        return sizeof(struct sockaddr_in6);
-    }
-    return -1;
-}

src/gevent/resolver/dnspython.py

@@ -393,13 +393,17 @@ class Resolver(AbstractResolver):
         aliases = self._resolver.hosts_resolver.getaliases(hostname)
         net_resolver = self._resolver.network_resolver
         rdtype = _family_to_rdtype(family)
-        while True:
+        while 1:
             try:
                 ans = net_resolver.query(hostname, dns.rdatatype.CNAME, rdtype)
             except (dns.resolver.NoAnswer, dns.resolver.NXDOMAIN, dns.resolver.NoNameservers):
                 break
             except dTimeout:
                 break
+            except AttributeError as ex:
+                if hostname is None or isinstance(hostname, int):
+                    raise TypeError(ex)
+                raise
             else:
                 aliases.extend(str(rr.target) for rr in ans.rrset)
                 hostname = ans[0].target

src/gevent/resolver/libcares.pxd

 cdef extern from "ares.h":
+    # These two are defined in <sys/socket.h> and <netdb.h>, respectively,
+    # on POSIX. On Windows, they are in <winsock2.h>. "ares.h" winds up
+    # indirectly including both of those.
+    struct sockaddr:
+        pass
+    struct hostent:
+        pass
+
     struct ares_options:
         int flags
         void* sock_state_cb
@@ -36,30 +44,31 @@ cdef extern from "ares.h":
     int ARES_SOCKET_BAD
 
     int ARES_SUCCESS
-    int ARES_ENODATA
-    int ARES_EFORMERR
-    int ARES_ESERVFAIL
-    int ARES_ENOTFOUND
-    int ARES_ENOTIMP
-    int ARES_EREFUSED
-    int ARES_EBADQUERY
-    int ARES_EBADNAME
+    int ARES_EADDRGETNETWORKPARAMS
     int ARES_EBADFAMILY
+    int ARES_EBADFLAGS
+    int ARES_EBADHINTS
+    int ARES_EBADNAME
+    int ARES_EBADQUERY
     int ARES_EBADRESP
+    int ARES_EBADSTR
+    int ARES_ECANCELLED
     int ARES_ECONNREFUSED
-    int ARES_ETIMEOUT
-    int ARES_EOF
+    int ARES_EDESTRUCTION
     int ARES_EFILE
+    int ARES_EFORMERR
+    int ARES_ELOADIPHLPAPI
+    int ARES_ENODATA
     int ARES_ENOMEM
-    int ARES_EDESTRUCTION
-    int ARES_EBADSTR
-    int ARES_EBADFLAGS
     int ARES_ENONAME
-    int ARES_EBADHINTS
+    int ARES_ENOTFOUND
+    int ARES_ENOTIMP
     int ARES_ENOTINITIALIZED
-    int ARES_ELOADIPHLPAPI
-    int ARES_EADDRGETNETWORKPARAMS
-    int ARES_ECANCELLED
+    int ARES_EOF
+    int ARES_EREFUSED
+    int ARES_ESERVFAIL
+    int ARES_ESERVICE
+    int ARES_ETIMEOUT
 
     int ARES_NI_NOFQDN
     int ARES_NI_NUMERICHOST
@@ -74,6 +83,7 @@ cdef extern from "ares.h":
     int ARES_NI_LOOKUPHOST
     int ARES_NI_LOOKUPSERVICE
 
+    ctypedef int ares_socklen_t
 
     int ares_library_init(int flags)
     void ares_library_cleanup()
@@ -87,6 +97,11 @@ cdef extern from "ares.h":
     void ares_cancel(void* channel)
     void ares_getnameinfo(void* channel, void* sa, int salen, int flags, void* callback, void *arg)
 
+    # Added in 1.10
+    int ares_inet_pton(int af, const char *src, void *dst)
+    const char* ares_inet_ntop(int af, const void *src, char *dst, ares_socklen_t size);
+
+
     struct in_addr:
         pass
 
@@ -104,6 +119,45 @@ cdef extern from "ares.h":
 
     int ares_set_servers(void* channel, ares_addr_node *servers)
 
+    # Added in 1.16
+    int ARES_AI_NOSORT
+    int ARES_AI_ENVHOSTS
+    int ARES_AI_CANONNAME
+    int ARES_AI_NUMERICSERV
+
+    struct ares_addrinfo_hints:
+        int ai_flags
+        int ai_family
+        int ai_socktype
+        int ai_protocol
+
+    struct ares_addrinfo_node:
+        int ai_ttl
+        int ai_flags
+        int ai_family
+        int ai_socktype
+        int ai_protocol
+        ares_socklen_t ai_addrlen
+        sockaddr *ai_addr
+        ares_addrinfo_node *ai_next
+
+    struct ares_addrinfo_cname:
+        int ttl
+        char *alias
+        char *name
+        ares_addrinfo_cname *next
+
+    struct ares_addrinfo:
+        ares_addrinfo_cname *cnames
+        ares_addrinfo_node  *nodes
+
+    void ares_getaddrinfo(
+        void* channel,
+        const char *name,
+        const char* service,
+        const ares_addrinfo_hints *hints,
+        #ares_addrinfo_callback callback,
+        void* callback,
+        void *arg)
 
-cdef extern from "cares_pton.h":
-    int ares_inet_pton(int af, char *src, void *dst)
+    void ares_freeaddrinfo(ares_addrinfo *ai)

src/gevent/tests/test__socket_dns6.py

@@ -36,7 +36,7 @@ from gevent.testing.sysinfo import RESOLVER_DNSPYTHON
 # by default we skip the tests everywhere else.
 
 class Test6(TestCase):
-
+    NORMALIZE_GHBA_IGNORE_ALIAS = True
     # host that only has AAAA record
     host = 'aaaa.test-ipv6.com'