Skip to content

G
gevent
Commit e0da583e authored by Jason Madden's avatar Jason Madden
Browse files
Options

Use ares_getaddrinfo instead of a hand-built method. 

Does not yet handle canonical names, but this doesn't result in test failures.
parent ff3cc4d2
Hide whitespace changes
Inline Side-by-side
Showing with 287 additions and 189 deletions
docs/changes/1012.bugfix
View file @ e0da583e
Use ``ares_getaddrinfo`` instead of a manual lookup.
This requires c-ares 1.16.0.
Note that this may change the results, in particular their order.
As part of this, certain parts of the c-ares extension were adapted to
use modern Cython idioms.
......
src/gevent/resolver/ares.py
View file @ e0da583e
......@@ -4,16 +4,14 @@ c-ares based hostname resolver.
"""
from __future__ import absolute_import, print_function, division
import os
import sys
from _socket import getaddrinfo
from _socket import getaddrinfo as native_getaddrinfo
from _socket import gaierror
from _socket import error
from gevent._compat import string_types
from gevent._compat import text_type
from gevent._compat import reraise
from gevent._compat import integer_types
from gevent._compat import PY3
from gevent.hub import Waiter
......@@ -22,9 +20,10 @@ from gevent.hub import get_hub
from gevent.socket import AF_UNSPEC
from gevent.socket import AF_INET
from gevent.socket import AF_INET6
from gevent.socket import SOCK_STREAM
from gevent.socket import SOCK_DGRAM
from gevent.socket import SOCK_RAW
from gevent.socket import SOCK_STREAM
from gevent.socket import SOL_TCP
from gevent.socket import SOL_UDP
from gevent.socket import AI_NUMERICHOST
from gevent._config import config
......@@ -179,12 +178,25 @@ class Resolver(AbstractResolver):
def _lookup_port(self, port, socktype):
return lookup_port(port, socktype)
def _getaddrinfo(self, host, port, family=0, socktype=0, proto=0, flags=0):
def _getaddrinfo(
self, host, port,
family=0, socktype=0, proto=0, flags=0,
fill_in_type_proto=True
):
"""
Returns a list ``(family, socktype, proto, canonname, sockaddr)``
TODO: Implement canonical names.
:raises gaierror: If no results are found.
"""
# pylint:disable=too-many-locals,too-many-branches
if isinstance(host, text_type):
host = host.encode('idna')
if not isinstance(host, bytes) or (flags & AI_NUMERICHOST) or host in (
b'localhost', b'ip6-localhost'):
# XXX: Now that we're using ares_getaddrinfo, how much of this is still
# necessary?
# this handles cases which do not require network access
# 1) host is None
# 2) host is of an invalid type
......@@ -193,65 +205,68 @@ class Resolver(AbstractResolver):
# really want to do. It's here because it resolves a discrepancy with the system
# resolvers caught by test cases. In gevent 20.4.0, this only worked correctly on
# Python 3 and not Python 2, by accident.
return getaddrinfo(host, port, family, socktype, proto, flags)
# we also call _socket.getaddrinfo below if family is not one of AF_*
port, socktypes = self._lookup_port(port, socktype)
socktype_proto = [(SOCK_STREAM, 6), (SOCK_DGRAM, 17), (SOCK_RAW, 0)]
if socktypes:
socktype_proto = [(x, y) for (x, y) in socktype_proto if x in socktypes]
if proto:
socktype_proto = [(x, y) for (x, y) in socktype_proto if proto == y]
ares = self.cares
if family == AF_UNSPEC:
ares_values = _Values(self.hub, 2)
ares.gethostbyname(ares_values, host, AF_INET)
ares.gethostbyname(ares_values, host, AF_INET6)
elif family == AF_INET:
ares_values = _Values(self.hub, 1)
ares.gethostbyname(ares_values, host, AF_INET)
elif family == AF_INET6:
ares_values = _Values(self.hub, 1)
ares.gethostbyname(ares_values, host, AF_INET6)
else:
raise gaierror(5, 'ai_family not supported: %r' % (family, ))
values = ares_values.get()
if len(values) == 2 and values[0] == values[1]:
values.pop()
result = []
result4 = []
result6 = []
for addrs in values:
if addrs.family == AF_INET:
for addr in addrs[-1]:
sockaddr = (addr, port)
for socktype4, proto4 in socktype_proto:
result4.append((AF_INET, socktype4, proto4, '', sockaddr))
elif addrs.family == AF_INET6:
for addr in addrs[-1]:
if addr == '::1':
dest = result
else:
dest = result6
sockaddr = (addr, port, 0, 0)
for socktype6, proto6 in socktype_proto:
dest.append((AF_INET6, socktype6, proto6, '', sockaddr))
# As of 2016, some platforms return IPV6 first and some do IPV4 first,
# and some might even allow configuration of which is which. For backwards
# compatibility with earlier releases (but not necessarily resolver_thread!)
# we return 4 first. See https://github.com/gevent/gevent/issues/815 for more.
result += result4 + result6
return native_getaddrinfo(host, port, family, socktype, proto, flags)
if isinstance(port, text_type):
port = port.encode('ascii')
elif isinstance(port, integer_types):
if port == 0:
port = None
else:
port = str(port).encode('ascii')
waiter = Waiter(self.hub)
self.cares.getaddrinfo(
waiter,
host,
port,
family,
socktype,
proto,
flags,
)
# Result is a list of:
# (family, socktype, proto, canonname, sockaddr)
# Where sockaddr depends on family; for INET it is
# (address, port)
# and INET6 is
# (address, port, flow info, scope id)
result = waiter.get()
if not result:
raise gaierror(-5, 'No address associated with hostname')
if fill_in_type_proto:
# c-ares 1.16 DOES NOT fill in socktype or proto in the results,
# ever. It's at least supposed to do that if they were given as
# hints, but it doesn't (https://github.com/c-ares/c-ares/issues/317)
# Sigh.
if socktype:
hard_type_proto = [
(socktype, SOL_TCP if socktype == SOCK_STREAM else SOL_UDP),
]
elif proto:
hard_type_proto = [
(SOCK_STREAM if proto == SOL_TCP else SOCK_DGRAM, proto),
]
else:
hard_type_proto = [
(SOCK_STREAM, SOL_TCP),
(SOCK_DGRAM, SOL_UDP),
]
result = [
(rfamily,
hard_type if not rtype else rtype,
hard_proto if not rproto else rproto,
rcanon,
raddr)
for rfamily, rtype, rproto, rcanon, raddr
in result
for hard_type, hard_proto
in hard_type_proto
]
return result
def getaddrinfo(self, host, port, family=0, socktype=0, proto=0, flags=0):
......@@ -319,12 +334,17 @@ class Resolver(AbstractResolver):
if not isinstance(port, int):
raise TypeError('port must be an integer, not %s' % type(port))
waiter = Waiter(self.hub)
result = self._getaddrinfo(address, str(sockaddr[1]), family=AF_UNSPEC, socktype=SOCK_DGRAM)
if not result:
reraise(*sys.exc_info())
elif len(result) != 1:
if len(sockaddr) > 2:
# Must be IPv6: (host, port, [flowinfo, [scopeid]])
flowinfo = sockaddr[2]
if flowinfo > 0xfffff:
raise OverflowError("getnameinfo(): flowinfo must be 0-1048575.")
result = self._getaddrinfo(address, str(sockaddr[1]),
family=AF_UNSPEC, socktype=SOCK_DGRAM, fill_in_type_proto=False)
if len(result) != 1:
raise error('sockaddr resolved to multiple addresses')
family, _socktype, _proto, _name, address = result[0]
if family == AF_INET:
......@@ -333,21 +353,21 @@ class Resolver(AbstractResolver):
elif family == AF_INET6:
address = address[:2] + sockaddr[2:]
waiter = Waiter(self.hub)
self.cares.getnameinfo(waiter, address, flags)
node, service = waiter.get()
if service is None:
if PY3:
# ares docs: "If the query did not complete
# successfully, or one of the values was not
# requested, node or service will be NULL ". Python 2
# allows that for the service, but Python 3 raises
# an error. This is tested by test_socket in py 3.4
err = gaierror('nodename nor servname provided, or not known')
err.errno = 8
raise err
service = '0'
return node, service
if service is None and PY3:
# ares docs: "If the query did not complete
# successfully, or one of the values was not
# requested, node or service will be NULL ". Python 2
# allows that for the service, but Python 3 raises
# an error. This is tested by test_socket in py 3.4
err = gaierror('nodename nor servname provided, or not known')
err.errno = 8
raise err
return node, service or '0'
def getnameinfo(self, sockaddr, flags):
while True:
......@@ -357,34 +377,3 @@ class Resolver(AbstractResolver):
except gaierror:
if ares is self.cares:
raise
class _Values(object):
# helper to collect the results of multiple c-ares calls
# and ignore errors unless nothing has succeeded
# QQQ could probably be moved somewhere - hub.py?
__slots__ = ['count', 'values', 'error', 'waiter']
def __init__(self, hub, count):
self.count = count
self.values = []
self.error = None
self.waiter = Waiter(hub)
def __call__(self, source):
self.count -= 1
if source.exception is None:
self.values.append(source.value)
else:
self.error = source.exception
if self.count <= 0:
self.waiter.switch(None)
def get(self):
self.waiter.get()
if self.values:
return self.values
assert error is not None
raise self.error # pylint:disable=raising-bad-type
src/gevent/resolver/cares.pyx
View file @ e0da583e
......@@ -16,6 +16,7 @@ from cpython.mem cimport PyMem_Malloc
from cpython.mem cimport PyMem_Free
from libc.string cimport memset
import _socket
from _socket import gaierror
......@@ -84,67 +85,50 @@ cdef extern from "ares.h":
unsigned int htons(unsigned int hostshort)
unsigned int ntohs(unsigned int hostshort)
unsigned int htonl(unsigned int hostlong)
unsigned int ntohl(unsigned int hostlong)
cdef int AI_NUMERICSERV = _socket.AI_NUMERICSERV
cdef int AI_CANONNAME = _socket.AI_CANONNAME
cdef int NI_NUMERICHOST = _socket.NI_NUMERICHOST
cdef int NI_NUMERICSERV = _socket.NI_NUMERICSERV
cdef int NI_NOFQDN = _socket.NI_NOFQDN
cdef int NI_NAMEREQD = _socket.NI_NAMEREQD
cdef int NI_DGRAM = _socket.NI_DGRAM
_ares_errors = dict([
(cares.ARES_SUCCESS, 'ARES_SUCCESS'),
(cares.ARES_ENODATA, 'ARES_ENODATA'),
(cares.ARES_EFORMERR, 'ARES_EFORMERR'),
(cares.ARES_ESERVFAIL, 'ARES_ESERVFAIL'),
(cares.ARES_ENOTFOUND, 'ARES_ENOTFOUND'),
(cares.ARES_ENOTIMP, 'ARES_ENOTIMP'),
(cares.ARES_EREFUSED, 'ARES_EREFUSED'),
(cares.ARES_EBADQUERY, 'ARES_EBADQUERY'),
(cares.ARES_EBADNAME, 'ARES_EBADNAME'),
(cares.ARES_EADDRGETNETWORKPARAMS, 'ARES_EADDRGETNETWORKPARAMS'),
(cares.ARES_EBADFAMILY, 'ARES_EBADFAMILY'),
(cares.ARES_EBADFLAGS, 'ARES_EBADFLAGS'),
(cares.ARES_EBADHINTS, 'ARES_EBADHINTS'),
(cares.ARES_EBADNAME, 'ARES_EBADNAME'),
(cares.ARES_EBADQUERY, 'ARES_EBADQUERY'),
(cares.ARES_EBADRESP, 'ARES_EBADRESP'),
(cares.ARES_EBADSTR, 'ARES_EBADSTR'),
(cares.ARES_ECANCELLED, 'ARES_ECANCELLED'),
(cares.ARES_ECONNREFUSED, 'ARES_ECONNREFUSED'),
(cares.ARES_ETIMEOUT, 'ARES_ETIMEOUT'),
(cares.ARES_EOF, 'ARES_EOF'),
(cares.ARES_EDESTRUCTION, 'ARES_EDESTRUCTION'),
(cares.ARES_EFILE, 'ARES_EFILE'),
(cares.ARES_EFORMERR, 'ARES_EFORMERR'),
(cares.ARES_ELOADIPHLPAPI, 'ARES_ELOADIPHLPAPI'),
(cares.ARES_ENODATA, 'ARES_ENODATA'),
(cares.ARES_ENOMEM, 'ARES_ENOMEM'),
(cares.ARES_EDESTRUCTION, 'ARES_EDESTRUCTION'),
(cares.ARES_EBADSTR, 'ARES_EBADSTR'),
(cares.ARES_EBADFLAGS, 'ARES_EBADFLAGS'),
(cares.ARES_ENONAME, 'ARES_ENONAME'),
(cares.ARES_EBADHINTS, 'ARES_EBADHINTS'),
(cares.ARES_ENOTFOUND, 'ARES_ENOTFOUND'),
(cares.ARES_ENOTIMP, 'ARES_ENOTIMP'),
(cares.ARES_ENOTINITIALIZED, 'ARES_ENOTINITIALIZED'),
(cares.ARES_ELOADIPHLPAPI, 'ARES_ELOADIPHLPAPI'),
(cares.ARES_EADDRGETNETWORKPARAMS, 'ARES_EADDRGETNETWORKPARAMS'),
(cares.ARES_ECANCELLED, 'ARES_ECANCELLED')
(cares.ARES_EOF, 'ARES_EOF'),
(cares.ARES_EREFUSED, 'ARES_EREFUSED'),
(cares.ARES_ESERVICE, 'ARES_ESERVICE'),
(cares.ARES_ESERVFAIL, 'ARES_ESERVFAIL'),
(cares.ARES_ETIMEOUT, 'ARES_ETIMEOUT'),
])
# maps c-ares getnameinfo() flag to _socket module flag
cdef list _cares_ni_flag_map = None
cdef _prepare_cares_ni_flag_map():
global _cares_ni_flag_map
import _socket
_cares_ni_flag_map = [
(_socket.NI_NUMERICHOST, cares.ARES_NI_NUMERICHOST),
(_socket.NI_NUMERICSERV, cares.ARES_NI_NUMERICSERV),
(_socket.NI_NOFQDN, cares.ARES_NI_NOFQDN),
(_socket.NI_NAMEREQD, cares.ARES_NI_NAMEREQD),
(_socket.NI_DGRAM, cares.ARES_NI_DGRAM)
]
cdef _convert_cares_ni_flags(int flags,
int default=cares.ARES_NI_LOOKUPHOST|cares.ARES_NI_LOOKUPSERVICE):
if _cares_ni_flag_map is None:
_prepare_cares_ni_flag_map()
cdef int result = default
for socket_flag, cares_flag in _cares_ni_flag_map:
# XXX: This is doing a lot of bouncing back and forth between
# C ints and Python objects, so it's slower than it has to be.
if socket_flag & flags:
result |= cares_flag
return result
cpdef strerror(code):
return '%s: %s' % (_ares_errors.get(code) or code, cares.ares_strerror(code))
......@@ -160,7 +144,7 @@ cdef void gevent_sock_state_callback(void *data, int s, int read, int write):
ch._sock_state_callback(s, read, write)
cdef class result:
cdef class Result(object):
cdef public object value
cdef public object exception
......@@ -225,7 +209,6 @@ cdef list _parse_h_addr_list(hostent* host):
while addr_list[0]:
if not cares.ares_inet_ntop(host.h_addrtype, addr_list[0], tmpbuf, INET6_ADDRSTRLEN):
import _socket
raise _socket.error("Failed in ares_inet_ntop")
result.append(_as_str(tmpbuf))
......@@ -242,7 +225,7 @@ cdef void gevent_ares_host_callback(void *arg, int status, int timeouts, hostent
cdef object host_result
try:
if status or not host:
callback(result(None, gaierror(status, strerror(status))))
callback(Result(None, gaierror(status, strerror(status))))
else:
try:
host_result = ares_host_result(host.h_addrtype,
......@@ -250,9 +233,9 @@ cdef void gevent_ares_host_callback(void *arg, int status, int timeouts, hostent
_parse_h_aliases(host),
_parse_h_addr_list(host)))
except:
callback(result(None, sys.exc_info()[1]))
callback(Result(None, sys.exc_info()[1]))
else:
callback(result(host_result))
callback(Result(host_result))
except:
channel.loop.handle_error(callback, *sys.exc_info())
......@@ -275,11 +258,11 @@ cdef void gevent_ares_nameinfo_callback(void *arg, int status, int timeouts, cha
cdef object service
try:
if status:
callback(result(None, gaierror(status, strerror(status))))
callback(Result(None, gaierror(status, strerror(status))))
else:
node = _as_str(c_node)
service = _as_str(c_service)
callback(result((node, service)))
callback(Result((node, service)))
except:
channel.loop.handle_error(callback, *sys.exc_info())
......@@ -472,7 +455,8 @@ cdef class channel:
# note that for file lookups still AF_INET can be returned for AF_INET6 request
cdef object arg = (self, callback)
Py_INCREF(arg)
cares.ares_gethostbyname(self.channel, name, family, <void*>gevent_ares_host_callback, <void*>arg)
cares.ares_gethostbyname(self.channel, name, family,
<void*>gevent_ares_host_callback, <void*>arg)
def gethostbyaddr(self, object callback, char* addr):
if not self.channel:
......@@ -514,17 +498,130 @@ cdef class channel:
cdef sockaddr_t* x = <sockaddr_t*>&sa6
cares.ares_getnameinfo(self.channel, x, length, flags, <void*>gevent_ares_nameinfo_callback, <void*>arg)
@staticmethod
cdef int _convert_cares_ni_flags(int flags):
cdef int cares_flags = cares.ARES_NI_LOOKUPHOST | cares.ARES_NI_LOOKUPSERVICE
if flags & NI_NUMERICHOST:
cares_flags |= cares.ARES_NI_NUMERICHOST
if flags & NI_NUMERICSERV:
cares_flags |= cares.ARES_NI_NUMERICSERV
if flags & NI_NOFQDN:
cares_flags |= cares.ARES_NI_NOFQDN
if flags & NI_NAMEREQD:
cares_flags |= cares.ARES_NI_NAMEREQD
if flags & NI_DGRAM:
cares_flags |= cares.ARES_NI_DGRAM
return cares_flags
def getnameinfo(self, object callback, tuple sockaddr, int flags):
flags = _convert_cares_ni_flags(flags)
flags = channel._convert_cares_ni_flags(flags)
return self._getnameinfo(callback, sockaddr, flags)
def getaddrinfo(self, object callback, const char* name,
const char* service, # AKA port
@staticmethod
cdef int _convert_cares_ai_flags(int flags):
# c-ares supports a limited set of flags.
# We always want NOSORT, because that implies that
# c-ares will not connect to resolved addresses.
cdef int cares_flags = cares.ARES_AI_NOSORT
if flags & AI_CANONNAME:
cares_flags |= cares.ARES_AI_CANONNAME
if flags & AI_NUMERICSERV:
cares_flags |= cares.ARES_AI_NUMERICSERV
return cares_flags
@staticmethod
cdef void _getaddrinfo_cb(void *arg,
int status,
int timeouts,
cares.ares_addrinfo* result):
cdef cares.ares_addrinfo_node* nodes
cdef sockaddr_in* sadr4
cdef sockaddr_in6* sadr6
cdef channel channel
cdef object callback
# INET6_ADDRSTRLEN is 46, but we can't use that named constant
# here; cython doesn't like it.
cdef char tmpbuf[46]
channel, callback = <tuple>arg
Py_DECREF(<tuple>arg)
# Result is a list of:
# (family, socktype, proto, canonname, sockaddr)
# Where sockaddr depends on family; for INET it is
# (address, port)
# and INET6 is
# (address, port, flow info, scope id)
# TODO: Check the canonnames.
addrs = []
try:
if status != cares.ARES_SUCCESS:
callback(Result(None, gaierror(status, strerror(status))))
return
nodes = result.nodes
while nodes:
if nodes.ai_family == AF_INET:
sadr4 = <sockaddr_in*>nodes.ai_addr
cares.ares_inet_ntop(nodes.ai_family, &sadr4.sin_addr, tmpbuf,
INET6_ADDRSTRLEN)
sockaddr = (
_as_str(tmpbuf),
ntohs(sadr4.sin_port),
)
elif nodes.ai_family == AF_INET6:
sadr6 = <sockaddr_in6*>nodes.ai_addr
cares.ares_inet_ntop(nodes.ai_family, &sadr6.sin6_addr, tmpbuf,
INET6_ADDRSTRLEN)
sockaddr = (
_as_str(tmpbuf),
ntohs(sadr6.sin6_port),
sadr6.sin6_flowinfo,
sadr6.sin6_scope_id,
)
addrs.append((
nodes.ai_family,
nodes.ai_socktype,
nodes.ai_protocol,
'',
sockaddr,
))
nodes = nodes.ai_next
callback(Result(addrs, None))
except:
channel.loop.handle_error(callback, *sys.exc_info())
finally:
if result:
cares.ares_freeaddrinfo(result)
def getaddrinfo(self,
object callback,
const char* name,
object service, # AKA port
int family=0,
int type=0,
int proto=0,
int flags=0):
cdef cares.ares_addrinfo_hints hints;
if not self.channel:
raise gaierror(cares.ARES_EDESTRUCTION, 'this ares channel has been destroyed')
cdef cares.ares_addrinfo_hints hints
memset(&hints, 0, sizeof(cares.ares_addrinfo_hints))
# c-ares supports a limited set of flags.
flags |= cares.ARES_AI_NOSORT # Do not attempt connections to the resolved address
hints.ai_flags = channel._convert_cares_ai_flags(flags)
hints.ai_family = family
hints.ai_socktype = type
hints.ai_protocol = proto
cdef object arg = (self, callback)
Py_INCREF(arg)
cares.ares_getaddrinfo(
self.channel,
name,
NULL if service is None else <char*>service,
&hints,
<void*>channel._getaddrinfo_cb,
<void*>arg
)
src/gevent/resolver/libcares.pxd
View file @ e0da583e
......@@ -44,30 +44,31 @@ cdef extern from "ares.h":
int ARES_SOCKET_BAD
int ARES_SUCCESS
int ARES_ENODATA
int ARES_EFORMERR
int ARES_ESERVFAIL
int ARES_ENOTFOUND
int ARES_ENOTIMP
int ARES_EREFUSED
int ARES_EBADQUERY
int ARES_EBADNAME
int ARES_EADDRGETNETWORKPARAMS
int ARES_EBADFAMILY
int ARES_EBADFLAGS
int ARES_EBADHINTS
int ARES_EBADNAME
int ARES_EBADQUERY
int ARES_EBADRESP
int ARES_EBADSTR
int ARES_ECANCELLED
int ARES_ECONNREFUSED
int ARES_ETIMEOUT
int ARES_EOF
int ARES_EDESTRUCTION
int ARES_EFILE
int ARES_EFORMERR
int ARES_ELOADIPHLPAPI
int ARES_ENODATA
int ARES_ENOMEM
int ARES_EDESTRUCTION
int ARES_EBADSTR
int ARES_EBADFLAGS
int ARES_ENONAME
int ARES_EBADHINTS
int ARES_ENOTFOUND
int ARES_ENOTIMP
int ARES_ENOTINITIALIZED
int ARES_ELOADIPHLPAPI
int ARES_EADDRGETNETWORKPARAMS
int ARES_ECANCELLED
int ARES_EOF
int ARES_EREFUSED
int ARES_ESERVFAIL
int ARES_ESERVICE
int ARES_ETIMEOUT
int ARES_NI_NOFQDN
int ARES_NI_NUMERICHOST
......
src/gevent/tests/test__socket_dns.py
View file @ e0da583e
......@@ -47,10 +47,15 @@ def trace(message, *args, **kwargs):
util.debug(message, *args, **kwargs)
def _run(function, *args):
# Things that the stdlib should never raise and neither should we;
# these indicate bugs in our code and we want to raise them.
REAL_ERRORS = (AttributeError, ValueError, NameError)
try:
result = function(*args)
assert not isinstance(result, BaseException), repr(result)
return result
except REAL_ERRORS:
raise
except Exception as ex:
if TRACE:
traceback.print_exc()
......@@ -230,7 +235,7 @@ def add(klass, hostname, name=None,
@skipWithoutExternalNetwork("Tries to resolve and compare hostnames/addrinfo")
class TestCase(greentest.TestCase):
maxDiff = None
__timeout__ = 30
switch_expected = None
verbose_dns = TRACE
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment
GitLab Nexedi Edition | About GitLab | About Nexedi | 沪ICP备2021021310号-2 | 沪ICP备2021021310号-7