Commit 1580ab63 authored by Eric Dumazet's avatar Eric Dumazet Committed by David S. Miller

tcp/dccp: better use of ephemeral ports in connect()

In commit 07f4c900 ("tcp/dccp: try to not exhaust ip_local_port_range
in connect()"), I added a very simple heuristic, so that we got better
chances to use even ports, and allow bind() users to have more available
slots.

It gave nice results, but with more than 200,000 TCP sessions on a typical
server, the ~30,000 ephemeral ports are still a rare resource.

I chose to go a step further, by looking at all even ports, and if none
was available, fallback to odd ports.

The companion patch does the same in bind(), but in opposite way.

I've seen exec times of up to 30ms on busy servers, so I no longer
disable BH for the whole traversal, but only for each hash bucket.
I also call cond_resched() to be gentle to other tasks.
Signed-off-by: default avatarEric Dumazet <edumazet@google.com>
Signed-off-by: default avatarDavid S. Miller <davem@davemloft.net>
parent 3134b9f0
...@@ -565,43 +565,59 @@ int __inet_hash_connect(struct inet_timewait_death_row *death_row, ...@@ -565,43 +565,59 @@ int __inet_hash_connect(struct inet_timewait_death_row *death_row,
struct sock *, __u16, struct inet_timewait_sock **)) struct sock *, __u16, struct inet_timewait_sock **))
{ {
struct inet_hashinfo *hinfo = death_row->hashinfo; struct inet_hashinfo *hinfo = death_row->hashinfo;
const unsigned short snum = inet_sk(sk)->inet_num; struct inet_timewait_sock *tw = NULL;
struct inet_bind_hashbucket *head; struct inet_bind_hashbucket *head;
struct inet_bind_bucket *tb; int port = inet_sk(sk)->inet_num;
int ret;
struct net *net = sock_net(sk); struct net *net = sock_net(sk);
struct inet_bind_bucket *tb;
if (!snum) { u32 remaining, offset;
int i, remaining, low, high, port; int ret, i, low, high;
static u32 hint; static u32 hint;
u32 offset = hint + port_offset;
struct inet_timewait_sock *tw = NULL;
inet_get_local_port_range(net, &low, &high); if (port) {
remaining = (high - low) + 1; head = &hinfo->bhash[inet_bhashfn(net, port,
hinfo->bhash_size)];
tb = inet_csk(sk)->icsk_bind_hash;
spin_lock_bh(&head->lock);
if (sk_head(&tb->owners) == sk && !sk->sk_bind_node.next) {
inet_ehash_nolisten(sk, NULL);
spin_unlock_bh(&head->lock);
return 0;
}
spin_unlock(&head->lock);
/* No definite answer... Walk to established hash table */
ret = check_established(death_row, sk, port, NULL);
local_bh_enable();
return ret;
}
/* By starting with offset being an even number, inet_get_local_port_range(net, &low, &high);
* we tend to leave about 50% of ports for other uses, high++; /* [32768, 60999] -> [32768, 61000[ */
* like bind(0). remaining = high - low;
if (likely(remaining > 1))
remaining &= ~1U;
offset = (hint + port_offset) % remaining;
/* In first pass we try ports of @low parity.
* inet_csk_get_port() does the opposite choice.
*/ */
offset &= ~1; offset &= ~1U;
other_parity_scan:
local_bh_disable(); port = low + offset;
for (i = 0; i < remaining; i++) { for (i = 0; i < remaining; i += 2, port += 2) {
port = low + (i + offset) % remaining; if (unlikely(port >= high))
port -= remaining;
if (inet_is_local_reserved_port(net, port)) if (inet_is_local_reserved_port(net, port))
continue; continue;
head = &hinfo->bhash[inet_bhashfn(net, port, head = &hinfo->bhash[inet_bhashfn(net, port,
hinfo->bhash_size)]; hinfo->bhash_size)];
spin_lock(&head->lock); spin_lock_bh(&head->lock);
/* Does not bother with rcv_saddr checks, /* Does not bother with rcv_saddr checks, because
* because the established check is already * the established check is already unique enough.
* unique enough.
*/ */
inet_bind_bucket_for_each(tb, &head->chain) { inet_bind_bucket_for_each(tb, &head->chain) {
if (net_eq(ib_net(tb), net) && if (net_eq(ib_net(tb), net) && tb->port == port) {
tb->port == port) {
if (tb->fastreuse >= 0 || if (tb->fastreuse >= 0 ||
tb->fastreuseport >= 0) tb->fastreuseport >= 0)
goto next_port; goto next_port;
...@@ -616,22 +632,25 @@ int __inet_hash_connect(struct inet_timewait_death_row *death_row, ...@@ -616,22 +632,25 @@ int __inet_hash_connect(struct inet_timewait_death_row *death_row,
tb = inet_bind_bucket_create(hinfo->bind_bucket_cachep, tb = inet_bind_bucket_create(hinfo->bind_bucket_cachep,
net, head, port); net, head, port);
if (!tb) { if (!tb) {
spin_unlock(&head->lock); spin_unlock_bh(&head->lock);
break; return -ENOMEM;
} }
tb->fastreuse = -1; tb->fastreuse = -1;
tb->fastreuseport = -1; tb->fastreuseport = -1;
goto ok; goto ok;
next_port:
next_port: spin_unlock_bh(&head->lock);
spin_unlock(&head->lock); cond_resched();
} }
local_bh_enable();
offset++;
if ((offset & 1) && remaining > 1)
goto other_parity_scan;
return -EADDRNOTAVAIL; return -EADDRNOTAVAIL;
ok: ok:
hint += (i + 2) & ~1; hint += i + 2;
/* Head lock still held and bh's disabled */ /* Head lock still held and bh's disabled */
inet_bind_hash(sk, tb, port); inet_bind_hash(sk, tb, port);
...@@ -642,29 +661,10 @@ int __inet_hash_connect(struct inet_timewait_death_row *death_row, ...@@ -642,29 +661,10 @@ int __inet_hash_connect(struct inet_timewait_death_row *death_row,
if (tw) if (tw)
inet_twsk_bind_unhash(tw, hinfo); inet_twsk_bind_unhash(tw, hinfo);
spin_unlock(&head->lock); spin_unlock(&head->lock);
if (tw) if (tw)
inet_twsk_deschedule_put(tw); inet_twsk_deschedule_put(tw);
ret = 0;
goto out;
}
head = &hinfo->bhash[inet_bhashfn(net, snum, hinfo->bhash_size)];
tb = inet_csk(sk)->icsk_bind_hash;
spin_lock_bh(&head->lock);
if (sk_head(&tb->owners) == sk && !sk->sk_bind_node.next) {
inet_ehash_nolisten(sk, NULL);
spin_unlock_bh(&head->lock);
return 0;
} else {
spin_unlock(&head->lock);
/* No definite answer... Walk to established hash table */
ret = check_established(death_row, sk, snum, NULL);
out:
local_bh_enable(); local_bh_enable();
return ret; return 0;
}
} }
/* /*
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment