Commit 1f4c6eb2 authored by Eric Dumazet's avatar Eric Dumazet Committed by David S. Miller

ipv6: factorize sk_wmem_alloc updates done by __ip6_append_data()

While testing my inet defrag changes, I found that the senders
could spend ~20% of cpu cycles in skb_set_owner_w() updating
sk->sk_wmem_alloc for every fragment they cook, competing
with TX completion of prior skbs possibly happening on another cpus.

The solution to this problem is to use alloc_skb() instead
of sock_wmalloc() and manually perform a single sk_wmem_alloc change.
Signed-off-by: default avatarEric Dumazet <edumazet@google.com>
Signed-off-by: default avatarDavid S. Miller <davem@davemloft.net>
parent 694aba69
...@@ -1259,6 +1259,7 @@ static int __ip6_append_data(struct sock *sk, ...@@ -1259,6 +1259,7 @@ static int __ip6_append_data(struct sock *sk,
struct ipv6_txoptions *opt = v6_cork->opt; struct ipv6_txoptions *opt = v6_cork->opt;
int csummode = CHECKSUM_NONE; int csummode = CHECKSUM_NONE;
unsigned int maxnonfragsize, headersize; unsigned int maxnonfragsize, headersize;
unsigned int wmem_alloc_delta = 0;
skb = skb_peek_tail(queue); skb = skb_peek_tail(queue);
if (!skb) { if (!skb) {
...@@ -1411,10 +1412,9 @@ static int __ip6_append_data(struct sock *sk, ...@@ -1411,10 +1412,9 @@ static int __ip6_append_data(struct sock *sk,
(flags & MSG_DONTWAIT), &err); (flags & MSG_DONTWAIT), &err);
} else { } else {
skb = NULL; skb = NULL;
if (refcount_read(&sk->sk_wmem_alloc) <= if (refcount_read(&sk->sk_wmem_alloc) + wmem_alloc_delta <=
2 * sk->sk_sndbuf) 2 * sk->sk_sndbuf)
skb = sock_wmalloc(sk, skb = alloc_skb(alloclen + hh_len,
alloclen + hh_len, 1,
sk->sk_allocation); sk->sk_allocation);
if (unlikely(!skb)) if (unlikely(!skb))
err = -ENOBUFS; err = -ENOBUFS;
...@@ -1474,6 +1474,11 @@ static int __ip6_append_data(struct sock *sk, ...@@ -1474,6 +1474,11 @@ static int __ip6_append_data(struct sock *sk,
/* /*
* Put the packet on the pending queue * Put the packet on the pending queue
*/ */
if (!skb->destructor) {
skb->destructor = sock_wfree;
skb->sk = sk;
wmem_alloc_delta += skb->truesize;
}
__skb_queue_tail(queue, skb); __skb_queue_tail(queue, skb);
continue; continue;
} }
...@@ -1520,12 +1525,13 @@ static int __ip6_append_data(struct sock *sk, ...@@ -1520,12 +1525,13 @@ static int __ip6_append_data(struct sock *sk,
skb->len += copy; skb->len += copy;
skb->data_len += copy; skb->data_len += copy;
skb->truesize += copy; skb->truesize += copy;
refcount_add(copy, &sk->sk_wmem_alloc); wmem_alloc_delta += copy;
} }
offset += copy; offset += copy;
length -= copy; length -= copy;
} }
refcount_add(wmem_alloc_delta, &sk->sk_wmem_alloc);
return 0; return 0;
error_efault: error_efault:
...@@ -1533,6 +1539,7 @@ static int __ip6_append_data(struct sock *sk, ...@@ -1533,6 +1539,7 @@ static int __ip6_append_data(struct sock *sk,
error: error:
cork->length -= length; cork->length -= length;
IP6_INC_STATS(sock_net(sk), rt->rt6i_idev, IPSTATS_MIB_OUTDISCARDS); IP6_INC_STATS(sock_net(sk), rt->rt6i_idev, IPSTATS_MIB_OUTDISCARDS);
refcount_add(wmem_alloc_delta, &sk->sk_wmem_alloc);
return err; return err;
} }
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment