Commit 1d2077ac authored by Eric Dumazet's avatar Eric Dumazet Committed by David S. Miller

net: add __sock_wfree() helper

Hosts sending lot of ACK packets exhibit high sock_wfree() cost
because of cache line miss to test SOCK_USE_WRITE_QUEUE

We could move this flag close to sk_wmem_alloc but it is better
to perform the atomic_sub_and_test() on a clean cache line,
as it avoid one extra bus transaction.

skb_orphan_partial() can also have a fast track for packets that either
are TCP acks, or already went through another skb_orphan_partial()
Signed-off-by: default avatarEric Dumazet <edumazet@google.com>
Signed-off-by: default avatarDavid S. Miller <davem@davemloft.net>
parent e34b1638
...@@ -1445,6 +1445,7 @@ struct sock *sk_clone_lock(const struct sock *sk, const gfp_t priority); ...@@ -1445,6 +1445,7 @@ struct sock *sk_clone_lock(const struct sock *sk, const gfp_t priority);
struct sk_buff *sock_wmalloc(struct sock *sk, unsigned long size, int force, struct sk_buff *sock_wmalloc(struct sock *sk, unsigned long size, int force,
gfp_t priority); gfp_t priority);
void __sock_wfree(struct sk_buff *skb);
void sock_wfree(struct sk_buff *skb); void sock_wfree(struct sk_buff *skb);
void skb_orphan_partial(struct sk_buff *skb); void skb_orphan_partial(struct sk_buff *skb);
void sock_rfree(struct sk_buff *skb); void sock_rfree(struct sk_buff *skb);
......
...@@ -1655,6 +1655,17 @@ void sock_wfree(struct sk_buff *skb) ...@@ -1655,6 +1655,17 @@ void sock_wfree(struct sk_buff *skb)
} }
EXPORT_SYMBOL(sock_wfree); EXPORT_SYMBOL(sock_wfree);
/* This variant of sock_wfree() is used by TCP,
* since it sets SOCK_USE_WRITE_QUEUE.
*/
void __sock_wfree(struct sk_buff *skb)
{
struct sock *sk = skb->sk;
if (atomic_sub_and_test(skb->truesize, &sk->sk_wmem_alloc))
__sk_free(sk);
}
void skb_set_owner_w(struct sk_buff *skb, struct sock *sk) void skb_set_owner_w(struct sk_buff *skb, struct sock *sk)
{ {
skb_orphan(skb); skb_orphan(skb);
...@@ -1677,8 +1688,21 @@ void skb_set_owner_w(struct sk_buff *skb, struct sock *sk) ...@@ -1677,8 +1688,21 @@ void skb_set_owner_w(struct sk_buff *skb, struct sock *sk)
} }
EXPORT_SYMBOL(skb_set_owner_w); EXPORT_SYMBOL(skb_set_owner_w);
/* This helper is used by netem, as it can hold packets in its
* delay queue. We want to allow the owner socket to send more
* packets, as if they were already TX completed by a typical driver.
* But we also want to keep skb->sk set because some packet schedulers
* rely on it (sch_fq for example). So we set skb->truesize to a small
* amount (1) and decrease sk_wmem_alloc accordingly.
*/
void skb_orphan_partial(struct sk_buff *skb) void skb_orphan_partial(struct sk_buff *skb)
{ {
/* If this skb is a TCP pure ACK or already went here,
* we have nothing to do. 2 is already a very small truesize.
*/
if (skb->truesize <= 2)
return;
/* TCP stack sets skb->ooo_okay based on sk_wmem_alloc, /* TCP stack sets skb->ooo_okay based on sk_wmem_alloc,
* so we do not completely orphan skb, but transfert all * so we do not completely orphan skb, but transfert all
* accounted bytes but one, to avoid unexpected reorders. * accounted bytes but one, to avoid unexpected reorders.
......
...@@ -949,7 +949,7 @@ static int tcp_transmit_skb(struct sock *sk, struct sk_buff *skb, int clone_it, ...@@ -949,7 +949,7 @@ static int tcp_transmit_skb(struct sock *sk, struct sk_buff *skb, int clone_it,
skb_orphan(skb); skb_orphan(skb);
skb->sk = sk; skb->sk = sk;
skb->destructor = skb_is_tcp_pure_ack(skb) ? sock_wfree : tcp_wfree; skb->destructor = skb_is_tcp_pure_ack(skb) ? __sock_wfree : tcp_wfree;
skb_set_hash_from_sk(skb, sk); skb_set_hash_from_sk(skb, sk);
atomic_add(skb->truesize, &sk->sk_wmem_alloc); atomic_add(skb->truesize, &sk->sk_wmem_alloc);
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment