Commit 2e77d89b authored by Eric Dumazet's avatar Eric Dumazet Committed by David S. Miller

net: avoid a pair of dst_hold()/dst_release() in ip_append_data()

We can reduce pressure on dst entry refcount that slowdown UDP transmit
path on SMP machines. This pressure is visible on RTP servers when
delivering content to mediagateways, especially big ones, handling
thousand of streams. Several cpus send UDP frames to the same
destination, hence use the same dst entry.

This patch makes ip_append_data() eventually steal the refcount its
callers had to take on the dst entry.

This doesnt avoid all refcounting, but still gives speedups on SMP,
on UDP/RAW transmit path
Signed-off-by: default avatarEric Dumazet <dada1@cosmosbay.com>
Signed-off-by: default avatarDavid S. Miller <davem@davemloft.net>
parent 4db0acf3
...@@ -110,7 +110,7 @@ extern int ip_append_data(struct sock *sk, ...@@ -110,7 +110,7 @@ extern int ip_append_data(struct sock *sk,
int odd, struct sk_buff *skb), int odd, struct sk_buff *skb),
void *from, int len, int protolen, void *from, int len, int protolen,
struct ipcm_cookie *ipc, struct ipcm_cookie *ipc,
struct rtable *rt, struct rtable **rt,
unsigned int flags); unsigned int flags);
extern int ip_generic_getfrag(void *from, char *to, int offset, int len, int odd, struct sk_buff *skb); extern int ip_generic_getfrag(void *from, char *to, int offset, int len, int odd, struct sk_buff *skb);
extern ssize_t ip_append_page(struct sock *sk, struct page *page, extern ssize_t ip_append_page(struct sock *sk, struct page *page,
......
...@@ -321,12 +321,12 @@ static int icmp_glue_bits(void *from, char *to, int offset, int len, int odd, ...@@ -321,12 +321,12 @@ static int icmp_glue_bits(void *from, char *to, int offset, int len, int odd,
} }
static void icmp_push_reply(struct icmp_bxm *icmp_param, static void icmp_push_reply(struct icmp_bxm *icmp_param,
struct ipcm_cookie *ipc, struct rtable *rt) struct ipcm_cookie *ipc, struct rtable **rt)
{ {
struct sock *sk; struct sock *sk;
struct sk_buff *skb; struct sk_buff *skb;
sk = icmp_sk(dev_net(rt->u.dst.dev)); sk = icmp_sk(dev_net((*rt)->u.dst.dev));
if (ip_append_data(sk, icmp_glue_bits, icmp_param, if (ip_append_data(sk, icmp_glue_bits, icmp_param,
icmp_param->data_len+icmp_param->head_len, icmp_param->data_len+icmp_param->head_len,
icmp_param->head_len, icmp_param->head_len,
...@@ -392,7 +392,7 @@ static void icmp_reply(struct icmp_bxm *icmp_param, struct sk_buff *skb) ...@@ -392,7 +392,7 @@ static void icmp_reply(struct icmp_bxm *icmp_param, struct sk_buff *skb)
} }
if (icmpv4_xrlim_allow(net, rt, icmp_param->data.icmph.type, if (icmpv4_xrlim_allow(net, rt, icmp_param->data.icmph.type,
icmp_param->data.icmph.code)) icmp_param->data.icmph.code))
icmp_push_reply(icmp_param, &ipc, rt); icmp_push_reply(icmp_param, &ipc, &rt);
ip_rt_put(rt); ip_rt_put(rt);
out_unlock: out_unlock:
icmp_xmit_unlock(sk); icmp_xmit_unlock(sk);
...@@ -635,7 +635,7 @@ void icmp_send(struct sk_buff *skb_in, int type, int code, __be32 info) ...@@ -635,7 +635,7 @@ void icmp_send(struct sk_buff *skb_in, int type, int code, __be32 info)
icmp_param.data_len = room; icmp_param.data_len = room;
icmp_param.head_len = sizeof(struct icmphdr); icmp_param.head_len = sizeof(struct icmphdr);
icmp_push_reply(&icmp_param, &ipc, rt); icmp_push_reply(&icmp_param, &ipc, &rt);
ende: ende:
ip_rt_put(rt); ip_rt_put(rt);
out_unlock: out_unlock:
......
...@@ -778,7 +778,7 @@ int ip_append_data(struct sock *sk, ...@@ -778,7 +778,7 @@ int ip_append_data(struct sock *sk,
int getfrag(void *from, char *to, int offset, int len, int getfrag(void *from, char *to, int offset, int len,
int odd, struct sk_buff *skb), int odd, struct sk_buff *skb),
void *from, int length, int transhdrlen, void *from, int length, int transhdrlen,
struct ipcm_cookie *ipc, struct rtable *rt, struct ipcm_cookie *ipc, struct rtable **rtp,
unsigned int flags) unsigned int flags)
{ {
struct inet_sock *inet = inet_sk(sk); struct inet_sock *inet = inet_sk(sk);
...@@ -793,6 +793,7 @@ int ip_append_data(struct sock *sk, ...@@ -793,6 +793,7 @@ int ip_append_data(struct sock *sk,
int offset = 0; int offset = 0;
unsigned int maxfraglen, fragheaderlen; unsigned int maxfraglen, fragheaderlen;
int csummode = CHECKSUM_NONE; int csummode = CHECKSUM_NONE;
struct rtable *rt;
if (flags&MSG_PROBE) if (flags&MSG_PROBE)
return 0; return 0;
...@@ -812,7 +813,11 @@ int ip_append_data(struct sock *sk, ...@@ -812,7 +813,11 @@ int ip_append_data(struct sock *sk,
inet->cork.flags |= IPCORK_OPT; inet->cork.flags |= IPCORK_OPT;
inet->cork.addr = ipc->addr; inet->cork.addr = ipc->addr;
} }
dst_hold(&rt->u.dst); rt = *rtp;
/*
* We steal reference to this route, caller should not release it
*/
*rtp = NULL;
inet->cork.fragsize = mtu = inet->pmtudisc == IP_PMTUDISC_PROBE ? inet->cork.fragsize = mtu = inet->pmtudisc == IP_PMTUDISC_PROBE ?
rt->u.dst.dev->mtu : rt->u.dst.dev->mtu :
dst_mtu(rt->u.dst.path); dst_mtu(rt->u.dst.path);
...@@ -1391,7 +1396,7 @@ void ip_send_reply(struct sock *sk, struct sk_buff *skb, struct ip_reply_arg *ar ...@@ -1391,7 +1396,7 @@ void ip_send_reply(struct sock *sk, struct sk_buff *skb, struct ip_reply_arg *ar
sk->sk_protocol = ip_hdr(skb)->protocol; sk->sk_protocol = ip_hdr(skb)->protocol;
sk->sk_bound_dev_if = arg->bound_dev_if; sk->sk_bound_dev_if = arg->bound_dev_if;
ip_append_data(sk, ip_reply_glue_bits, arg->iov->iov_base, len, 0, ip_append_data(sk, ip_reply_glue_bits, arg->iov->iov_base, len, 0,
&ipc, rt, MSG_DONTWAIT); &ipc, &rt, MSG_DONTWAIT);
if ((skb = skb_peek(&sk->sk_write_queue)) != NULL) { if ((skb = skb_peek(&sk->sk_write_queue)) != NULL) {
if (arg->csumoffset >= 0) if (arg->csumoffset >= 0)
*((__sum16 *)skb_transport_header(skb) + *((__sum16 *)skb_transport_header(skb) +
......
...@@ -572,7 +572,7 @@ static int raw_sendmsg(struct kiocb *iocb, struct sock *sk, struct msghdr *msg, ...@@ -572,7 +572,7 @@ static int raw_sendmsg(struct kiocb *iocb, struct sock *sk, struct msghdr *msg,
ipc.addr = rt->rt_dst; ipc.addr = rt->rt_dst;
lock_sock(sk); lock_sock(sk);
err = ip_append_data(sk, ip_generic_getfrag, msg->msg_iov, len, 0, err = ip_append_data(sk, ip_generic_getfrag, msg->msg_iov, len, 0,
&ipc, rt, msg->msg_flags); &ipc, &rt, msg->msg_flags);
if (err) if (err)
ip_flush_pending_frames(sk); ip_flush_pending_frames(sk);
else if (!(msg->msg_flags & MSG_MORE)) else if (!(msg->msg_flags & MSG_MORE))
......
...@@ -719,7 +719,7 @@ int udp_sendmsg(struct kiocb *iocb, struct sock *sk, struct msghdr *msg, ...@@ -719,7 +719,7 @@ int udp_sendmsg(struct kiocb *iocb, struct sock *sk, struct msghdr *msg,
up->len += ulen; up->len += ulen;
getfrag = is_udplite ? udplite_getfrag : ip_generic_getfrag; getfrag = is_udplite ? udplite_getfrag : ip_generic_getfrag;
err = ip_append_data(sk, getfrag, msg->msg_iov, ulen, err = ip_append_data(sk, getfrag, msg->msg_iov, ulen,
sizeof(struct udphdr), &ipc, rt, sizeof(struct udphdr), &ipc, &rt,
corkreq ? msg->msg_flags|MSG_MORE : msg->msg_flags); corkreq ? msg->msg_flags|MSG_MORE : msg->msg_flags);
if (err) if (err)
udp_flush_pending_frames(sk); udp_flush_pending_frames(sk);
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment