Commit 903ab86d authored by Herbert Xu's avatar Herbert Xu Committed by David S. Miller

udp: Add lockless transmit path

The UDP transmit path has been running under the socket lock
for a long time because of the corking feature.  This means that
transmitting to the same socket in multiple threads does not
scale at all.

However, as most users don't actually use corking, the locking
can be removed in the common case.

This patch creates a lockless fast path where corking is not used.

Please note that this does create a slight inaccuracy in the
enforcement of socket send buffer limits.  In particular, we
may exceed the socket limit by up to (number of CPUs) * (packet
size) because of the way the limit is computed.

As the primary purpose of socket buffers is to indicate congestion,
this should not be a great problem for now.
Signed-off-by: default avatarHerbert Xu <herbert@gondor.apana.org.au>
Acked-by: default avatarEric Dumazet <eric.dumazet@gmail.com>
Signed-off-by: default avatarDavid S. Miller <davem@davemloft.net>
parent f6b9664f
...@@ -802,6 +802,7 @@ int udp_sendmsg(struct kiocb *iocb, struct sock *sk, struct msghdr *msg, ...@@ -802,6 +802,7 @@ int udp_sendmsg(struct kiocb *iocb, struct sock *sk, struct msghdr *msg,
int err, is_udplite = IS_UDPLITE(sk); int err, is_udplite = IS_UDPLITE(sk);
int corkreq = up->corkflag || msg->msg_flags&MSG_MORE; int corkreq = up->corkflag || msg->msg_flags&MSG_MORE;
int (*getfrag)(void *, char *, int, int, int, struct sk_buff *); int (*getfrag)(void *, char *, int, int, int, struct sk_buff *);
struct sk_buff *skb;
if (len > 0xFFFF) if (len > 0xFFFF)
return -EMSGSIZE; return -EMSGSIZE;
...@@ -816,6 +817,8 @@ int udp_sendmsg(struct kiocb *iocb, struct sock *sk, struct msghdr *msg, ...@@ -816,6 +817,8 @@ int udp_sendmsg(struct kiocb *iocb, struct sock *sk, struct msghdr *msg,
ipc.opt = NULL; ipc.opt = NULL;
ipc.tx_flags = 0; ipc.tx_flags = 0;
getfrag = is_udplite ? udplite_getfrag : ip_generic_getfrag;
if (up->pending) { if (up->pending) {
/* /*
* There are pending frames. * There are pending frames.
...@@ -940,6 +943,17 @@ int udp_sendmsg(struct kiocb *iocb, struct sock *sk, struct msghdr *msg, ...@@ -940,6 +943,17 @@ int udp_sendmsg(struct kiocb *iocb, struct sock *sk, struct msghdr *msg,
if (!ipc.addr) if (!ipc.addr)
daddr = ipc.addr = rt->rt_dst; daddr = ipc.addr = rt->rt_dst;
/* Lockless fast path for the non-corking case. */
if (!corkreq) {
skb = ip_make_skb(sk, getfrag, msg->msg_iov, ulen,
sizeof(struct udphdr), &ipc, &rt,
msg->msg_flags);
err = PTR_ERR(skb);
if (skb && !IS_ERR(skb))
err = udp_send_skb(skb, daddr, dport);
goto out;
}
lock_sock(sk); lock_sock(sk);
if (unlikely(up->pending)) { if (unlikely(up->pending)) {
/* The socket is already corked while preparing it. */ /* The socket is already corked while preparing it. */
...@@ -961,7 +975,6 @@ int udp_sendmsg(struct kiocb *iocb, struct sock *sk, struct msghdr *msg, ...@@ -961,7 +975,6 @@ int udp_sendmsg(struct kiocb *iocb, struct sock *sk, struct msghdr *msg,
do_append_data: do_append_data:
up->len += ulen; up->len += ulen;
getfrag = is_udplite ? udplite_getfrag : ip_generic_getfrag;
err = ip_append_data(sk, getfrag, msg->msg_iov, ulen, err = ip_append_data(sk, getfrag, msg->msg_iov, ulen,
sizeof(struct udphdr), &ipc, &rt, sizeof(struct udphdr), &ipc, &rt,
corkreq ? msg->msg_flags|MSG_MORE : msg->msg_flags); corkreq ? msg->msg_flags|MSG_MORE : msg->msg_flags);
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment