Commit 97bc3633 authored by Shirley Ma's avatar Shirley Ma Committed by David S. Miller

macvtap: macvtapTX zero-copy support

Only 128 bytes is copied, the rest of data is DMA mapped directly from
userspace.
Signed-off-by: default avatarShirley Ma <xma@...ibm.com>
Signed-off-by: default avatarDavid S. Miller <davem@davemloft.net>
parent a6686f2f
......@@ -60,6 +60,7 @@ static struct proto macvtap_proto = {
*/
static dev_t macvtap_major;
#define MACVTAP_NUM_DEVS 65536
#define GOODCOPY_LEN 128
static struct class *macvtap_class;
static struct cdev macvtap_cdev;
......@@ -340,6 +341,7 @@ static int macvtap_open(struct inode *inode, struct file *file)
{
struct net *net = current->nsproxy->net_ns;
struct net_device *dev = dev_get_by_index(net, iminor(inode));
struct macvlan_dev *vlan = netdev_priv(dev);
struct macvtap_queue *q;
int err;
......@@ -369,6 +371,16 @@ static int macvtap_open(struct inode *inode, struct file *file)
q->flags = IFF_VNET_HDR | IFF_NO_PI | IFF_TAP;
q->vnet_hdr_sz = sizeof(struct virtio_net_hdr);
/*
* so far only KVM virtio_net uses macvtap, enable zero copy between
* guest kernel and host kernel when lower device supports zerocopy
*/
if (vlan) {
if ((vlan->lowerdev->features & NETIF_F_HIGHDMA) &&
(vlan->lowerdev->features & NETIF_F_SG))
sock_set_flag(&q->sk, SOCK_ZEROCOPY);
}
err = macvtap_set_queue(dev, file, q);
if (err)
sock_put(&q->sk);
......@@ -433,6 +445,80 @@ static inline struct sk_buff *macvtap_alloc_skb(struct sock *sk, size_t prepad,
return skb;
}
/* set skb frags from iovec, this can move to core network code for reuse */
static int zerocopy_sg_from_iovec(struct sk_buff *skb, const struct iovec *from,
int offset, size_t count)
{
int len = iov_length(from, count) - offset;
int copy = skb_headlen(skb);
int size, offset1 = 0;
int i = 0;
skb_frag_t *f;
/* Skip over from offset */
while (count && (offset >= from->iov_len)) {
offset -= from->iov_len;
++from;
--count;
}
/* copy up to skb headlen */
while (count && (copy > 0)) {
size = min_t(unsigned int, copy, from->iov_len - offset);
if (copy_from_user(skb->data + offset1, from->iov_base + offset,
size))
return -EFAULT;
if (copy > size) {
++from;
--count;
}
copy -= size;
offset1 += size;
offset = 0;
}
if (len == offset1)
return 0;
while (count--) {
struct page *page[MAX_SKB_FRAGS];
int num_pages;
unsigned long base;
len = from->iov_len - offset1;
if (!len) {
offset1 = 0;
++from;
continue;
}
base = (unsigned long)from->iov_base + offset1;
size = ((base & ~PAGE_MASK) + len + ~PAGE_MASK) >> PAGE_SHIFT;
num_pages = get_user_pages_fast(base, size, 0, &page[i]);
if ((num_pages != size) ||
(num_pages > MAX_SKB_FRAGS - skb_shinfo(skb)->nr_frags))
/* put_page is in skb free */
return -EFAULT;
skb->data_len += len;
skb->len += len;
skb->truesize += len;
atomic_add(len, &skb->sk->sk_wmem_alloc);
while (len) {
f = &skb_shinfo(skb)->frags[i];
f->page = page[i];
f->page_offset = base & ~PAGE_MASK;
f->size = min_t(int, len, PAGE_SIZE - f->page_offset);
skb_shinfo(skb)->nr_frags++;
/* increase sk_wmem_alloc */
base += f->size;
len -= f->size;
i++;
}
offset1 = 0;
++from;
}
return 0;
}
/*
* macvtap_skb_from_vnet_hdr and macvtap_skb_to_vnet_hdr should
* be shared with the tun/tap driver.
......@@ -517,16 +603,18 @@ static int macvtap_skb_to_vnet_hdr(const struct sk_buff *skb,
/* Get packet from user space buffer */
static ssize_t macvtap_get_user(struct macvtap_queue *q,
const struct iovec *iv, size_t count,
int noblock)
static ssize_t macvtap_get_user(struct macvtap_queue *q, struct msghdr *m,
const struct iovec *iv, unsigned long total_len,
size_t count, int noblock)
{
struct sk_buff *skb;
struct macvlan_dev *vlan;
size_t len = count;
unsigned long len = total_len;
int err;
struct virtio_net_hdr vnet_hdr = { 0 };
int vnet_hdr_len = 0;
int copylen;
bool zerocopy = false;
if (q->flags & IFF_VNET_HDR) {
vnet_hdr_len = q->vnet_hdr_sz;
......@@ -554,12 +642,31 @@ static ssize_t macvtap_get_user(struct macvtap_queue *q,
if (unlikely(len < ETH_HLEN))
goto err;
skb = macvtap_alloc_skb(&q->sk, NET_IP_ALIGN, len, vnet_hdr.hdr_len,
noblock, &err);
if (m && m->msg_control && sock_flag(&q->sk, SOCK_ZEROCOPY))
zerocopy = true;
if (zerocopy) {
/* There are 256 bytes to be copied in skb, so there is enough
* room for skb expand head in case it is used.
* The rest buffer is mapped from userspace.
*/
copylen = vnet_hdr.hdr_len;
if (!copylen)
copylen = GOODCOPY_LEN;
} else
copylen = len;
skb = macvtap_alloc_skb(&q->sk, NET_IP_ALIGN, copylen,
vnet_hdr.hdr_len, noblock, &err);
if (!skb)
goto err;
err = skb_copy_datagram_from_iovec(skb, 0, iv, vnet_hdr_len, len);
if (zerocopy) {
err = zerocopy_sg_from_iovec(skb, iv, vnet_hdr_len, count);
skb_shinfo(skb)->tx_flags |= SKBTX_DEV_ZEROCOPY;
} else
err = skb_copy_datagram_from_iovec(skb, 0, iv, vnet_hdr_len,
len);
if (err)
goto err_kfree;
......@@ -575,13 +682,16 @@ static ssize_t macvtap_get_user(struct macvtap_queue *q,
rcu_read_lock_bh();
vlan = rcu_dereference_bh(q->vlan);
/* copy skb_ubuf_info for callback when skb has no error */
if (zerocopy)
skb_shinfo(skb)->destructor_arg = m->msg_control;
if (vlan)
macvlan_start_xmit(skb, vlan->dev);
else
kfree_skb(skb);
rcu_read_unlock_bh();
return count;
return total_len;
err_kfree:
kfree_skb(skb);
......@@ -603,7 +713,7 @@ static ssize_t macvtap_aio_write(struct kiocb *iocb, const struct iovec *iv,
ssize_t result = -ENOLINK;
struct macvtap_queue *q = file->private_data;
result = macvtap_get_user(q, iv, iov_length(iv, count),
result = macvtap_get_user(q, NULL, iv, iov_length(iv, count), count,
file->f_flags & O_NONBLOCK);
return result;
}
......@@ -817,7 +927,7 @@ static int macvtap_sendmsg(struct kiocb *iocb, struct socket *sock,
struct msghdr *m, size_t total_len)
{
struct macvtap_queue *q = container_of(sock, struct macvtap_queue, sock);
return macvtap_get_user(q, m->msg_iov, total_len,
return macvtap_get_user(q, m, m->msg_iov, total_len, m->msg_iovlen,
m->msg_flags & MSG_DONTWAIT);
}
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment