Commit fb32856b authored by Xuan Zhuo's avatar Xuan Zhuo Committed by David S. Miller

virtio-net: page_to_skb() use build_skb when there's sufficient tailroom

In page_to_skb(), if we have enough tailroom to save skb_shared_info, we
can use build_skb to create skb directly. No need to alloc for
additional space. And it can save a 'frags slot', which is very friendly
to GRO.

Here, if the payload of the received package is too small (less than
GOOD_COPY_LEN), we still choose to copy it directly to the space got by
napi_alloc_skb. So we can reuse these pages.

Testing Machine:
    The four queues of the network card are bound to the cpu1.

Test command:
    for ((i=0;i<5;++i)); do sockperf tp --ip 192.168.122.64 -m 1000 -t 150& done

The size of the udp package is 1000, so in the case of this patch, there
will always be enough tailroom to use build_skb. The sent udp packet
will be discarded because there is no port to receive it. The irqsoftd
of the machine is 100%, we observe the received quantity displayed by
sar -n DEV 1:

no build_skb:  956864.00 rxpck/s
build_skb:    1158465.00 rxpck/s
Signed-off-by: default avatarXuan Zhuo <xuanzhuo@linux.alibaba.com>
Suggested-by: default avatarJason Wang <jasowang@redhat.com>
Signed-off-by: default avatarDavid S. Miller <davem@davemloft.net>
parent fa588eba
...@@ -379,21 +379,17 @@ static struct sk_buff *page_to_skb(struct virtnet_info *vi, ...@@ -379,21 +379,17 @@ static struct sk_buff *page_to_skb(struct virtnet_info *vi,
struct receive_queue *rq, struct receive_queue *rq,
struct page *page, unsigned int offset, struct page *page, unsigned int offset,
unsigned int len, unsigned int truesize, unsigned int len, unsigned int truesize,
bool hdr_valid, unsigned int metasize) bool hdr_valid, unsigned int metasize,
unsigned int headroom)
{ {
struct sk_buff *skb; struct sk_buff *skb;
struct virtio_net_hdr_mrg_rxbuf *hdr; struct virtio_net_hdr_mrg_rxbuf *hdr;
unsigned int copy, hdr_len, hdr_padded_len; unsigned int copy, hdr_len, hdr_padded_len;
char *p; int tailroom, shinfo_size;
char *p, *hdr_p;
p = page_address(page) + offset; p = page_address(page) + offset;
hdr_p = p;
/* copy small packet so we can reuse these pages for small data */
skb = napi_alloc_skb(&rq->napi, GOOD_COPY_LEN);
if (unlikely(!skb))
return NULL;
hdr = skb_vnet_hdr(skb);
hdr_len = vi->hdr_len; hdr_len = vi->hdr_len;
if (vi->mergeable_rx_bufs) if (vi->mergeable_rx_bufs)
...@@ -401,14 +397,38 @@ static struct sk_buff *page_to_skb(struct virtnet_info *vi, ...@@ -401,14 +397,38 @@ static struct sk_buff *page_to_skb(struct virtnet_info *vi,
else else
hdr_padded_len = sizeof(struct padded_vnet_hdr); hdr_padded_len = sizeof(struct padded_vnet_hdr);
/* hdr_valid means no XDP, so we can copy the vnet header */ /* If headroom is not 0, there is an offset between the beginning of the
if (hdr_valid) * data and the allocated space, otherwise the data and the allocated
memcpy(hdr, p, hdr_len); * space are aligned.
*/
if (headroom) {
/* The actual allocated space size is PAGE_SIZE. */
truesize = PAGE_SIZE;
tailroom = truesize - len - offset;
} else {
tailroom = truesize - len;
}
len -= hdr_len; len -= hdr_len;
offset += hdr_padded_len; offset += hdr_padded_len;
p += hdr_padded_len; p += hdr_padded_len;
shinfo_size = SKB_DATA_ALIGN(sizeof(struct skb_shared_info));
if (len > GOOD_COPY_LEN && tailroom >= shinfo_size) {
skb = build_skb(p, truesize);
if (unlikely(!skb))
return NULL;
skb_put(skb, len);
goto ok;
}
/* copy small packet so we can reuse these pages for small data */
skb = napi_alloc_skb(&rq->napi, GOOD_COPY_LEN);
if (unlikely(!skb))
return NULL;
/* Copy all frame if it fits skb->head, otherwise /* Copy all frame if it fits skb->head, otherwise
* we let virtio_net_hdr_to_skb() and GRO pull headers as needed. * we let virtio_net_hdr_to_skb() and GRO pull headers as needed.
*/ */
...@@ -418,11 +438,6 @@ static struct sk_buff *page_to_skb(struct virtnet_info *vi, ...@@ -418,11 +438,6 @@ static struct sk_buff *page_to_skb(struct virtnet_info *vi,
copy = ETH_HLEN + metasize; copy = ETH_HLEN + metasize;
skb_put_data(skb, p, copy); skb_put_data(skb, p, copy);
if (metasize) {
__skb_pull(skb, metasize);
skb_metadata_set(skb, metasize);
}
len -= copy; len -= copy;
offset += copy; offset += copy;
...@@ -431,7 +446,7 @@ static struct sk_buff *page_to_skb(struct virtnet_info *vi, ...@@ -431,7 +446,7 @@ static struct sk_buff *page_to_skb(struct virtnet_info *vi,
skb_add_rx_frag(skb, 0, page, offset, len, truesize); skb_add_rx_frag(skb, 0, page, offset, len, truesize);
else else
put_page(page); put_page(page);
return skb; goto ok;
} }
/* /*
...@@ -458,6 +473,18 @@ static struct sk_buff *page_to_skb(struct virtnet_info *vi, ...@@ -458,6 +473,18 @@ static struct sk_buff *page_to_skb(struct virtnet_info *vi,
if (page) if (page)
give_pages(rq, page); give_pages(rq, page);
ok:
/* hdr_valid means no XDP, so we can copy the vnet header */
if (hdr_valid) {
hdr = skb_vnet_hdr(skb);
memcpy(hdr, hdr_p, hdr_len);
}
if (metasize) {
__skb_pull(skb, metasize);
skb_metadata_set(skb, metasize);
}
return skb; return skb;
} }
...@@ -808,7 +835,7 @@ static struct sk_buff *receive_big(struct net_device *dev, ...@@ -808,7 +835,7 @@ static struct sk_buff *receive_big(struct net_device *dev,
{ {
struct page *page = buf; struct page *page = buf;
struct sk_buff *skb = struct sk_buff *skb =
page_to_skb(vi, rq, page, 0, len, PAGE_SIZE, true, 0); page_to_skb(vi, rq, page, 0, len, PAGE_SIZE, true, 0, 0);
stats->bytes += len - vi->hdr_len; stats->bytes += len - vi->hdr_len;
if (unlikely(!skb)) if (unlikely(!skb))
...@@ -922,7 +949,7 @@ static struct sk_buff *receive_mergeable(struct net_device *dev, ...@@ -922,7 +949,7 @@ static struct sk_buff *receive_mergeable(struct net_device *dev,
put_page(page); put_page(page);
head_skb = page_to_skb(vi, rq, xdp_page, offset, head_skb = page_to_skb(vi, rq, xdp_page, offset,
len, PAGE_SIZE, false, len, PAGE_SIZE, false,
metasize); metasize, headroom);
return head_skb; return head_skb;
} }
break; break;
...@@ -980,7 +1007,7 @@ static struct sk_buff *receive_mergeable(struct net_device *dev, ...@@ -980,7 +1007,7 @@ static struct sk_buff *receive_mergeable(struct net_device *dev,
} }
head_skb = page_to_skb(vi, rq, page, offset, len, truesize, !xdp_prog, head_skb = page_to_skb(vi, rq, page, offset, len, truesize, !xdp_prog,
metasize); metasize, headroom);
curr_skb = head_skb; curr_skb = head_skb;
if (unlikely(!curr_skb)) if (unlikely(!curr_skb))
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment