Commit 6aab0bb0 authored by Magnus Karlsson's avatar Magnus Karlsson Committed by Daniel Borkmann

i40e: Use the xsk batched rx allocation interface

Use the new xsk batched rx allocation interface for the zero-copy data
path. As the array of struct xdp_buff pointers kept by the driver is
really a ring that wraps, the allocation routine is modified to detect
a wrap and in that case call the allocation function twice. The
allocation function cannot deal with wrapped rings, only arrays. As we
now know exactly how many buffers we get and that there is no
wrapping, the allocation function can be simplified even more as all
if-statements in the allocation loop can be removed, improving
performance.
Signed-off-by: default avatarMagnus Karlsson <magnus.karlsson@intel.com>
Signed-off-by: default avatarDaniel Borkmann <daniel@iogearbox.net>
Link: https://lore.kernel.org/bpf/20210922075613.12186-6-magnus.karlsson@gmail.com
parent db804cfc
...@@ -193,42 +193,40 @@ bool i40e_alloc_rx_buffers_zc(struct i40e_ring *rx_ring, u16 count) ...@@ -193,42 +193,40 @@ bool i40e_alloc_rx_buffers_zc(struct i40e_ring *rx_ring, u16 count)
{ {
u16 ntu = rx_ring->next_to_use; u16 ntu = rx_ring->next_to_use;
union i40e_rx_desc *rx_desc; union i40e_rx_desc *rx_desc;
struct xdp_buff **bi, *xdp; struct xdp_buff **xdp;
u32 nb_buffs, i;
dma_addr_t dma; dma_addr_t dma;
bool ok = true;
rx_desc = I40E_RX_DESC(rx_ring, ntu); rx_desc = I40E_RX_DESC(rx_ring, ntu);
bi = i40e_rx_bi(rx_ring, ntu); xdp = i40e_rx_bi(rx_ring, ntu);
do {
xdp = xsk_buff_alloc(rx_ring->xsk_pool); nb_buffs = min_t(u16, count, rx_ring->count - ntu);
if (!xdp) { nb_buffs = xsk_buff_alloc_batch(rx_ring->xsk_pool, xdp, nb_buffs);
ok = false; if (!nb_buffs)
goto no_buffers; return false;
}
*bi = xdp; i = nb_buffs;
dma = xsk_buff_xdp_get_dma(xdp); while (i--) {
dma = xsk_buff_xdp_get_dma(*xdp);
rx_desc->read.pkt_addr = cpu_to_le64(dma); rx_desc->read.pkt_addr = cpu_to_le64(dma);
rx_desc->read.hdr_addr = 0; rx_desc->read.hdr_addr = 0;
rx_desc++; rx_desc++;
bi++; xdp++;
ntu++; }
if (unlikely(ntu == rx_ring->count)) {
rx_desc = I40E_RX_DESC(rx_ring, 0);
bi = i40e_rx_bi(rx_ring, 0);
ntu = 0;
}
} while (--count);
no_buffers: ntu += nb_buffs;
if (rx_ring->next_to_use != ntu) { if (ntu == rx_ring->count) {
/* clear the status bits for the next_to_use descriptor */ rx_desc = I40E_RX_DESC(rx_ring, 0);
rx_desc->wb.qword1.status_error_len = 0; xdp = i40e_rx_bi(rx_ring, 0);
i40e_release_rx_desc(rx_ring, ntu); ntu = 0;
} }
return ok; /* clear the status bits for the next_to_use descriptor */
rx_desc->wb.qword1.status_error_len = 0;
i40e_release_rx_desc(rx_ring, ntu);
return count == nb_buffs ? true : false;
} }
/** /**
...@@ -365,7 +363,7 @@ int i40e_clean_rx_irq_zc(struct i40e_ring *rx_ring, int budget) ...@@ -365,7 +363,7 @@ int i40e_clean_rx_irq_zc(struct i40e_ring *rx_ring, int budget)
break; break;
bi = *i40e_rx_bi(rx_ring, next_to_clean); bi = *i40e_rx_bi(rx_ring, next_to_clean);
bi->data_end = bi->data + size; xsk_buff_set_size(bi, size);
xsk_buff_dma_sync_for_cpu(bi, rx_ring->xsk_pool); xsk_buff_dma_sync_for_cpu(bi, rx_ring->xsk_pool);
xdp_res = i40e_run_xdp_zc(rx_ring, bi); xdp_res = i40e_run_xdp_zc(rx_ring, bi);
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment