Commit f17a1a55 authored by Andy Grover's avatar Andy Grover

RDS: Refill recv ring directly from tasklet

Performance is better if we use allocations that don't block
to refill the receive ring. Since the whole reason we were
kicking out to the worker thread was so we could do blocking
allocs, we no longer need to do this.

Remove gfp params from rds_ib_recv_refill(); we always use
GFP_NOWAIT.
Signed-off-by: default avatarAndy Grover <andy.grover@oracle.com>
parent 77dd550e
...@@ -297,8 +297,7 @@ void rds_ib_flush_mrs(void); ...@@ -297,8 +297,7 @@ void rds_ib_flush_mrs(void);
int __init rds_ib_recv_init(void); int __init rds_ib_recv_init(void);
void rds_ib_recv_exit(void); void rds_ib_recv_exit(void);
int rds_ib_recv(struct rds_connection *conn); int rds_ib_recv(struct rds_connection *conn);
int rds_ib_recv_refill(struct rds_connection *conn, gfp_t kptr_gfp, int rds_ib_recv_refill(struct rds_connection *conn, int prefill);
gfp_t page_gfp, int prefill);
void rds_ib_inc_free(struct rds_incoming *inc); void rds_ib_inc_free(struct rds_incoming *inc);
int rds_ib_inc_copy_to_user(struct rds_incoming *inc, struct iovec *iov, int rds_ib_inc_copy_to_user(struct rds_incoming *inc, struct iovec *iov,
size_t size); size_t size);
......
...@@ -135,7 +135,7 @@ void rds_ib_cm_connect_complete(struct rds_connection *conn, struct rdma_cm_even ...@@ -135,7 +135,7 @@ void rds_ib_cm_connect_complete(struct rds_connection *conn, struct rdma_cm_even
rds_ib_recv_init_ring(ic); rds_ib_recv_init_ring(ic);
/* Post receive buffers - as a side effect, this will update /* Post receive buffers - as a side effect, this will update
* the posted credit count. */ * the posted credit count. */
rds_ib_recv_refill(conn, GFP_KERNEL, GFP_HIGHUSER, 1); rds_ib_recv_refill(conn, 1);
/* Tune RNR behavior */ /* Tune RNR behavior */
rds_ib_tune_rnr(ic, &qp_attr); rds_ib_tune_rnr(ic, &qp_attr);
......
...@@ -135,8 +135,7 @@ void rds_ib_recv_clear_ring(struct rds_ib_connection *ic) ...@@ -135,8 +135,7 @@ void rds_ib_recv_clear_ring(struct rds_ib_connection *ic)
} }
static int rds_ib_recv_refill_one(struct rds_connection *conn, static int rds_ib_recv_refill_one(struct rds_connection *conn,
struct rds_ib_recv_work *recv, struct rds_ib_recv_work *recv)
gfp_t kptr_gfp, gfp_t page_gfp)
{ {
struct rds_ib_connection *ic = conn->c_transport_data; struct rds_ib_connection *ic = conn->c_transport_data;
dma_addr_t dma_addr; dma_addr_t dma_addr;
...@@ -148,8 +147,7 @@ static int rds_ib_recv_refill_one(struct rds_connection *conn, ...@@ -148,8 +147,7 @@ static int rds_ib_recv_refill_one(struct rds_connection *conn,
rds_ib_stats_inc(s_ib_rx_alloc_limit); rds_ib_stats_inc(s_ib_rx_alloc_limit);
goto out; goto out;
} }
recv->r_ibinc = kmem_cache_alloc(rds_ib_incoming_slab, recv->r_ibinc = kmem_cache_alloc(rds_ib_incoming_slab, GFP_NOWAIT);
kptr_gfp);
if (!recv->r_ibinc) { if (!recv->r_ibinc) {
atomic_dec(&rds_ib_allocation); atomic_dec(&rds_ib_allocation);
goto out; goto out;
...@@ -159,7 +157,7 @@ static int rds_ib_recv_refill_one(struct rds_connection *conn, ...@@ -159,7 +157,7 @@ static int rds_ib_recv_refill_one(struct rds_connection *conn,
} }
if (!recv->r_frag) { if (!recv->r_frag) {
recv->r_frag = kmem_cache_alloc(rds_ib_frag_slab, kptr_gfp); recv->r_frag = kmem_cache_alloc(rds_ib_frag_slab, GFP_NOWAIT);
if (!recv->r_frag) if (!recv->r_frag)
goto out; goto out;
INIT_LIST_HEAD(&recv->r_frag->f_item); INIT_LIST_HEAD(&recv->r_frag->f_item);
...@@ -167,7 +165,7 @@ static int rds_ib_recv_refill_one(struct rds_connection *conn, ...@@ -167,7 +165,7 @@ static int rds_ib_recv_refill_one(struct rds_connection *conn,
} }
if (!ic->i_frag.f_page) { if (!ic->i_frag.f_page) {
ic->i_frag.f_page = alloc_page(page_gfp); ic->i_frag.f_page = alloc_page(GFP_NOWAIT);
if (!ic->i_frag.f_page) if (!ic->i_frag.f_page)
goto out; goto out;
ic->i_frag.f_offset = 0; ic->i_frag.f_offset = 0;
...@@ -221,8 +219,7 @@ static int rds_ib_recv_refill_one(struct rds_connection *conn, ...@@ -221,8 +219,7 @@ static int rds_ib_recv_refill_one(struct rds_connection *conn,
* *
* -1 is returned if posting fails due to temporary resource exhaustion. * -1 is returned if posting fails due to temporary resource exhaustion.
*/ */
int rds_ib_recv_refill(struct rds_connection *conn, gfp_t kptr_gfp, int rds_ib_recv_refill(struct rds_connection *conn, int prefill)
gfp_t page_gfp, int prefill)
{ {
struct rds_ib_connection *ic = conn->c_transport_data; struct rds_ib_connection *ic = conn->c_transport_data;
struct rds_ib_recv_work *recv; struct rds_ib_recv_work *recv;
...@@ -241,7 +238,7 @@ int rds_ib_recv_refill(struct rds_connection *conn, gfp_t kptr_gfp, ...@@ -241,7 +238,7 @@ int rds_ib_recv_refill(struct rds_connection *conn, gfp_t kptr_gfp,
} }
recv = &ic->i_recvs[pos]; recv = &ic->i_recvs[pos];
ret = rds_ib_recv_refill_one(conn, recv, kptr_gfp, page_gfp); ret = rds_ib_recv_refill_one(conn, recv);
if (ret) { if (ret) {
ret = -1; ret = -1;
break; break;
...@@ -856,11 +853,8 @@ void rds_ib_recv_tasklet_fn(unsigned long data) ...@@ -856,11 +853,8 @@ void rds_ib_recv_tasklet_fn(unsigned long data)
if (rds_ib_ring_empty(&ic->i_recv_ring)) if (rds_ib_ring_empty(&ic->i_recv_ring))
rds_ib_stats_inc(s_ib_rx_ring_empty); rds_ib_stats_inc(s_ib_rx_ring_empty);
/*
* If the ring is running low, then schedule the thread to refill.
*/
if (rds_ib_ring_low(&ic->i_recv_ring)) if (rds_ib_ring_low(&ic->i_recv_ring))
queue_delayed_work(rds_wq, &conn->c_recv_w, 0); rds_ib_recv_refill(conn, 0);
} }
int rds_ib_recv(struct rds_connection *conn) int rds_ib_recv(struct rds_connection *conn)
...@@ -875,7 +869,7 @@ int rds_ib_recv(struct rds_connection *conn) ...@@ -875,7 +869,7 @@ int rds_ib_recv(struct rds_connection *conn)
* we're really low and we want the caller to back off for a bit. * we're really low and we want the caller to back off for a bit.
*/ */
mutex_lock(&ic->i_recv_mutex); mutex_lock(&ic->i_recv_mutex);
if (rds_ib_recv_refill(conn, GFP_KERNEL, GFP_HIGHUSER, 0)) if (rds_ib_recv_refill(conn, 0))
ret = -ENOMEM; ret = -ENOMEM;
else else
rds_ib_stats_inc(s_ib_rx_refill_from_thread); rds_ib_stats_inc(s_ib_rx_refill_from_thread);
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment