Commit d245bca6 authored by Pavel Begunkov's avatar Pavel Begunkov Committed by Jens Axboe

io_uring: don't expose io_fill_cqe_aux()

Deduplicate some code and add a helper for filling an aux CQE, locking
and notification.
Signed-off-by: default avatarPavel Begunkov <asml.silence@gmail.com>
Link: https://lore.kernel.org/r/b7c6557c8f9dc5c4cfb01292116c682a0ff61081.1655455613.git.asml.silence@gmail.comSigned-off-by: default avatarJens Axboe <axboe@kernel.dk>
parent f09c8643
...@@ -676,8 +676,8 @@ bool io_cqring_event_overflow(struct io_ring_ctx *ctx, u64 user_data, s32 res, ...@@ -676,8 +676,8 @@ bool io_cqring_event_overflow(struct io_ring_ctx *ctx, u64 user_data, s32 res,
return true; return true;
} }
bool io_fill_cqe_aux(struct io_ring_ctx *ctx, u64 user_data, s32 res, static bool io_fill_cqe_aux(struct io_ring_ctx *ctx,
u32 cflags) u64 user_data, s32 res, u32 cflags)
{ {
struct io_uring_cqe *cqe; struct io_uring_cqe *cqe;
...@@ -704,6 +704,20 @@ bool io_fill_cqe_aux(struct io_ring_ctx *ctx, u64 user_data, s32 res, ...@@ -704,6 +704,20 @@ bool io_fill_cqe_aux(struct io_ring_ctx *ctx, u64 user_data, s32 res,
return io_cqring_event_overflow(ctx, user_data, res, cflags, 0, 0); return io_cqring_event_overflow(ctx, user_data, res, cflags, 0, 0);
} }
bool io_post_aux_cqe(struct io_ring_ctx *ctx,
u64 user_data, s32 res, u32 cflags)
{
bool filled;
spin_lock(&ctx->completion_lock);
filled = io_fill_cqe_aux(ctx, user_data, res, cflags);
io_commit_cqring(ctx);
spin_unlock(&ctx->completion_lock);
if (filled)
io_cqring_ev_posted(ctx);
return filled;
}
static void __io_req_complete_put(struct io_kiocb *req) static void __io_req_complete_put(struct io_kiocb *req)
{ {
/* /*
......
...@@ -239,8 +239,7 @@ void io_req_complete_failed(struct io_kiocb *req, s32 res); ...@@ -239,8 +239,7 @@ void io_req_complete_failed(struct io_kiocb *req, s32 res);
void __io_req_complete(struct io_kiocb *req, unsigned issue_flags); void __io_req_complete(struct io_kiocb *req, unsigned issue_flags);
void io_req_complete_post(struct io_kiocb *req); void io_req_complete_post(struct io_kiocb *req);
void __io_req_complete_post(struct io_kiocb *req); void __io_req_complete_post(struct io_kiocb *req);
bool io_fill_cqe_aux(struct io_ring_ctx *ctx, u64 user_data, s32 res, bool io_post_aux_cqe(struct io_ring_ctx *ctx, u64 user_data, s32 res, u32 cflags);
u32 cflags);
void io_cqring_ev_posted(struct io_ring_ctx *ctx); void io_cqring_ev_posted(struct io_ring_ctx *ctx);
void __io_commit_cqring_flush(struct io_ring_ctx *ctx); void __io_commit_cqring_flush(struct io_ring_ctx *ctx);
......
...@@ -34,7 +34,6 @@ int io_msg_ring(struct io_kiocb *req, unsigned int issue_flags) ...@@ -34,7 +34,6 @@ int io_msg_ring(struct io_kiocb *req, unsigned int issue_flags)
{ {
struct io_msg *msg = io_kiocb_to_cmd(req); struct io_msg *msg = io_kiocb_to_cmd(req);
struct io_ring_ctx *target_ctx; struct io_ring_ctx *target_ctx;
bool filled;
int ret; int ret;
ret = -EBADFD; ret = -EBADFD;
...@@ -43,16 +42,8 @@ int io_msg_ring(struct io_kiocb *req, unsigned int issue_flags) ...@@ -43,16 +42,8 @@ int io_msg_ring(struct io_kiocb *req, unsigned int issue_flags)
ret = -EOVERFLOW; ret = -EOVERFLOW;
target_ctx = req->file->private_data; target_ctx = req->file->private_data;
if (io_post_aux_cqe(target_ctx, msg->user_data, msg->len, 0))
spin_lock(&target_ctx->completion_lock);
filled = io_fill_cqe_aux(target_ctx, msg->user_data, msg->len, 0);
io_commit_cqring(target_ctx);
spin_unlock(&target_ctx->completion_lock);
if (filled) {
io_cqring_ev_posted(target_ctx);
ret = 0; ret = 0;
}
done: done:
if (ret < 0) if (ret < 0)
......
...@@ -644,22 +644,12 @@ int io_accept(struct io_kiocb *req, unsigned int issue_flags) ...@@ -644,22 +644,12 @@ int io_accept(struct io_kiocb *req, unsigned int issue_flags)
io_req_set_res(req, ret, 0); io_req_set_res(req, ret, 0);
return IOU_OK; return IOU_OK;
} }
if (ret >= 0) {
bool filled;
spin_lock(&ctx->completion_lock);
filled = io_fill_cqe_aux(ctx, req->cqe.user_data, ret,
IORING_CQE_F_MORE);
io_commit_cqring(ctx);
spin_unlock(&ctx->completion_lock);
if (filled) {
io_cqring_ev_posted(ctx);
goto retry;
}
ret = -ECANCELED;
}
if (ret < 0)
return ret; return ret;
if (io_post_aux_cqe(ctx, req->cqe.user_data, ret, IORING_CQE_F_MORE))
goto retry;
return -ECANCELED;
} }
int io_socket_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) int io_socket_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
......
...@@ -214,23 +214,15 @@ static int io_poll_check_events(struct io_kiocb *req, bool *locked) ...@@ -214,23 +214,15 @@ static int io_poll_check_events(struct io_kiocb *req, bool *locked)
if (!(req->flags & REQ_F_APOLL_MULTISHOT)) { if (!(req->flags & REQ_F_APOLL_MULTISHOT)) {
__poll_t mask = mangle_poll(req->cqe.res & __poll_t mask = mangle_poll(req->cqe.res &
req->apoll_events); req->apoll_events);
bool filled;
spin_lock(&ctx->completion_lock);
filled = io_fill_cqe_aux(ctx, req->cqe.user_data,
mask, IORING_CQE_F_MORE);
io_commit_cqring(ctx);
spin_unlock(&ctx->completion_lock);
if (filled) {
io_cqring_ev_posted(ctx);
continue;
}
return -ECANCELED;
}
if (!io_post_aux_cqe(ctx, req->cqe.user_data,
mask, IORING_CQE_F_MORE))
return -ECANCELED;
} else {
ret = io_poll_issue(req, locked); ret = io_poll_issue(req, locked);
if (ret) if (ret)
return ret; return ret;
}
/* /*
* Release all references, retry if someone tried to restart * Release all references, retry if someone tried to restart
......
...@@ -174,17 +174,13 @@ static void __io_rsrc_put_work(struct io_rsrc_node *ref_node) ...@@ -174,17 +174,13 @@ static void __io_rsrc_put_work(struct io_rsrc_node *ref_node)
list_del(&prsrc->list); list_del(&prsrc->list);
if (prsrc->tag) { if (prsrc->tag) {
if (ctx->flags & IORING_SETUP_IOPOLL) if (ctx->flags & IORING_SETUP_IOPOLL) {
mutex_lock(&ctx->uring_lock); mutex_lock(&ctx->uring_lock);
io_post_aux_cqe(ctx, prsrc->tag, 0, 0);
spin_lock(&ctx->completion_lock);
io_fill_cqe_aux(ctx, prsrc->tag, 0, 0);
io_commit_cqring(ctx);
spin_unlock(&ctx->completion_lock);
io_cqring_ev_posted(ctx);
if (ctx->flags & IORING_SETUP_IOPOLL)
mutex_unlock(&ctx->uring_lock); mutex_unlock(&ctx->uring_lock);
} else {
io_post_aux_cqe(ctx, prsrc->tag, 0, 0);
}
} }
rsrc_data->do_put(ctx, prsrc); rsrc_data->do_put(ctx, prsrc);
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment