Commit 44456565 authored by Linus Torvalds's avatar Linus Torvalds

Merge tag 'io_uring-5.7-2020-05-22' of git://git.kernel.dk/linux-block

Pull io_uring fixes from Jens Axboe:
 "A small collection of small fixes that should go into this release:

   - Two fixes for async request preparation (Pavel)

   - Busy clear fix for SQPOLL (Xiaoguang)

   - Don't use kiocb->private for O_DIRECT buf index, some file systems
     use it (Bijan)

   - Kill dead check in io_splice()

   - Ensure sqo_wait is initialized early

   - Cancel task_work if we fail adding to original process

   - Only add (IO)pollable requests to iopoll list, fixing a regression
     in this merge window"

* tag 'io_uring-5.7-2020-05-22' of git://git.kernel.dk/linux-block:
  io_uring: reset -EBUSY error when io sq thread is waken up
  io_uring: don't add non-IO requests to iopoll pending list
  io_uring: don't use kiocb.private to store buf_index
  io_uring: cancel work if task_work_add() fails
  io_uring: remove dead check in io_splice()
  io_uring: fix FORCE_ASYNC req preparation
  io_uring: don't prepare DRAIN reqs twice
  io_uring: initialize ctx->sqo_wait earlier
parents db9f3847 d4ae271d
...@@ -619,6 +619,8 @@ struct io_kiocb { ...@@ -619,6 +619,8 @@ struct io_kiocb {
bool needs_fixed_file; bool needs_fixed_file;
u8 opcode; u8 opcode;
u16 buf_index;
struct io_ring_ctx *ctx; struct io_ring_ctx *ctx;
struct list_head list; struct list_head list;
unsigned int flags; unsigned int flags;
...@@ -924,6 +926,7 @@ static struct io_ring_ctx *io_ring_ctx_alloc(struct io_uring_params *p) ...@@ -924,6 +926,7 @@ static struct io_ring_ctx *io_ring_ctx_alloc(struct io_uring_params *p)
goto err; goto err;
ctx->flags = p->flags; ctx->flags = p->flags;
init_waitqueue_head(&ctx->sqo_wait);
init_waitqueue_head(&ctx->cq_wait); init_waitqueue_head(&ctx->cq_wait);
INIT_LIST_HEAD(&ctx->cq_overflow_list); INIT_LIST_HEAD(&ctx->cq_overflow_list);
init_completion(&ctx->completions[0]); init_completion(&ctx->completions[0]);
...@@ -2100,9 +2103,7 @@ static int io_prep_rw(struct io_kiocb *req, const struct io_uring_sqe *sqe, ...@@ -2100,9 +2103,7 @@ static int io_prep_rw(struct io_kiocb *req, const struct io_uring_sqe *sqe,
req->rw.addr = READ_ONCE(sqe->addr); req->rw.addr = READ_ONCE(sqe->addr);
req->rw.len = READ_ONCE(sqe->len); req->rw.len = READ_ONCE(sqe->len);
/* we own ->private, reuse it for the buffer index / buffer ID */ req->buf_index = READ_ONCE(sqe->buf_index);
req->rw.kiocb.private = (void *) (unsigned long)
READ_ONCE(sqe->buf_index);
return 0; return 0;
} }
...@@ -2145,7 +2146,7 @@ static ssize_t io_import_fixed(struct io_kiocb *req, int rw, ...@@ -2145,7 +2146,7 @@ static ssize_t io_import_fixed(struct io_kiocb *req, int rw,
struct io_ring_ctx *ctx = req->ctx; struct io_ring_ctx *ctx = req->ctx;
size_t len = req->rw.len; size_t len = req->rw.len;
struct io_mapped_ubuf *imu; struct io_mapped_ubuf *imu;
unsigned index, buf_index; u16 index, buf_index;
size_t offset; size_t offset;
u64 buf_addr; u64 buf_addr;
...@@ -2153,7 +2154,7 @@ static ssize_t io_import_fixed(struct io_kiocb *req, int rw, ...@@ -2153,7 +2154,7 @@ static ssize_t io_import_fixed(struct io_kiocb *req, int rw,
if (unlikely(!ctx->user_bufs)) if (unlikely(!ctx->user_bufs))
return -EFAULT; return -EFAULT;
buf_index = (unsigned long) req->rw.kiocb.private; buf_index = req->buf_index;
if (unlikely(buf_index >= ctx->nr_user_bufs)) if (unlikely(buf_index >= ctx->nr_user_bufs))
return -EFAULT; return -EFAULT;
...@@ -2269,10 +2270,10 @@ static void __user *io_rw_buffer_select(struct io_kiocb *req, size_t *len, ...@@ -2269,10 +2270,10 @@ static void __user *io_rw_buffer_select(struct io_kiocb *req, size_t *len,
bool needs_lock) bool needs_lock)
{ {
struct io_buffer *kbuf; struct io_buffer *kbuf;
int bgid; u16 bgid;
kbuf = (struct io_buffer *) (unsigned long) req->rw.addr; kbuf = (struct io_buffer *) (unsigned long) req->rw.addr;
bgid = (int) (unsigned long) req->rw.kiocb.private; bgid = req->buf_index;
kbuf = io_buffer_select(req, len, bgid, kbuf, needs_lock); kbuf = io_buffer_select(req, len, bgid, kbuf, needs_lock);
if (IS_ERR(kbuf)) if (IS_ERR(kbuf))
return kbuf; return kbuf;
...@@ -2363,7 +2364,7 @@ static ssize_t io_import_iovec(int rw, struct io_kiocb *req, ...@@ -2363,7 +2364,7 @@ static ssize_t io_import_iovec(int rw, struct io_kiocb *req,
} }
/* buffer index only valid with fixed read/write, or buffer select */ /* buffer index only valid with fixed read/write, or buffer select */
if (req->rw.kiocb.private && !(req->flags & REQ_F_BUFFER_SELECT)) if (req->buf_index && !(req->flags & REQ_F_BUFFER_SELECT))
return -EINVAL; return -EINVAL;
if (opcode == IORING_OP_READ || opcode == IORING_OP_WRITE) { if (opcode == IORING_OP_READ || opcode == IORING_OP_WRITE) {
...@@ -2771,11 +2772,8 @@ static int io_splice(struct io_kiocb *req, bool force_nonblock) ...@@ -2771,11 +2772,8 @@ static int io_splice(struct io_kiocb *req, bool force_nonblock)
poff_in = (sp->off_in == -1) ? NULL : &sp->off_in; poff_in = (sp->off_in == -1) ? NULL : &sp->off_in;
poff_out = (sp->off_out == -1) ? NULL : &sp->off_out; poff_out = (sp->off_out == -1) ? NULL : &sp->off_out;
if (sp->len) { if (sp->len)
ret = do_splice(in, poff_in, out, poff_out, sp->len, flags); ret = do_splice(in, poff_in, out, poff_out, sp->len, flags);
if (force_nonblock && ret == -EAGAIN)
return -EAGAIN;
}
io_put_file(req, in, (sp->flags & SPLICE_F_FD_IN_FIXED)); io_put_file(req, in, (sp->flags & SPLICE_F_FD_IN_FIXED));
req->flags &= ~REQ_F_NEED_CLEANUP; req->flags &= ~REQ_F_NEED_CLEANUP;
...@@ -4137,12 +4135,14 @@ static int __io_async_wake(struct io_kiocb *req, struct io_poll_iocb *poll, ...@@ -4137,12 +4135,14 @@ static int __io_async_wake(struct io_kiocb *req, struct io_poll_iocb *poll,
req->result = mask; req->result = mask;
init_task_work(&req->task_work, func); init_task_work(&req->task_work, func);
/* /*
* If this fails, then the task is exiting. Punt to one of the io-wq * If this fails, then the task is exiting. When a task exits, the
* threads to ensure the work gets run, we can't always rely on exit * work gets canceled, so just cancel this request as well instead
* cancelation taking care of this. * of executing it. We can't safely execute it anyway, as we may not
* have the needed state needed for it anyway.
*/ */
ret = task_work_add(tsk, &req->task_work, true); ret = task_work_add(tsk, &req->task_work, true);
if (unlikely(ret)) { if (unlikely(ret)) {
WRITE_ONCE(poll->canceled, true);
tsk = io_wq_get_task(req->ctx->io_wq); tsk = io_wq_get_task(req->ctx->io_wq);
task_work_add(tsk, &req->task_work, true); task_work_add(tsk, &req->task_work, true);
} }
...@@ -5013,12 +5013,13 @@ static int io_req_defer(struct io_kiocb *req, const struct io_uring_sqe *sqe) ...@@ -5013,12 +5013,13 @@ static int io_req_defer(struct io_kiocb *req, const struct io_uring_sqe *sqe)
if (!req_need_defer(req) && list_empty_careful(&ctx->defer_list)) if (!req_need_defer(req) && list_empty_careful(&ctx->defer_list))
return 0; return 0;
if (!req->io && io_alloc_async_ctx(req)) if (!req->io) {
return -EAGAIN; if (io_alloc_async_ctx(req))
return -EAGAIN;
ret = io_req_defer_prep(req, sqe); ret = io_req_defer_prep(req, sqe);
if (ret < 0) if (ret < 0)
return ret; return ret;
}
spin_lock_irq(&ctx->completion_lock); spin_lock_irq(&ctx->completion_lock);
if (!req_need_defer(req) && list_empty(&ctx->defer_list)) { if (!req_need_defer(req) && list_empty(&ctx->defer_list)) {
...@@ -5305,7 +5306,8 @@ static int io_issue_sqe(struct io_kiocb *req, const struct io_uring_sqe *sqe, ...@@ -5305,7 +5306,8 @@ static int io_issue_sqe(struct io_kiocb *req, const struct io_uring_sqe *sqe,
if (ret) if (ret)
return ret; return ret;
if (ctx->flags & IORING_SETUP_IOPOLL) { /* If the op doesn't have a file, we're not polling for it */
if ((ctx->flags & IORING_SETUP_IOPOLL) && req->file) {
const bool in_async = io_wq_current_is_worker(); const bool in_async = io_wq_current_is_worker();
if (req->result == -EAGAIN) if (req->result == -EAGAIN)
...@@ -5606,9 +5608,15 @@ static void io_queue_sqe(struct io_kiocb *req, const struct io_uring_sqe *sqe) ...@@ -5606,9 +5608,15 @@ static void io_queue_sqe(struct io_kiocb *req, const struct io_uring_sqe *sqe)
io_double_put_req(req); io_double_put_req(req);
} }
} else if (req->flags & REQ_F_FORCE_ASYNC) { } else if (req->flags & REQ_F_FORCE_ASYNC) {
ret = io_req_defer_prep(req, sqe); if (!req->io) {
if (unlikely(ret < 0)) ret = -EAGAIN;
goto fail_req; if (io_alloc_async_ctx(req))
goto fail_req;
ret = io_req_defer_prep(req, sqe);
if (unlikely(ret < 0))
goto fail_req;
}
/* /*
* Never try inline submit of IOSQE_ASYNC is set, go straight * Never try inline submit of IOSQE_ASYNC is set, go straight
* to async execution. * to async execution.
...@@ -6024,6 +6032,7 @@ static int io_sq_thread(void *data) ...@@ -6024,6 +6032,7 @@ static int io_sq_thread(void *data)
finish_wait(&ctx->sqo_wait, &wait); finish_wait(&ctx->sqo_wait, &wait);
ctx->rings->sq_flags &= ~IORING_SQ_NEED_WAKEUP; ctx->rings->sq_flags &= ~IORING_SQ_NEED_WAKEUP;
ret = 0;
continue; continue;
} }
finish_wait(&ctx->sqo_wait, &wait); finish_wait(&ctx->sqo_wait, &wait);
...@@ -6837,7 +6846,6 @@ static int io_sq_offload_start(struct io_ring_ctx *ctx, ...@@ -6837,7 +6846,6 @@ static int io_sq_offload_start(struct io_ring_ctx *ctx,
{ {
int ret; int ret;
init_waitqueue_head(&ctx->sqo_wait);
mmgrab(current->mm); mmgrab(current->mm);
ctx->sqo_mm = current->mm; ctx->sqo_mm = current->mm;
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment