Commit e70cb608 authored by Pavel Begunkov's avatar Pavel Begunkov Committed by Jens Axboe

io_uring: export io_put_task()

Make io_put_task() available to non-core parts of io_uring, we'll need
it for notification infrastructure.
Signed-off-by: default avatarPavel Begunkov <asml.silence@gmail.com>
Link: https://lore.kernel.org/r/3686807d4c03b72e389947b0e8692d4d44334ef0.1657643355.git.asml.silence@gmail.comSigned-off-by: default avatarJens Axboe <axboe@kernel.dk>
parent e02b6651
...@@ -4,6 +4,7 @@ ...@@ -4,6 +4,7 @@
#include <linux/blkdev.h> #include <linux/blkdev.h>
#include <linux/task_work.h> #include <linux/task_work.h>
#include <linux/bitmap.h> #include <linux/bitmap.h>
#include <linux/llist.h>
#include <uapi/linux/io_uring.h> #include <uapi/linux/io_uring.h>
struct io_wq_work_node { struct io_wq_work_node {
...@@ -43,6 +44,30 @@ struct io_hash_table { ...@@ -43,6 +44,30 @@ struct io_hash_table {
unsigned hash_bits; unsigned hash_bits;
}; };
/*
* Arbitrary limit, can be raised if need be
*/
#define IO_RINGFD_REG_MAX 16
struct io_uring_task {
/* submission side */
int cached_refs;
const struct io_ring_ctx *last;
struct io_wq *io_wq;
struct file *registered_rings[IO_RINGFD_REG_MAX];
struct xarray xa;
struct wait_queue_head wait;
atomic_t in_idle;
atomic_t inflight_tracked;
struct percpu_counter inflight;
struct { /* task_work */
struct llist_head task_list;
struct callback_head task_work;
} ____cacheline_aligned_in_smp;
};
struct io_uring { struct io_uring {
u32 head ____cacheline_aligned_in_smp; u32 head ____cacheline_aligned_in_smp;
u32 tail ____cacheline_aligned_in_smp; u32 tail ____cacheline_aligned_in_smp;
......
...@@ -608,7 +608,7 @@ static bool io_cqring_overflow_flush(struct io_ring_ctx *ctx) ...@@ -608,7 +608,7 @@ static bool io_cqring_overflow_flush(struct io_ring_ctx *ctx)
return ret; return ret;
} }
static void __io_put_task(struct task_struct *task, int nr) void __io_put_task(struct task_struct *task, int nr)
{ {
struct io_uring_task *tctx = task->io_uring; struct io_uring_task *tctx = task->io_uring;
...@@ -618,15 +618,6 @@ static void __io_put_task(struct task_struct *task, int nr) ...@@ -618,15 +618,6 @@ static void __io_put_task(struct task_struct *task, int nr)
put_task_struct_many(task, nr); put_task_struct_many(task, nr);
} }
/* must to be called somewhat shortly after putting a request */
static inline void io_put_task(struct task_struct *task, int nr)
{
if (likely(task == current))
task->io_uring->cached_refs += nr;
else
__io_put_task(task, nr);
}
static void io_task_refs_refill(struct io_uring_task *tctx) static void io_task_refs_refill(struct io_uring_task *tctx)
{ {
unsigned int refill = -tctx->cached_refs + IO_TCTX_REFS_CACHE_NR; unsigned int refill = -tctx->cached_refs + IO_TCTX_REFS_CACHE_NR;
......
...@@ -71,6 +71,7 @@ void io_wq_submit_work(struct io_wq_work *work); ...@@ -71,6 +71,7 @@ void io_wq_submit_work(struct io_wq_work *work);
void io_free_req(struct io_kiocb *req); void io_free_req(struct io_kiocb *req);
void io_queue_next(struct io_kiocb *req); void io_queue_next(struct io_kiocb *req);
void __io_put_task(struct task_struct *task, int nr);
bool io_match_task_safe(struct io_kiocb *head, struct task_struct *task, bool io_match_task_safe(struct io_kiocb *head, struct task_struct *task,
bool cancel_all); bool cancel_all);
...@@ -258,4 +259,13 @@ static inline void io_commit_cqring_flush(struct io_ring_ctx *ctx) ...@@ -258,4 +259,13 @@ static inline void io_commit_cqring_flush(struct io_ring_ctx *ctx)
__io_commit_cqring_flush(ctx); __io_commit_cqring_flush(ctx);
} }
/* must to be called somewhat shortly after putting a request */
static inline void io_put_task(struct task_struct *task, int nr)
{
if (likely(task == current))
task->io_uring->cached_refs += nr;
else
__io_put_task(task, nr);
}
#endif #endif
// SPDX-License-Identifier: GPL-2.0 // SPDX-License-Identifier: GPL-2.0
#include <linux/llist.h>
/*
* Arbitrary limit, can be raised if need be
*/
#define IO_RINGFD_REG_MAX 16
struct io_uring_task {
/* submission side */
int cached_refs;
const struct io_ring_ctx *last;
struct io_wq *io_wq;
struct file *registered_rings[IO_RINGFD_REG_MAX];
struct xarray xa;
struct wait_queue_head wait;
atomic_t in_idle;
atomic_t inflight_tracked;
struct percpu_counter inflight;
struct { /* task_work */
struct llist_head task_list;
struct callback_head task_work;
} ____cacheline_aligned_in_smp;
};
struct io_tctx_node { struct io_tctx_node {
struct list_head ctx_node; struct list_head ctx_node;
struct task_struct *task; struct task_struct *task;
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment