Commit c13660a0 authored by Jens Axboe's avatar Jens Axboe

blk-mq-sched: change ->dispatch_requests() to ->dispatch_request()

When we invoke dispatch_requests(), the scheduler empties everything
into the passed in list. This isn't always a good thing, since it
means that we remove items that we could have potentially merged
with.

Change the function to dispatch single requests at the time. If
we do that, we can backoff exactly at the point where the device
can't consume more IO, and leave the rest with the scheduler for
better merging and future dispatch decision making.
Signed-off-by: default avatarJens Axboe <axboe@fb.com>
Reviewed-by: default avatarOmar Sandoval <osandov@fb.com>
Tested-by: default avatarHannes Reinecke <hare@suse.com>
parent 50e1dab8
...@@ -201,15 +201,22 @@ void blk_mq_sched_dispatch_requests(struct blk_mq_hw_ctx *hctx) ...@@ -201,15 +201,22 @@ void blk_mq_sched_dispatch_requests(struct blk_mq_hw_ctx *hctx)
* leave them there for as long as we can. Mark the hw queue as * leave them there for as long as we can. Mark the hw queue as
* needing a restart in that case. * needing a restart in that case.
*/ */
if (list_empty(&rq_list)) { if (!list_empty(&rq_list)) {
if (e && e->type->ops.mq.dispatch_requests)
e->type->ops.mq.dispatch_requests(hctx, &rq_list);
else
blk_mq_flush_busy_ctxs(hctx, &rq_list);
} else
blk_mq_sched_mark_restart(hctx); blk_mq_sched_mark_restart(hctx);
blk_mq_dispatch_rq_list(hctx, &rq_list); blk_mq_dispatch_rq_list(hctx, &rq_list);
} else if (!e || !e->type->ops.mq.dispatch_request) {
blk_mq_flush_busy_ctxs(hctx, &rq_list);
blk_mq_dispatch_rq_list(hctx, &rq_list);
} else {
do {
struct request *rq;
rq = e->type->ops.mq.dispatch_request(hctx);
if (!rq)
break;
list_add(&rq->queuelist, &rq_list);
} while (blk_mq_dispatch_rq_list(hctx, &rq_list));
}
} }
void blk_mq_sched_move_to_dispatch(struct blk_mq_hw_ctx *hctx, void blk_mq_sched_move_to_dispatch(struct blk_mq_hw_ctx *hctx,
......
...@@ -998,7 +998,7 @@ bool blk_mq_dispatch_rq_list(struct blk_mq_hw_ctx *hctx, struct list_head *list) ...@@ -998,7 +998,7 @@ bool blk_mq_dispatch_rq_list(struct blk_mq_hw_ctx *hctx, struct list_head *list)
*/ */
if (!list_empty(list)) { if (!list_empty(list)) {
spin_lock(&hctx->lock); spin_lock(&hctx->lock);
list_splice(list, &hctx->dispatch); list_splice_init(list, &hctx->dispatch);
spin_unlock(&hctx->lock); spin_unlock(&hctx->lock);
/* /*
......
...@@ -287,14 +287,16 @@ static struct request *__dd_dispatch_request(struct blk_mq_hw_ctx *hctx) ...@@ -287,14 +287,16 @@ static struct request *__dd_dispatch_request(struct blk_mq_hw_ctx *hctx)
return rq; return rq;
} }
static void dd_dispatch_requests(struct blk_mq_hw_ctx *hctx, static struct request *dd_dispatch_request(struct blk_mq_hw_ctx *hctx)
struct list_head *rq_list)
{ {
struct deadline_data *dd = hctx->queue->elevator->elevator_data; struct deadline_data *dd = hctx->queue->elevator->elevator_data;
struct request *rq;
spin_lock(&dd->lock); spin_lock(&dd->lock);
blk_mq_sched_move_to_dispatch(hctx, rq_list, __dd_dispatch_request); rq = __dd_dispatch_request(hctx);
spin_unlock(&dd->lock); spin_unlock(&dd->lock);
return rq;
} }
static void dd_exit_queue(struct elevator_queue *e) static void dd_exit_queue(struct elevator_queue *e)
...@@ -517,7 +519,7 @@ static struct elv_fs_entry deadline_attrs[] = { ...@@ -517,7 +519,7 @@ static struct elv_fs_entry deadline_attrs[] = {
static struct elevator_type mq_deadline = { static struct elevator_type mq_deadline = {
.ops.mq = { .ops.mq = {
.insert_requests = dd_insert_requests, .insert_requests = dd_insert_requests,
.dispatch_requests = dd_dispatch_requests, .dispatch_request = dd_dispatch_request,
.next_request = elv_rb_latter_request, .next_request = elv_rb_latter_request,
.former_request = elv_rb_former_request, .former_request = elv_rb_former_request,
.bio_merge = dd_bio_merge, .bio_merge = dd_bio_merge,
......
...@@ -92,7 +92,7 @@ struct elevator_mq_ops { ...@@ -92,7 +92,7 @@ struct elevator_mq_ops {
struct request *(*get_request)(struct request_queue *, unsigned int, struct blk_mq_alloc_data *); struct request *(*get_request)(struct request_queue *, unsigned int, struct blk_mq_alloc_data *);
void (*put_request)(struct request *); void (*put_request)(struct request *);
void (*insert_requests)(struct blk_mq_hw_ctx *, struct list_head *, bool); void (*insert_requests)(struct blk_mq_hw_ctx *, struct list_head *, bool);
void (*dispatch_requests)(struct blk_mq_hw_ctx *, struct list_head *); struct request *(*dispatch_request)(struct blk_mq_hw_ctx *);
bool (*has_work)(struct blk_mq_hw_ctx *); bool (*has_work)(struct blk_mq_hw_ctx *);
void (*completed_request)(struct blk_mq_hw_ctx *, struct request *); void (*completed_request)(struct blk_mq_hw_ctx *, struct request *);
void (*started_request)(struct request *); void (*started_request)(struct request *);
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment