Commit af76e555 authored by Christoph Hellwig's avatar Christoph Hellwig Committed by Jens Axboe

blk-mq: initialize struct request fields individually

This allows us to avoid a non-atomic memset over ->atomic_flags as well
as killing lots of duplicate initializations.
Signed-off-by: default avatarChristoph Hellwig <hch@lst.de>
Signed-off-by: default avatarJens Axboe <axboe@fb.com>
parent 9fccfed8
...@@ -82,9 +82,7 @@ static struct request *__blk_mq_alloc_request(struct blk_mq_hw_ctx *hctx, ...@@ -82,9 +82,7 @@ static struct request *__blk_mq_alloc_request(struct blk_mq_hw_ctx *hctx,
tag = blk_mq_get_tag(hctx->tags, gfp, reserved); tag = blk_mq_get_tag(hctx->tags, gfp, reserved);
if (tag != BLK_MQ_TAG_FAIL) { if (tag != BLK_MQ_TAG_FAIL) {
rq = hctx->tags->rqs[tag]; rq = hctx->tags->rqs[tag];
blk_rq_init(hctx->queue, rq);
rq->tag = tag; rq->tag = tag;
return rq; return rq;
} }
...@@ -187,10 +185,54 @@ static void blk_mq_rq_ctx_init(struct request_queue *q, struct blk_mq_ctx *ctx, ...@@ -187,10 +185,54 @@ static void blk_mq_rq_ctx_init(struct request_queue *q, struct blk_mq_ctx *ctx,
if (blk_queue_io_stat(q)) if (blk_queue_io_stat(q))
rw_flags |= REQ_IO_STAT; rw_flags |= REQ_IO_STAT;
INIT_LIST_HEAD(&rq->queuelist);
/* csd/requeue_work/fifo_time is initialized before use */
rq->q = q;
rq->mq_ctx = ctx; rq->mq_ctx = ctx;
rq->cmd_flags = rw_flags; rq->cmd_flags = rw_flags;
rq->cmd_type = 0;
/* do not touch atomic flags, it needs atomic ops against the timer */
rq->cpu = -1;
rq->__data_len = 0;
rq->__sector = (sector_t) -1;
rq->bio = NULL;
rq->biotail = NULL;
INIT_HLIST_NODE(&rq->hash);
RB_CLEAR_NODE(&rq->rb_node);
memset(&rq->flush, 0, max(sizeof(rq->flush), sizeof(rq->elv)));
rq->rq_disk = NULL;
rq->part = NULL;
rq->start_time = jiffies; rq->start_time = jiffies;
#ifdef CONFIG_BLK_CGROUP
rq->rl = NULL;
set_start_time_ns(rq); set_start_time_ns(rq);
rq->io_start_time_ns = 0;
#endif
rq->nr_phys_segments = 0;
#if defined(CONFIG_BLK_DEV_INTEGRITY)
rq->nr_integrity_segments = 0;
#endif
rq->ioprio = 0;
rq->special = NULL;
/* tag was already set */
rq->errors = 0;
memset(rq->__cmd, 0, sizeof(rq->__cmd));
rq->cmd = rq->__cmd;
rq->cmd_len = BLK_MAX_CDB;
rq->extra_len = 0;
rq->sense_len = 0;
rq->resid_len = 0;
rq->sense = NULL;
rq->deadline = 0;
INIT_LIST_HEAD(&rq->timeout_list);
rq->timeout = 0;
rq->retries = 0;
rq->end_io = NULL;
rq->end_io_data = NULL;
rq->next_rq = NULL;
ctx->rq_dispatched[rw_is_sync(rw_flags)]++; ctx->rq_dispatched[rw_is_sync(rw_flags)]++;
} }
...@@ -258,6 +300,7 @@ static void __blk_mq_free_request(struct blk_mq_hw_ctx *hctx, ...@@ -258,6 +300,7 @@ static void __blk_mq_free_request(struct blk_mq_hw_ctx *hctx,
const int tag = rq->tag; const int tag = rq->tag;
struct request_queue *q = rq->q; struct request_queue *q = rq->q;
clear_bit(REQ_ATOM_STARTED, &rq->atomic_flags);
blk_mq_put_tag(hctx->tags, tag); blk_mq_put_tag(hctx->tags, tag);
blk_mq_queue_exit(q); blk_mq_queue_exit(q);
} }
......
...@@ -90,9 +90,10 @@ enum rq_cmd_type_bits { ...@@ -90,9 +90,10 @@ enum rq_cmd_type_bits {
#define BLK_MAX_CDB 16 #define BLK_MAX_CDB 16
/* /*
* try to put the fields that are referenced together in the same cacheline. * Try to put the fields that are referenced together in the same cacheline.
* if you modify this structure, be sure to check block/blk-core.c:blk_rq_init() *
* as well! * If you modify this structure, make sure to update blk_rq_init() and
* especially blk_mq_rq_ctx_init() to take care of the added fields.
*/ */
struct request { struct request {
struct list_head queuelist; struct list_head queuelist;
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment