Commit 720b8ccc authored by Stephen Bates's avatar Stephen Bates Committed by Jens Axboe

blk-mq: Add a polling specific stats function

Rather than bucketing IO statisics based on direction only we also
bucket based on the IO size. This leads to improved polling
performance. Update the bucket callback function and use it in the
polling latency estimation.
Signed-off-by: default avatarStephen Bates <sbates@raithlin.com>
Signed-off-by: default avatarJens Axboe <axboe@fb.com>
parent a37244e4
...@@ -42,6 +42,25 @@ static LIST_HEAD(all_q_list); ...@@ -42,6 +42,25 @@ static LIST_HEAD(all_q_list);
static void blk_mq_poll_stats_start(struct request_queue *q); static void blk_mq_poll_stats_start(struct request_queue *q);
static void blk_mq_poll_stats_fn(struct blk_stat_callback *cb); static void blk_mq_poll_stats_fn(struct blk_stat_callback *cb);
/* Must be consisitent with function below */
#define BLK_MQ_POLL_STATS_BKTS 16
static int blk_mq_poll_stats_bkt(const struct request *rq)
{
int ddir, bytes, bucket;
ddir = blk_stat_rq_ddir(rq);
bytes = blk_rq_bytes(rq);
bucket = ddir + 2*(ilog2(bytes) - 9);
if (bucket < 0)
return -1;
else if (bucket >= BLK_MQ_POLL_STATS_BKTS)
return ddir + BLK_MQ_POLL_STATS_BKTS - 2;
return bucket;
}
/* /*
* Check if any of the ctx's have pending work in this hardware queue * Check if any of the ctx's have pending work in this hardware queue
*/ */
...@@ -2257,7 +2276,8 @@ struct request_queue *blk_mq_init_allocated_queue(struct blk_mq_tag_set *set, ...@@ -2257,7 +2276,8 @@ struct request_queue *blk_mq_init_allocated_queue(struct blk_mq_tag_set *set,
q->mq_ops = set->ops; q->mq_ops = set->ops;
q->poll_cb = blk_stat_alloc_callback(blk_mq_poll_stats_fn, q->poll_cb = blk_stat_alloc_callback(blk_mq_poll_stats_fn,
blk_stat_rq_ddir, 2, q); blk_mq_poll_stats_bkt,
BLK_MQ_POLL_STATS_BKTS, q);
if (!q->poll_cb) if (!q->poll_cb)
goto err_exit; goto err_exit;
...@@ -2683,11 +2703,12 @@ static void blk_mq_poll_stats_start(struct request_queue *q) ...@@ -2683,11 +2703,12 @@ static void blk_mq_poll_stats_start(struct request_queue *q)
static void blk_mq_poll_stats_fn(struct blk_stat_callback *cb) static void blk_mq_poll_stats_fn(struct blk_stat_callback *cb)
{ {
struct request_queue *q = cb->data; struct request_queue *q = cb->data;
int bucket;
if (cb->stat[READ].nr_samples) for (bucket = 0; bucket < BLK_MQ_POLL_STATS_BKTS; bucket++) {
q->poll_stat[READ] = cb->stat[READ]; if (cb->stat[bucket].nr_samples)
if (cb->stat[WRITE].nr_samples) q->poll_stat[bucket] = cb->stat[bucket];
q->poll_stat[WRITE] = cb->stat[WRITE]; }
} }
static unsigned long blk_mq_poll_nsecs(struct request_queue *q, static unsigned long blk_mq_poll_nsecs(struct request_queue *q,
...@@ -2695,6 +2716,7 @@ static unsigned long blk_mq_poll_nsecs(struct request_queue *q, ...@@ -2695,6 +2716,7 @@ static unsigned long blk_mq_poll_nsecs(struct request_queue *q,
struct request *rq) struct request *rq)
{ {
unsigned long ret = 0; unsigned long ret = 0;
int bucket;
/* /*
* If stats collection isn't on, don't sleep but turn it on for * If stats collection isn't on, don't sleep but turn it on for
...@@ -2709,12 +2731,15 @@ static unsigned long blk_mq_poll_nsecs(struct request_queue *q, ...@@ -2709,12 +2731,15 @@ static unsigned long blk_mq_poll_nsecs(struct request_queue *q,
* For instance, if the completion latencies are tight, we can * For instance, if the completion latencies are tight, we can
* get closer than just half the mean. This is especially * get closer than just half the mean. This is especially
* important on devices where the completion latencies are longer * important on devices where the completion latencies are longer
* than ~10 usec. * than ~10 usec. We do use the stats for the relevant IO size
* if available which does lead to better estimates.
*/ */
if (req_op(rq) == REQ_OP_READ && q->poll_stat[READ].nr_samples) bucket = blk_mq_poll_stats_bkt(rq);
ret = (q->poll_stat[READ].mean + 1) / 2; if (bucket < 0)
else if (req_op(rq) == REQ_OP_WRITE && q->poll_stat[WRITE].nr_samples) return ret;
ret = (q->poll_stat[WRITE].mean + 1) / 2;
if (q->poll_stat[bucket].nr_samples)
ret = (q->poll_stat[bucket].mean + 1) / 2;
return ret; return ret;
} }
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment