Commit f5fe1b51 authored by NeilBrown's avatar NeilBrown Committed by Jens Axboe

blk: Ensure users for current->bio_list can see the full list.

Commit 79bd9959 ("blk: improve order of bio handling in generic_make_request()")
changed current->bio_list so that it did not contain *all* of the
queued bios, but only those submitted by the currently running
make_request_fn.

There are two places which walk the list and requeue selected bios,
and others that check if the list is empty.  These are no longer
correct.

So redefine current->bio_list to point to an array of two lists, which
contain all queued bios, and adjust various code to test or walk both
lists.
Signed-off-by: default avatarNeilBrown <neilb@suse.com>
Fixes: 79bd9959 ("blk: improve order of bio handling in generic_make_request()")
Signed-off-by: default avatarJens Axboe <axboe@fb.com>
parent 84c37c16
...@@ -376,10 +376,14 @@ static void punt_bios_to_rescuer(struct bio_set *bs) ...@@ -376,10 +376,14 @@ static void punt_bios_to_rescuer(struct bio_set *bs)
bio_list_init(&punt); bio_list_init(&punt);
bio_list_init(&nopunt); bio_list_init(&nopunt);
while ((bio = bio_list_pop(current->bio_list))) while ((bio = bio_list_pop(&current->bio_list[0])))
bio_list_add(bio->bi_pool == bs ? &punt : &nopunt, bio); bio_list_add(bio->bi_pool == bs ? &punt : &nopunt, bio);
current->bio_list[0] = nopunt;
*current->bio_list = nopunt; bio_list_init(&nopunt);
while ((bio = bio_list_pop(&current->bio_list[1])))
bio_list_add(bio->bi_pool == bs ? &punt : &nopunt, bio);
current->bio_list[1] = nopunt;
spin_lock(&bs->rescue_lock); spin_lock(&bs->rescue_lock);
bio_list_merge(&bs->rescue_list, &punt); bio_list_merge(&bs->rescue_list, &punt);
...@@ -466,7 +470,9 @@ struct bio *bio_alloc_bioset(gfp_t gfp_mask, int nr_iovecs, struct bio_set *bs) ...@@ -466,7 +470,9 @@ struct bio *bio_alloc_bioset(gfp_t gfp_mask, int nr_iovecs, struct bio_set *bs)
* we retry with the original gfp_flags. * we retry with the original gfp_flags.
*/ */
if (current->bio_list && !bio_list_empty(current->bio_list)) if (current->bio_list &&
(!bio_list_empty(&current->bio_list[0]) ||
!bio_list_empty(&current->bio_list[1])))
gfp_mask &= ~__GFP_DIRECT_RECLAIM; gfp_mask &= ~__GFP_DIRECT_RECLAIM;
p = mempool_alloc(bs->bio_pool, gfp_mask); p = mempool_alloc(bs->bio_pool, gfp_mask);
......
...@@ -1973,7 +1973,14 @@ generic_make_request_checks(struct bio *bio) ...@@ -1973,7 +1973,14 @@ generic_make_request_checks(struct bio *bio)
*/ */
blk_qc_t generic_make_request(struct bio *bio) blk_qc_t generic_make_request(struct bio *bio)
{ {
struct bio_list bio_list_on_stack; /*
* bio_list_on_stack[0] contains bios submitted by the current
* make_request_fn.
* bio_list_on_stack[1] contains bios that were submitted before
* the current make_request_fn, but that haven't been processed
* yet.
*/
struct bio_list bio_list_on_stack[2];
blk_qc_t ret = BLK_QC_T_NONE; blk_qc_t ret = BLK_QC_T_NONE;
if (!generic_make_request_checks(bio)) if (!generic_make_request_checks(bio))
...@@ -1990,7 +1997,7 @@ blk_qc_t generic_make_request(struct bio *bio) ...@@ -1990,7 +1997,7 @@ blk_qc_t generic_make_request(struct bio *bio)
* should be added at the tail * should be added at the tail
*/ */
if (current->bio_list) { if (current->bio_list) {
bio_list_add(current->bio_list, bio); bio_list_add(&current->bio_list[0], bio);
goto out; goto out;
} }
...@@ -2009,18 +2016,17 @@ blk_qc_t generic_make_request(struct bio *bio) ...@@ -2009,18 +2016,17 @@ blk_qc_t generic_make_request(struct bio *bio)
* bio_list, and call into ->make_request() again. * bio_list, and call into ->make_request() again.
*/ */
BUG_ON(bio->bi_next); BUG_ON(bio->bi_next);
bio_list_init(&bio_list_on_stack); bio_list_init(&bio_list_on_stack[0]);
current->bio_list = &bio_list_on_stack; current->bio_list = bio_list_on_stack;
do { do {
struct request_queue *q = bdev_get_queue(bio->bi_bdev); struct request_queue *q = bdev_get_queue(bio->bi_bdev);
if (likely(blk_queue_enter(q, false) == 0)) { if (likely(blk_queue_enter(q, false) == 0)) {
struct bio_list hold;
struct bio_list lower, same; struct bio_list lower, same;
/* Create a fresh bio_list for all subordinate requests */ /* Create a fresh bio_list for all subordinate requests */
hold = bio_list_on_stack; bio_list_on_stack[1] = bio_list_on_stack[0];
bio_list_init(&bio_list_on_stack); bio_list_init(&bio_list_on_stack[0]);
ret = q->make_request_fn(q, bio); ret = q->make_request_fn(q, bio);
blk_queue_exit(q); blk_queue_exit(q);
...@@ -2030,19 +2036,19 @@ blk_qc_t generic_make_request(struct bio *bio) ...@@ -2030,19 +2036,19 @@ blk_qc_t generic_make_request(struct bio *bio)
*/ */
bio_list_init(&lower); bio_list_init(&lower);
bio_list_init(&same); bio_list_init(&same);
while ((bio = bio_list_pop(&bio_list_on_stack)) != NULL) while ((bio = bio_list_pop(&bio_list_on_stack[0])) != NULL)
if (q == bdev_get_queue(bio->bi_bdev)) if (q == bdev_get_queue(bio->bi_bdev))
bio_list_add(&same, bio); bio_list_add(&same, bio);
else else
bio_list_add(&lower, bio); bio_list_add(&lower, bio);
/* now assemble so we handle the lowest level first */ /* now assemble so we handle the lowest level first */
bio_list_merge(&bio_list_on_stack, &lower); bio_list_merge(&bio_list_on_stack[0], &lower);
bio_list_merge(&bio_list_on_stack, &same); bio_list_merge(&bio_list_on_stack[0], &same);
bio_list_merge(&bio_list_on_stack, &hold); bio_list_merge(&bio_list_on_stack[0], &bio_list_on_stack[1]);
} else { } else {
bio_io_error(bio); bio_io_error(bio);
} }
bio = bio_list_pop(current->bio_list); bio = bio_list_pop(&bio_list_on_stack[0]);
} while (bio); } while (bio);
current->bio_list = NULL; /* deactivate */ current->bio_list = NULL; /* deactivate */
......
...@@ -989,26 +989,29 @@ static void flush_current_bio_list(struct blk_plug_cb *cb, bool from_schedule) ...@@ -989,26 +989,29 @@ static void flush_current_bio_list(struct blk_plug_cb *cb, bool from_schedule)
struct dm_offload *o = container_of(cb, struct dm_offload, cb); struct dm_offload *o = container_of(cb, struct dm_offload, cb);
struct bio_list list; struct bio_list list;
struct bio *bio; struct bio *bio;
int i;
INIT_LIST_HEAD(&o->cb.list); INIT_LIST_HEAD(&o->cb.list);
if (unlikely(!current->bio_list)) if (unlikely(!current->bio_list))
return; return;
list = *current->bio_list; for (i = 0; i < 2; i++) {
bio_list_init(current->bio_list); list = current->bio_list[i];
bio_list_init(&current->bio_list[i]);
while ((bio = bio_list_pop(&list))) {
struct bio_set *bs = bio->bi_pool; while ((bio = bio_list_pop(&list))) {
if (unlikely(!bs) || bs == fs_bio_set) { struct bio_set *bs = bio->bi_pool;
bio_list_add(current->bio_list, bio); if (unlikely(!bs) || bs == fs_bio_set) {
continue; bio_list_add(&current->bio_list[i], bio);
continue;
}
spin_lock(&bs->rescue_lock);
bio_list_add(&bs->rescue_list, bio);
queue_work(bs->rescue_workqueue, &bs->rescue_work);
spin_unlock(&bs->rescue_lock);
} }
spin_lock(&bs->rescue_lock);
bio_list_add(&bs->rescue_list, bio);
queue_work(bs->rescue_workqueue, &bs->rescue_work);
spin_unlock(&bs->rescue_lock);
} }
} }
......
...@@ -974,7 +974,8 @@ static void wait_barrier(struct r10conf *conf) ...@@ -974,7 +974,8 @@ static void wait_barrier(struct r10conf *conf)
!conf->barrier || !conf->barrier ||
(atomic_read(&conf->nr_pending) && (atomic_read(&conf->nr_pending) &&
current->bio_list && current->bio_list &&
!bio_list_empty(current->bio_list)), (!bio_list_empty(&current->bio_list[0]) ||
!bio_list_empty(&current->bio_list[1]))),
conf->resync_lock); conf->resync_lock);
conf->nr_waiting--; conf->nr_waiting--;
if (!conf->nr_waiting) if (!conf->nr_waiting)
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment