Commit 60ac3a30 authored by Jens Axboe's avatar Jens Axboe

[PATCH] make bio->bi_max contain max vec entries

This changes the way we do pool lookups when freeing a bio.  Right now
we use bi_max as a handle into bvec_array[], to find the pool where it
came from.  This used to be just fine, because if you had a private bio,
you could specify your own destructor.  But now we have bio_add_page()
which also needs to know where the bio came from, or more precisely, it
needs to know how many entries the bio can hold.

So I've changed bi_max to bi_max_vecs, it now contains the number of vec
entries in the bio. Privately allocated bio's (or on stack) can now just
set bio->bi_max_vecs to reflect the max size. The pool index for the
default destructor is stored in the top bits of bi_flags.
parent 79703340
......@@ -61,7 +61,7 @@ static void slab_pool_free(void *ptr, void *data)
kmem_cache_free(data, ptr);
}
static inline struct bio_vec *bvec_alloc(int gfp_mask, int nr, int *idx)
static inline struct bio_vec *bvec_alloc(int gfp_mask, int nr, unsigned long *idx)
{
struct biovec_pool *bp;
struct bio_vec *bvl;
......@@ -95,9 +95,11 @@ static inline struct bio_vec *bvec_alloc(int gfp_mask, int nr, int *idx)
*/
void bio_destructor(struct bio *bio)
{
struct biovec_pool *bp = bvec_array + bio->bi_max;
const int pool_idx = BIO_POOL_IDX(bio);
struct biovec_pool *bp = bvec_array + pool_idx;
BIO_BUG_ON(pool_idx >= BIOVEC_NR_POOLS);
BIO_BUG_ON(bio->bi_max >= BIOVEC_NR_POOLS);
/*
* cloned bio doesn't own the veclist
*/
......@@ -117,6 +119,7 @@ inline void bio_init(struct bio *bio)
bio->bi_phys_segments = 0;
bio->bi_hw_segments = 0;
bio->bi_size = 0;
bio->bi_max_vecs = 0;
bio->bi_end_io = NULL;
atomic_set(&bio->bi_cnt, 1);
}
......@@ -133,27 +136,35 @@ inline void bio_init(struct bio *bio)
**/
struct bio *bio_alloc(int gfp_mask, int nr_iovecs)
{
struct bio *bio;
struct bio_vec *bvl = NULL;
int pf_flags = current->flags;
struct bio_vec *bvl = NULL;
unsigned long idx;
struct bio *bio;
current->flags |= PF_NOWARN;
bio = mempool_alloc(bio_pool, gfp_mask);
if (unlikely(!bio))
goto out;
if (!nr_iovecs || (bvl = bvec_alloc(gfp_mask,nr_iovecs,&bio->bi_max))) {
bio_init(bio);
bio->bi_destructor = bio_destructor;
bio->bi_io_vec = bvl;
if (unlikely(!nr_iovecs))
goto out;
}
mempool_free(bio, bio_pool);
bio = NULL;
bvl = bvec_alloc(gfp_mask, nr_iovecs, &idx);
if (bvl) {
bio->bi_flags |= idx << BIO_POOL_OFFSET;
bio->bi_max_vecs = bvec_array[idx].nr_vecs;
bio->bi_io_vec = bvl;
bio->bi_destructor = bio_destructor;
out:
current->flags = pf_flags;
return bio;
}
mempool_free(bio, bio_pool);
bio = NULL;
goto out;
}
/**
......@@ -212,8 +223,8 @@ inline void __bio_clone(struct bio *bio, struct bio *bio_src)
bio->bi_rw = bio_src->bi_rw;
/*
* notes -- maybe just leave bi_idx alone. bi_max has no use
* on a cloned bio. assume identical mapping for the clone
* notes -- maybe just leave bi_idx alone. assume identical mapping
* for the clone
*/
bio->bi_vcnt = bio_src->bi_vcnt;
bio->bi_idx = bio_src->bi_idx;
......@@ -223,7 +234,14 @@ inline void __bio_clone(struct bio *bio, struct bio *bio_src)
bio->bi_flags |= (1 << BIO_SEG_VALID);
}
bio->bi_size = bio_src->bi_size;
bio->bi_max = bio_src->bi_max;
/*
* cloned bio does not own the bio_vec, so users cannot fiddle with
* it. clear bi_max_vecs and clear the BIO_POOL_BITS to make this
* apparent
*/
bio->bi_max_vecs = 0;
bio->bi_flags &= (BIO_POOL_MASK - 1);
}
/**
......@@ -368,12 +386,7 @@ int bio_add_page(struct bio *bio, struct page *page, unsigned int len,
if (unlikely(bio_flagged(bio, BIO_CLONED)))
return 1;
/*
* FIXME: change bi_max?
*/
BUG_ON(bio->bi_max > BIOVEC_NR_POOLS);
if (bio->bi_vcnt >= bvec_array[bio->bi_max].nr_vecs)
if (bio->bi_vcnt >= bio->bi_max_vecs)
return 1;
if (((bio->bi_size + len) >> 9) > q->max_sectors)
......
......@@ -81,8 +81,7 @@ struct bio {
unsigned short bi_hw_segments;
unsigned int bi_size; /* residual I/O count */
unsigned int bi_max; /* max bvl_vecs we can hold,
used as index into pool */
unsigned int bi_max_vecs; /* max bvl_vecs we can hold */
struct bio_vec *bi_io_vec; /* the actual vec list */
......@@ -104,6 +103,14 @@ struct bio {
#define BIO_CLONED 4 /* doesn't own data */
#define bio_flagged(bio, flag) ((bio)->bi_flags & (1 << (flag)))
/*
* top 4 bits of bio flags indicate the pool this bio came from
*/
#define BIO_POOL_BITS (4)
#define BIO_POOL_OFFSET (BITS_PER_LONG - BIO_POOL_BITS)
#define BIO_POOL_MASK (1UL << BIO_POOL_OFFSET)
#define BIO_POOL_IDX(bio) ((bio)->bi_flags >> BIO_POOL_OFFSET)
/*
* bio bi_rw flags
*
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment