[PATCH] make bio->bi_max contain max vec entries

This changes the way we do pool lookups when freeing a bio.  Right now
we use bi_max as a handle into bvec_array[], to find the pool where it
came from.  This used to be just fine, because if you had a private bio,
you could specify your own destructor.  But now we have bio_add_page()
which also needs to know where the bio came from, or more precisely, it
needs to know how many entries the bio can hold.

So I've changed bi_max to bi_max_vecs, it now contains the number of vec
entries in the bio. Privately allocated bio's (or on stack) can now just
set bio->bi_max_vecs to reflect the max size. The pool index for the
default destructor is stored in the top bits of bi_flags.

fs/bio.c

@@ -61,7 +61,7 @@ static void slab_pool_free(void *ptr, void *data)
 	kmem_cache_free(data, ptr);
 }
 
-static inline struct bio_vec *bvec_alloc(int gfp_mask, int nr, int *idx)
+static inline struct bio_vec *bvec_alloc(int gfp_mask, int nr, unsigned long *idx)
 {
 	struct biovec_pool *bp;
 	struct bio_vec *bvl;
@@ -95,9 +95,11 @@ static inline struct bio_vec *bvec_alloc(int gfp_mask, int nr, int *idx)
  */
 void bio_destructor(struct bio *bio)
 {
-	struct biovec_pool *bp = bvec_array + bio->bi_max;
+	const int pool_idx = BIO_POOL_IDX(bio);
+	struct biovec_pool *bp = bvec_array + pool_idx;
+
+	BIO_BUG_ON(pool_idx >= BIOVEC_NR_POOLS);
 
-	BIO_BUG_ON(bio->bi_max >= BIOVEC_NR_POOLS);
 	/*
 	 * cloned bio doesn't own the veclist
 	 */
@@ -117,6 +119,7 @@ inline void bio_init(struct bio *bio)
 	bio->bi_phys_segments = 0;
 	bio->bi_hw_segments = 0;
 	bio->bi_size = 0;
+	bio->bi_max_vecs = 0;
 	bio->bi_end_io = NULL;
 	atomic_set(&bio->bi_cnt, 1);
 }
@@ -133,27 +136,35 @@ inline void bio_init(struct bio *bio)
  **/
 struct bio *bio_alloc(int gfp_mask, int nr_iovecs)
 {
-	struct bio *bio;
-	struct bio_vec *bvl = NULL;
 	int pf_flags = current->flags;
+	struct bio_vec *bvl = NULL;
+	unsigned long idx;
+	struct bio *bio;
 
 	current->flags |= PF_NOWARN;
 	bio = mempool_alloc(bio_pool, gfp_mask);
 	if (unlikely(!bio))
 		goto out;
 
-	if (!nr_iovecs || (bvl = bvec_alloc(gfp_mask,nr_iovecs,&bio->bi_max))) {
 	bio_init(bio);
-		bio->bi_destructor = bio_destructor;
-		bio->bi_io_vec = bvl;
+
+	if (unlikely(!nr_iovecs))
 		goto out;
-	}
 
-	mempool_free(bio, bio_pool);
-	bio = NULL;
+	bvl = bvec_alloc(gfp_mask, nr_iovecs, &idx);
+	if (bvl) {
+		bio->bi_flags |= idx << BIO_POOL_OFFSET;
+		bio->bi_max_vecs = bvec_array[idx].nr_vecs;
+		bio->bi_io_vec = bvl;
+		bio->bi_destructor = bio_destructor;
 out:
 		current->flags = pf_flags;
 		return bio;
+	}
+
+	mempool_free(bio, bio_pool);
+	bio = NULL;
+	goto out;
 }
 
 /**
@@ -212,8 +223,8 @@ inline void __bio_clone(struct bio *bio, struct bio *bio_src)
 	bio->bi_rw = bio_src->bi_rw;
 
 	/*
-	 * notes -- maybe just leave bi_idx alone. bi_max has no use
-	 * on a cloned bio. assume identical mapping for the clone
+	 * notes -- maybe just leave bi_idx alone. assume identical mapping
+	 * for the clone
 	 */
 	bio->bi_vcnt = bio_src->bi_vcnt;
 	bio->bi_idx = bio_src->bi_idx;
@@ -223,7 +234,14 @@ inline void __bio_clone(struct bio *bio, struct bio *bio_src)
 		bio->bi_flags |= (1 << BIO_SEG_VALID);
 	}
 	bio->bi_size = bio_src->bi_size;
-	bio->bi_max = bio_src->bi_max;
+
+	/*
+	 * cloned bio does not own the bio_vec, so users cannot fiddle with
+	 * it. clear bi_max_vecs and clear the BIO_POOL_BITS to make this
+	 * apparent
+	 */
+	bio->bi_max_vecs = 0;
+	bio->bi_flags &= (BIO_POOL_MASK - 1);
 }
 
 /**
@@ -368,12 +386,7 @@ int bio_add_page(struct bio *bio, struct page *page, unsigned int len,
 	if (unlikely(bio_flagged(bio, BIO_CLONED)))
 		return 1;
 
-	/*
-	 * FIXME: change bi_max?
-	 */
-	BUG_ON(bio->bi_max > BIOVEC_NR_POOLS);
-
-	if (bio->bi_vcnt >= bvec_array[bio->bi_max].nr_vecs)
+	if (bio->bi_vcnt >= bio->bi_max_vecs)
 		return 1;
 
 	if (((bio->bi_size + len) >> 9) > q->max_sectors)

include/linux/bio.h

@@ -81,8 +81,7 @@ struct bio {
 	unsigned short		bi_hw_segments;
 
 	unsigned int		bi_size;	/* residual I/O count */
-	unsigned int		bi_max;		/* max bvl_vecs we can hold,
-						   used as index into pool */
+	unsigned int		bi_max_vecs;	/* max bvl_vecs we can hold */
 
 	struct bio_vec		*bi_io_vec;	/* the actual vec list */
 
@@ -104,6 +103,14 @@ struct bio {
 #define BIO_CLONED	4	/* doesn't own data */
 #define bio_flagged(bio, flag)	((bio)->bi_flags & (1 << (flag)))
 
+/*
+ * top 4 bits of bio flags indicate the pool this bio came from
+ */
+#define BIO_POOL_BITS		(4)
+#define BIO_POOL_OFFSET		(BITS_PER_LONG - BIO_POOL_BITS)
+#define BIO_POOL_MASK		(1UL << BIO_POOL_OFFSET)
+#define BIO_POOL_IDX(bio)	((bio)->bi_flags >> BIO_POOL_OFFSET)	
+
 /*
  * bio bi_rw flags
  *