Commit 7c20f116 authored by Christoph Hellwig's avatar Christoph Hellwig Committed by Jens Axboe

bio-integrity: stop abusing bi_end_io

And instead call directly into the integrity code from bio_end_io.
Signed-off-by: default avatarChristoph Hellwig <hch@lst.de>
Signed-off-by: default avatarJens Axboe <axboe@kernel.dk>
parent 63573e35
...@@ -102,7 +102,7 @@ EXPORT_SYMBOL(bio_integrity_alloc); ...@@ -102,7 +102,7 @@ EXPORT_SYMBOL(bio_integrity_alloc);
* Description: Used to free the integrity portion of a bio. Usually * Description: Used to free the integrity portion of a bio. Usually
* called from bio_free(). * called from bio_free().
*/ */
void bio_integrity_free(struct bio *bio) static void bio_integrity_free(struct bio *bio)
{ {
struct bio_integrity_payload *bip = bio_integrity(bio); struct bio_integrity_payload *bip = bio_integrity(bio);
struct bio_set *bs = bio->bi_pool; struct bio_set *bs = bio->bi_pool;
...@@ -120,8 +120,8 @@ void bio_integrity_free(struct bio *bio) ...@@ -120,8 +120,8 @@ void bio_integrity_free(struct bio *bio)
} }
bio->bi_integrity = NULL; bio->bi_integrity = NULL;
bio->bi_opf &= ~REQ_INTEGRITY;
} }
EXPORT_SYMBOL(bio_integrity_free);
/** /**
* bio_integrity_add_page - Attach integrity metadata * bio_integrity_add_page - Attach integrity metadata
...@@ -326,12 +326,6 @@ bool bio_integrity_prep(struct bio *bio) ...@@ -326,12 +326,6 @@ bool bio_integrity_prep(struct bio *bio)
offset = 0; offset = 0;
} }
/* Install custom I/O completion handler if read verify is enabled */
if (bio_data_dir(bio) == READ) {
bip->bip_end_io = bio->bi_end_io;
bio->bi_end_io = bio_integrity_endio;
}
/* Auto-generate integrity metadata if this is a write */ /* Auto-generate integrity metadata if this is a write */
if (bio_data_dir(bio) == WRITE) { if (bio_data_dir(bio) == WRITE) {
bio_integrity_process(bio, &bio->bi_iter, bio_integrity_process(bio, &bio->bi_iter,
...@@ -375,13 +369,12 @@ static void bio_integrity_verify_fn(struct work_struct *work) ...@@ -375,13 +369,12 @@ static void bio_integrity_verify_fn(struct work_struct *work)
bio->bi_status = BLK_STS_IOERR; bio->bi_status = BLK_STS_IOERR;
} }
/* Restore original bio completion handler */ bio_integrity_free(bio);
bio->bi_end_io = bip->bip_end_io;
bio_endio(bio); bio_endio(bio);
} }
/** /**
* bio_integrity_endio - Integrity I/O completion function * __bio_integrity_endio - Integrity I/O completion function
* @bio: Protected bio * @bio: Protected bio
* @error: Pointer to errno * @error: Pointer to errno
* *
...@@ -392,27 +385,19 @@ static void bio_integrity_verify_fn(struct work_struct *work) ...@@ -392,27 +385,19 @@ static void bio_integrity_verify_fn(struct work_struct *work)
* in process context. This function postpones completion * in process context. This function postpones completion
* accordingly. * accordingly.
*/ */
void bio_integrity_endio(struct bio *bio) bool __bio_integrity_endio(struct bio *bio)
{ {
struct bio_integrity_payload *bip = bio_integrity(bio); if (bio_op(bio) == REQ_OP_READ && !bio->bi_status) {
struct bio_integrity_payload *bip = bio_integrity(bio);
BUG_ON(bip->bip_bio != bio); INIT_WORK(&bip->bip_work, bio_integrity_verify_fn);
queue_work(kintegrityd_wq, &bip->bip_work);
/* In case of an I/O error there is no point in verifying the return false;
* integrity metadata. Restore original bio end_io handler
* and run it.
*/
if (bio->bi_status) {
bio->bi_end_io = bip->bip_end_io;
bio_endio(bio);
return;
} }
INIT_WORK(&bip->bip_work, bio_integrity_verify_fn); bio_integrity_free(bio);
queue_work(kintegrityd_wq, &bip->bip_work); return true;
} }
EXPORT_SYMBOL(bio_integrity_endio);
/** /**
* bio_integrity_advance - Advance integrity vector * bio_integrity_advance - Advance integrity vector
......
...@@ -243,9 +243,6 @@ struct bio_vec *bvec_alloc(gfp_t gfp_mask, int nr, unsigned long *idx, ...@@ -243,9 +243,6 @@ struct bio_vec *bvec_alloc(gfp_t gfp_mask, int nr, unsigned long *idx,
void bio_uninit(struct bio *bio) void bio_uninit(struct bio *bio)
{ {
bio_disassociate_task(bio); bio_disassociate_task(bio);
if (bio_integrity(bio))
bio_integrity_free(bio);
} }
EXPORT_SYMBOL(bio_uninit); EXPORT_SYMBOL(bio_uninit);
...@@ -1813,6 +1810,8 @@ void bio_endio(struct bio *bio) ...@@ -1813,6 +1810,8 @@ void bio_endio(struct bio *bio)
again: again:
if (!bio_remaining_done(bio)) if (!bio_remaining_done(bio))
return; return;
if (!bio_integrity_endio(bio))
return;
/* /*
* Need to have a real endio function for chained bios, otherwise * Need to have a real endio function for chained bios, otherwise
......
...@@ -81,10 +81,21 @@ static inline void blk_queue_enter_live(struct request_queue *q) ...@@ -81,10 +81,21 @@ static inline void blk_queue_enter_live(struct request_queue *q)
#ifdef CONFIG_BLK_DEV_INTEGRITY #ifdef CONFIG_BLK_DEV_INTEGRITY
void blk_flush_integrity(void); void blk_flush_integrity(void);
bool __bio_integrity_endio(struct bio *);
static inline bool bio_integrity_endio(struct bio *bio)
{
if (bio_integrity(bio))
return __bio_integrity_endio(bio);
return true;
}
#else #else
static inline void blk_flush_integrity(void) static inline void blk_flush_integrity(void)
{ {
} }
static inline bool bio_integrity_endio(struct bio *bio)
{
return true;
}
#endif #endif
void blk_timeout_work(struct work_struct *work); void blk_timeout_work(struct work_struct *work);
......
...@@ -320,8 +320,6 @@ struct bio_integrity_payload { ...@@ -320,8 +320,6 @@ struct bio_integrity_payload {
struct bvec_iter bip_iter; struct bvec_iter bip_iter;
bio_end_io_t *bip_end_io; /* saved I/O completion fn */
unsigned short bip_slab; /* slab the bip came from */ unsigned short bip_slab; /* slab the bip came from */
unsigned short bip_vcnt; /* # of integrity bio_vecs */ unsigned short bip_vcnt; /* # of integrity bio_vecs */
unsigned short bip_max_vcnt; /* integrity bio_vec slots */ unsigned short bip_max_vcnt; /* integrity bio_vec slots */
...@@ -739,10 +737,8 @@ struct biovec_slab { ...@@ -739,10 +737,8 @@ struct biovec_slab {
bip_for_each_vec(_bvl, _bio->bi_integrity, _iter) bip_for_each_vec(_bvl, _bio->bi_integrity, _iter)
extern struct bio_integrity_payload *bio_integrity_alloc(struct bio *, gfp_t, unsigned int); extern struct bio_integrity_payload *bio_integrity_alloc(struct bio *, gfp_t, unsigned int);
extern void bio_integrity_free(struct bio *);
extern int bio_integrity_add_page(struct bio *, struct page *, unsigned int, unsigned int); extern int bio_integrity_add_page(struct bio *, struct page *, unsigned int, unsigned int);
extern bool bio_integrity_prep(struct bio *); extern bool bio_integrity_prep(struct bio *);
extern void bio_integrity_endio(struct bio *);
extern void bio_integrity_advance(struct bio *, unsigned int); extern void bio_integrity_advance(struct bio *, unsigned int);
extern void bio_integrity_trim(struct bio *); extern void bio_integrity_trim(struct bio *);
extern int bio_integrity_clone(struct bio *, struct bio *, gfp_t); extern int bio_integrity_clone(struct bio *, struct bio *, gfp_t);
...@@ -772,11 +768,6 @@ static inline bool bio_integrity_prep(struct bio *bio) ...@@ -772,11 +768,6 @@ static inline bool bio_integrity_prep(struct bio *bio)
return true; return true;
} }
static inline void bio_integrity_free(struct bio *bio)
{
return;
}
static inline int bio_integrity_clone(struct bio *bio, struct bio *bio_src, static inline int bio_integrity_clone(struct bio *bio, struct bio *bio_src,
gfp_t gfp_mask) gfp_t gfp_mask)
{ {
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment