Commit b77c88c2 authored by Christoph Hellwig's avatar Christoph Hellwig Committed by Jens Axboe

block: pass a block_device and opf to bio_alloc_kiocb

Pass the block_device and operation that we plan to use this bio for to
bio_alloc_kiocb to optimize the assigment.
Signed-off-by: default avatarChristoph Hellwig <hch@lst.de>
Reviewed-by: default avatarChaitanya Kulkarni <kch@nvidia.com>
Link: https://lore.kernel.org/r/20220124091107.642561-17-hch@lst.deSigned-off-by: default avatarJens Axboe <axboe@kernel.dk>
parent 609be106
...@@ -1731,7 +1731,9 @@ EXPORT_SYMBOL(bioset_init_from_src); ...@@ -1731,7 +1731,9 @@ EXPORT_SYMBOL(bioset_init_from_src);
/** /**
* bio_alloc_kiocb - Allocate a bio from bio_set based on kiocb * bio_alloc_kiocb - Allocate a bio from bio_set based on kiocb
* @kiocb: kiocb describing the IO * @kiocb: kiocb describing the IO
* @bdev: block device to allocate the bio for (can be %NULL)
* @nr_vecs: number of iovecs to pre-allocate * @nr_vecs: number of iovecs to pre-allocate
* @opf: operation and flags for bio
* @bs: bio_set to allocate from * @bs: bio_set to allocate from
* *
* Description: * Description:
...@@ -1742,14 +1744,14 @@ EXPORT_SYMBOL(bioset_init_from_src); ...@@ -1742,14 +1744,14 @@ EXPORT_SYMBOL(bioset_init_from_src);
* MUST be done from process context, not hard/soft IRQ. * MUST be done from process context, not hard/soft IRQ.
* *
*/ */
struct bio *bio_alloc_kiocb(struct kiocb *kiocb, unsigned short nr_vecs, struct bio *bio_alloc_kiocb(struct kiocb *kiocb, struct block_device *bdev,
struct bio_set *bs) unsigned short nr_vecs, unsigned int opf, struct bio_set *bs)
{ {
struct bio_alloc_cache *cache; struct bio_alloc_cache *cache;
struct bio *bio; struct bio *bio;
if (!(kiocb->ki_flags & IOCB_ALLOC_CACHE) || nr_vecs > BIO_INLINE_VECS) if (!(kiocb->ki_flags & IOCB_ALLOC_CACHE) || nr_vecs > BIO_INLINE_VECS)
return bio_alloc_bioset(NULL, nr_vecs, 0, GFP_KERNEL, bs); return bio_alloc_bioset(bdev, nr_vecs, opf, GFP_KERNEL, bs);
cache = per_cpu_ptr(bs->cache, get_cpu()); cache = per_cpu_ptr(bs->cache, get_cpu());
if (cache->free_list) { if (cache->free_list) {
...@@ -1758,12 +1760,14 @@ struct bio *bio_alloc_kiocb(struct kiocb *kiocb, unsigned short nr_vecs, ...@@ -1758,12 +1760,14 @@ struct bio *bio_alloc_kiocb(struct kiocb *kiocb, unsigned short nr_vecs,
cache->nr--; cache->nr--;
put_cpu(); put_cpu();
bio_init(bio, nr_vecs ? bio->bi_inline_vecs : NULL, nr_vecs); bio_init(bio, nr_vecs ? bio->bi_inline_vecs : NULL, nr_vecs);
bio_set_dev(bio, bdev);
bio->bi_opf = opf;
bio->bi_pool = bs; bio->bi_pool = bs;
bio_set_flag(bio, BIO_PERCPU_CACHE); bio_set_flag(bio, BIO_PERCPU_CACHE);
return bio; return bio;
} }
put_cpu(); put_cpu();
bio = bio_alloc_bioset(NULL, nr_vecs, 0, GFP_KERNEL, bs); bio = bio_alloc_bioset(bdev, nr_vecs, opf, GFP_KERNEL, bs);
bio_set_flag(bio, BIO_PERCPU_CACHE); bio_set_flag(bio, BIO_PERCPU_CACHE);
return bio; return bio;
} }
......
...@@ -190,6 +190,7 @@ static ssize_t __blkdev_direct_IO(struct kiocb *iocb, struct iov_iter *iter, ...@@ -190,6 +190,7 @@ static ssize_t __blkdev_direct_IO(struct kiocb *iocb, struct iov_iter *iter,
struct blkdev_dio *dio; struct blkdev_dio *dio;
struct bio *bio; struct bio *bio;
bool is_read = (iov_iter_rw(iter) == READ), is_sync; bool is_read = (iov_iter_rw(iter) == READ), is_sync;
unsigned int opf = is_read ? REQ_OP_READ : dio_bio_write_op(iocb);
loff_t pos = iocb->ki_pos; loff_t pos = iocb->ki_pos;
int ret = 0; int ret = 0;
...@@ -197,7 +198,7 @@ static ssize_t __blkdev_direct_IO(struct kiocb *iocb, struct iov_iter *iter, ...@@ -197,7 +198,7 @@ static ssize_t __blkdev_direct_IO(struct kiocb *iocb, struct iov_iter *iter,
(bdev_logical_block_size(bdev) - 1)) (bdev_logical_block_size(bdev) - 1))
return -EINVAL; return -EINVAL;
bio = bio_alloc_kiocb(iocb, nr_pages, &blkdev_dio_pool); bio = bio_alloc_kiocb(iocb, bdev, nr_pages, opf, &blkdev_dio_pool);
dio = container_of(bio, struct blkdev_dio, bio); dio = container_of(bio, struct blkdev_dio, bio);
atomic_set(&dio->ref, 1); atomic_set(&dio->ref, 1);
...@@ -223,7 +224,6 @@ static ssize_t __blkdev_direct_IO(struct kiocb *iocb, struct iov_iter *iter, ...@@ -223,7 +224,6 @@ static ssize_t __blkdev_direct_IO(struct kiocb *iocb, struct iov_iter *iter,
blk_start_plug(&plug); blk_start_plug(&plug);
for (;;) { for (;;) {
bio_set_dev(bio, bdev);
bio->bi_iter.bi_sector = pos >> SECTOR_SHIFT; bio->bi_iter.bi_sector = pos >> SECTOR_SHIFT;
bio->bi_write_hint = iocb->ki_hint; bio->bi_write_hint = iocb->ki_hint;
bio->bi_private = dio; bio->bi_private = dio;
...@@ -238,11 +238,9 @@ static ssize_t __blkdev_direct_IO(struct kiocb *iocb, struct iov_iter *iter, ...@@ -238,11 +238,9 @@ static ssize_t __blkdev_direct_IO(struct kiocb *iocb, struct iov_iter *iter,
} }
if (is_read) { if (is_read) {
bio->bi_opf = REQ_OP_READ;
if (dio->flags & DIO_SHOULD_DIRTY) if (dio->flags & DIO_SHOULD_DIRTY)
bio_set_pages_dirty(bio); bio_set_pages_dirty(bio);
} else { } else {
bio->bi_opf = dio_bio_write_op(iocb);
task_io_account_write(bio->bi_iter.bi_size); task_io_account_write(bio->bi_iter.bi_size);
} }
if (iocb->ki_flags & IOCB_NOWAIT) if (iocb->ki_flags & IOCB_NOWAIT)
...@@ -259,6 +257,8 @@ static ssize_t __blkdev_direct_IO(struct kiocb *iocb, struct iov_iter *iter, ...@@ -259,6 +257,8 @@ static ssize_t __blkdev_direct_IO(struct kiocb *iocb, struct iov_iter *iter,
atomic_inc(&dio->ref); atomic_inc(&dio->ref);
submit_bio(bio); submit_bio(bio);
bio = bio_alloc(GFP_KERNEL, nr_pages); bio = bio_alloc(GFP_KERNEL, nr_pages);
bio_set_dev(bio, bdev);
bio->bi_opf = opf;
} }
blk_finish_plug(&plug); blk_finish_plug(&plug);
...@@ -311,6 +311,8 @@ static ssize_t __blkdev_direct_IO_async(struct kiocb *iocb, ...@@ -311,6 +311,8 @@ static ssize_t __blkdev_direct_IO_async(struct kiocb *iocb,
unsigned int nr_pages) unsigned int nr_pages)
{ {
struct block_device *bdev = iocb->ki_filp->private_data; struct block_device *bdev = iocb->ki_filp->private_data;
bool is_read = iov_iter_rw(iter) == READ;
unsigned int opf = is_read ? REQ_OP_READ : dio_bio_write_op(iocb);
struct blkdev_dio *dio; struct blkdev_dio *dio;
struct bio *bio; struct bio *bio;
loff_t pos = iocb->ki_pos; loff_t pos = iocb->ki_pos;
...@@ -320,11 +322,10 @@ static ssize_t __blkdev_direct_IO_async(struct kiocb *iocb, ...@@ -320,11 +322,10 @@ static ssize_t __blkdev_direct_IO_async(struct kiocb *iocb,
(bdev_logical_block_size(bdev) - 1)) (bdev_logical_block_size(bdev) - 1))
return -EINVAL; return -EINVAL;
bio = bio_alloc_kiocb(iocb, nr_pages, &blkdev_dio_pool); bio = bio_alloc_kiocb(iocb, bdev, nr_pages, opf, &blkdev_dio_pool);
dio = container_of(bio, struct blkdev_dio, bio); dio = container_of(bio, struct blkdev_dio, bio);
dio->flags = 0; dio->flags = 0;
dio->iocb = iocb; dio->iocb = iocb;
bio_set_dev(bio, bdev);
bio->bi_iter.bi_sector = pos >> SECTOR_SHIFT; bio->bi_iter.bi_sector = pos >> SECTOR_SHIFT;
bio->bi_write_hint = iocb->ki_hint; bio->bi_write_hint = iocb->ki_hint;
bio->bi_end_io = blkdev_bio_end_io_async; bio->bi_end_io = blkdev_bio_end_io_async;
...@@ -347,14 +348,12 @@ static ssize_t __blkdev_direct_IO_async(struct kiocb *iocb, ...@@ -347,14 +348,12 @@ static ssize_t __blkdev_direct_IO_async(struct kiocb *iocb,
} }
dio->size = bio->bi_iter.bi_size; dio->size = bio->bi_iter.bi_size;
if (iov_iter_rw(iter) == READ) { if (is_read) {
bio->bi_opf = REQ_OP_READ;
if (iter_is_iovec(iter)) { if (iter_is_iovec(iter)) {
dio->flags |= DIO_SHOULD_DIRTY; dio->flags |= DIO_SHOULD_DIRTY;
bio_set_pages_dirty(bio); bio_set_pages_dirty(bio);
} }
} else { } else {
bio->bi_opf = dio_bio_write_op(iocb);
task_io_account_write(bio->bi_iter.bi_size); task_io_account_write(bio->bi_iter.bi_size);
} }
......
...@@ -408,8 +408,8 @@ extern int bioset_init_from_src(struct bio_set *bs, struct bio_set *src); ...@@ -408,8 +408,8 @@ extern int bioset_init_from_src(struct bio_set *bs, struct bio_set *src);
struct bio *bio_alloc_bioset(struct block_device *bdev, unsigned short nr_vecs, struct bio *bio_alloc_bioset(struct block_device *bdev, unsigned short nr_vecs,
unsigned int opf, gfp_t gfp_mask, unsigned int opf, gfp_t gfp_mask,
struct bio_set *bs); struct bio_set *bs);
struct bio *bio_alloc_kiocb(struct kiocb *kiocb, unsigned short nr_vecs, struct bio *bio_alloc_kiocb(struct kiocb *kiocb, struct block_device *bdev,
struct bio_set *bs); unsigned short nr_vecs, unsigned int opf, struct bio_set *bs);
struct bio *bio_kmalloc(gfp_t gfp_mask, unsigned short nr_iovecs); struct bio *bio_kmalloc(gfp_t gfp_mask, unsigned short nr_iovecs);
extern void bio_put(struct bio *); extern void bio_put(struct bio *);
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment