Commit dff4d1f6 authored by Linus Torvalds's avatar Linus Torvalds

Merge tag 'for-4.14/dm-changes' of...

Merge tag 'for-4.14/dm-changes' of git://git.kernel.org/pub/scm/linux/kernel/git/device-mapper/linux-dm

Pull device mapper updates from Mike Snitzer:

 - Some request-based DM core and DM multipath fixes and cleanups

 - Constify a few variables in DM core and DM integrity

 - Add bufio optimization and checksum failure accounting to DM
   integrity

 - Fix DM integrity to avoid checking integrity of failed reads

 - Fix DM integrity to use init_completion

 - A couple DM log-writes target fixes

 - Simplify DAX flushing by eliminating the unnecessary flush
   abstraction that was stood up for DM's use.

* tag 'for-4.14/dm-changes' of git://git.kernel.org/pub/scm/linux/kernel/git/device-mapper/linux-dm:
  dax: remove the pmem_dax_ops->flush abstraction
  dm integrity: use init_completion instead of COMPLETION_INITIALIZER_ONSTACK
  dm integrity: make blk_integrity_profile structure const
  dm integrity: do not check integrity for failed read operations
  dm log writes: fix >512b sectorsize support
  dm log writes: don't use all the cpu while waiting to log blocks
  dm ioctl: constify ioctl lookup table
  dm: constify argument arrays
  dm integrity: count and display checksum failures
  dm integrity: optimize writing dm-bufio buffers that are partially changed
  dm rq: do not update rq partially in each ending bio
  dm rq: make dm-sq requeuing behavior consistent with dm-mq behavior
  dm mpath: complain about unsupported __multipath_map_bio() return values
  dm mpath: avoid that building with W=1 causes gcc 7 to complain about fall-through
parents 503f0453 c3ca015f
...@@ -201,8 +201,10 @@ static umode_t dax_visible(struct kobject *kobj, struct attribute *a, int n) ...@@ -201,8 +201,10 @@ static umode_t dax_visible(struct kobject *kobj, struct attribute *a, int n)
if (!dax_dev) if (!dax_dev)
return 0; return 0;
if (a == &dev_attr_write_cache.attr && !dax_dev->ops->flush) #ifndef CONFIG_ARCH_HAS_PMEM_API
if (a == &dev_attr_write_cache.attr)
return 0; return 0;
#endif
return a->mode; return a->mode;
} }
...@@ -267,18 +269,23 @@ size_t dax_copy_from_iter(struct dax_device *dax_dev, pgoff_t pgoff, void *addr, ...@@ -267,18 +269,23 @@ size_t dax_copy_from_iter(struct dax_device *dax_dev, pgoff_t pgoff, void *addr,
} }
EXPORT_SYMBOL_GPL(dax_copy_from_iter); EXPORT_SYMBOL_GPL(dax_copy_from_iter);
void dax_flush(struct dax_device *dax_dev, pgoff_t pgoff, void *addr, #ifdef CONFIG_ARCH_HAS_PMEM_API
size_t size) void arch_wb_cache_pmem(void *addr, size_t size);
void dax_flush(struct dax_device *dax_dev, void *addr, size_t size)
{ {
if (!dax_alive(dax_dev)) if (unlikely(!dax_alive(dax_dev)))
return; return;
if (!test_bit(DAXDEV_WRITE_CACHE, &dax_dev->flags)) if (unlikely(!test_bit(DAXDEV_WRITE_CACHE, &dax_dev->flags)))
return; return;
if (dax_dev->ops->flush) arch_wb_cache_pmem(addr, size);
dax_dev->ops->flush(dax_dev, pgoff, addr, size);
} }
#else
void dax_flush(struct dax_device *dax_dev, void *addr, size_t size)
{
}
#endif
EXPORT_SYMBOL_GPL(dax_flush); EXPORT_SYMBOL_GPL(dax_flush);
void dax_write_cache(struct dax_device *dax_dev, bool wc) void dax_write_cache(struct dax_device *dax_dev, bool wc)
......
...@@ -63,6 +63,12 @@ ...@@ -63,6 +63,12 @@
#define DM_BUFIO_BLOCK_SIZE_SLAB_LIMIT (PAGE_SIZE >> 1) #define DM_BUFIO_BLOCK_SIZE_SLAB_LIMIT (PAGE_SIZE >> 1)
#define DM_BUFIO_BLOCK_SIZE_GFP_LIMIT (PAGE_SIZE << (MAX_ORDER - 1)) #define DM_BUFIO_BLOCK_SIZE_GFP_LIMIT (PAGE_SIZE << (MAX_ORDER - 1))
/*
* Align buffer writes to this boundary.
* Tests show that SSDs have the highest IOPS when using 4k writes.
*/
#define DM_BUFIO_WRITE_ALIGN 4096
/* /*
* dm_buffer->list_mode * dm_buffer->list_mode
*/ */
...@@ -149,6 +155,10 @@ struct dm_buffer { ...@@ -149,6 +155,10 @@ struct dm_buffer {
blk_status_t write_error; blk_status_t write_error;
unsigned long state; unsigned long state;
unsigned long last_accessed; unsigned long last_accessed;
unsigned dirty_start;
unsigned dirty_end;
unsigned write_start;
unsigned write_end;
struct dm_bufio_client *c; struct dm_bufio_client *c;
struct list_head write_list; struct list_head write_list;
struct bio bio; struct bio bio;
...@@ -560,7 +570,7 @@ static void dmio_complete(unsigned long error, void *context) ...@@ -560,7 +570,7 @@ static void dmio_complete(unsigned long error, void *context)
} }
static void use_dmio(struct dm_buffer *b, int rw, sector_t sector, static void use_dmio(struct dm_buffer *b, int rw, sector_t sector,
unsigned n_sectors, bio_end_io_t *end_io) unsigned n_sectors, unsigned offset, bio_end_io_t *end_io)
{ {
int r; int r;
struct dm_io_request io_req = { struct dm_io_request io_req = {
...@@ -578,10 +588,10 @@ static void use_dmio(struct dm_buffer *b, int rw, sector_t sector, ...@@ -578,10 +588,10 @@ static void use_dmio(struct dm_buffer *b, int rw, sector_t sector,
if (b->data_mode != DATA_MODE_VMALLOC) { if (b->data_mode != DATA_MODE_VMALLOC) {
io_req.mem.type = DM_IO_KMEM; io_req.mem.type = DM_IO_KMEM;
io_req.mem.ptr.addr = b->data; io_req.mem.ptr.addr = (char *)b->data + offset;
} else { } else {
io_req.mem.type = DM_IO_VMA; io_req.mem.type = DM_IO_VMA;
io_req.mem.ptr.vma = b->data; io_req.mem.ptr.vma = (char *)b->data + offset;
} }
b->bio.bi_end_io = end_io; b->bio.bi_end_io = end_io;
...@@ -609,10 +619,10 @@ static void inline_endio(struct bio *bio) ...@@ -609,10 +619,10 @@ static void inline_endio(struct bio *bio)
} }
static void use_inline_bio(struct dm_buffer *b, int rw, sector_t sector, static void use_inline_bio(struct dm_buffer *b, int rw, sector_t sector,
unsigned n_sectors, bio_end_io_t *end_io) unsigned n_sectors, unsigned offset, bio_end_io_t *end_io)
{ {
char *ptr; char *ptr;
int len; unsigned len;
bio_init(&b->bio, b->bio_vec, DM_BUFIO_INLINE_VECS); bio_init(&b->bio, b->bio_vec, DM_BUFIO_INLINE_VECS);
b->bio.bi_iter.bi_sector = sector; b->bio.bi_iter.bi_sector = sector;
...@@ -625,29 +635,20 @@ static void use_inline_bio(struct dm_buffer *b, int rw, sector_t sector, ...@@ -625,29 +635,20 @@ static void use_inline_bio(struct dm_buffer *b, int rw, sector_t sector,
b->bio.bi_private = end_io; b->bio.bi_private = end_io;
bio_set_op_attrs(&b->bio, rw, 0); bio_set_op_attrs(&b->bio, rw, 0);
/* ptr = (char *)b->data + offset;
* We assume that if len >= PAGE_SIZE ptr is page-aligned.
* If len < PAGE_SIZE the buffer doesn't cross page boundary.
*/
ptr = b->data;
len = n_sectors << SECTOR_SHIFT; len = n_sectors << SECTOR_SHIFT;
if (len >= PAGE_SIZE)
BUG_ON((unsigned long)ptr & (PAGE_SIZE - 1));
else
BUG_ON((unsigned long)ptr & (len - 1));
do { do {
if (!bio_add_page(&b->bio, virt_to_page(ptr), unsigned this_step = min((unsigned)(PAGE_SIZE - offset_in_page(ptr)), len);
len < PAGE_SIZE ? len : PAGE_SIZE, if (!bio_add_page(&b->bio, virt_to_page(ptr), this_step,
offset_in_page(ptr))) { offset_in_page(ptr))) {
BUG_ON(b->c->block_size <= PAGE_SIZE); BUG_ON(b->c->block_size <= PAGE_SIZE);
use_dmio(b, rw, sector, n_sectors, end_io); use_dmio(b, rw, sector, n_sectors, offset, end_io);
return; return;
} }
len -= PAGE_SIZE; len -= this_step;
ptr += PAGE_SIZE; ptr += this_step;
} while (len > 0); } while (len > 0);
submit_bio(&b->bio); submit_bio(&b->bio);
...@@ -657,18 +658,33 @@ static void submit_io(struct dm_buffer *b, int rw, bio_end_io_t *end_io) ...@@ -657,18 +658,33 @@ static void submit_io(struct dm_buffer *b, int rw, bio_end_io_t *end_io)
{ {
unsigned n_sectors; unsigned n_sectors;
sector_t sector; sector_t sector;
unsigned offset, end;
if (rw == WRITE && b->c->write_callback)
b->c->write_callback(b);
sector = (b->block << b->c->sectors_per_block_bits) + b->c->start; sector = (b->block << b->c->sectors_per_block_bits) + b->c->start;
if (rw != WRITE) {
n_sectors = 1 << b->c->sectors_per_block_bits; n_sectors = 1 << b->c->sectors_per_block_bits;
offset = 0;
} else {
if (b->c->write_callback)
b->c->write_callback(b);
offset = b->write_start;
end = b->write_end;
offset &= -DM_BUFIO_WRITE_ALIGN;
end += DM_BUFIO_WRITE_ALIGN - 1;
end &= -DM_BUFIO_WRITE_ALIGN;
if (unlikely(end > b->c->block_size))
end = b->c->block_size;
sector += offset >> SECTOR_SHIFT;
n_sectors = (end - offset) >> SECTOR_SHIFT;
}
if (n_sectors <= ((DM_BUFIO_INLINE_VECS * PAGE_SIZE) >> SECTOR_SHIFT) && if (n_sectors <= ((DM_BUFIO_INLINE_VECS * PAGE_SIZE) >> SECTOR_SHIFT) &&
b->data_mode != DATA_MODE_VMALLOC) b->data_mode != DATA_MODE_VMALLOC)
use_inline_bio(b, rw, sector, n_sectors, end_io); use_inline_bio(b, rw, sector, n_sectors, offset, end_io);
else else
use_dmio(b, rw, sector, n_sectors, end_io); use_dmio(b, rw, sector, n_sectors, offset, end_io);
} }
/*---------------------------------------------------------------- /*----------------------------------------------------------------
...@@ -720,6 +736,9 @@ static void __write_dirty_buffer(struct dm_buffer *b, ...@@ -720,6 +736,9 @@ static void __write_dirty_buffer(struct dm_buffer *b,
clear_bit(B_DIRTY, &b->state); clear_bit(B_DIRTY, &b->state);
wait_on_bit_lock_io(&b->state, B_WRITING, TASK_UNINTERRUPTIBLE); wait_on_bit_lock_io(&b->state, B_WRITING, TASK_UNINTERRUPTIBLE);
b->write_start = b->dirty_start;
b->write_end = b->dirty_end;
if (!write_list) if (!write_list)
submit_io(b, WRITE, write_endio); submit_io(b, WRITE, write_endio);
else else
...@@ -1221,19 +1240,37 @@ void dm_bufio_release(struct dm_buffer *b) ...@@ -1221,19 +1240,37 @@ void dm_bufio_release(struct dm_buffer *b)
} }
EXPORT_SYMBOL_GPL(dm_bufio_release); EXPORT_SYMBOL_GPL(dm_bufio_release);
void dm_bufio_mark_buffer_dirty(struct dm_buffer *b) void dm_bufio_mark_partial_buffer_dirty(struct dm_buffer *b,
unsigned start, unsigned end)
{ {
struct dm_bufio_client *c = b->c; struct dm_bufio_client *c = b->c;
BUG_ON(start >= end);
BUG_ON(end > b->c->block_size);
dm_bufio_lock(c); dm_bufio_lock(c);
BUG_ON(test_bit(B_READING, &b->state)); BUG_ON(test_bit(B_READING, &b->state));
if (!test_and_set_bit(B_DIRTY, &b->state)) if (!test_and_set_bit(B_DIRTY, &b->state)) {
b->dirty_start = start;
b->dirty_end = end;
__relink_lru(b, LIST_DIRTY); __relink_lru(b, LIST_DIRTY);
} else {
if (start < b->dirty_start)
b->dirty_start = start;
if (end > b->dirty_end)
b->dirty_end = end;
}
dm_bufio_unlock(c); dm_bufio_unlock(c);
} }
EXPORT_SYMBOL_GPL(dm_bufio_mark_partial_buffer_dirty);
void dm_bufio_mark_buffer_dirty(struct dm_buffer *b)
{
dm_bufio_mark_partial_buffer_dirty(b, 0, b->c->block_size);
}
EXPORT_SYMBOL_GPL(dm_bufio_mark_buffer_dirty); EXPORT_SYMBOL_GPL(dm_bufio_mark_buffer_dirty);
void dm_bufio_write_dirty_buffers_async(struct dm_bufio_client *c) void dm_bufio_write_dirty_buffers_async(struct dm_bufio_client *c)
...@@ -1398,6 +1435,8 @@ void dm_bufio_release_move(struct dm_buffer *b, sector_t new_block) ...@@ -1398,6 +1435,8 @@ void dm_bufio_release_move(struct dm_buffer *b, sector_t new_block)
wait_on_bit_io(&b->state, B_WRITING, wait_on_bit_io(&b->state, B_WRITING,
TASK_UNINTERRUPTIBLE); TASK_UNINTERRUPTIBLE);
set_bit(B_DIRTY, &b->state); set_bit(B_DIRTY, &b->state);
b->dirty_start = 0;
b->dirty_end = c->block_size;
__unlink_buffer(b); __unlink_buffer(b);
__link_buffer(b, new_block, LIST_DIRTY); __link_buffer(b, new_block, LIST_DIRTY);
} else { } else {
......
...@@ -93,6 +93,15 @@ void dm_bufio_release(struct dm_buffer *b); ...@@ -93,6 +93,15 @@ void dm_bufio_release(struct dm_buffer *b);
*/ */
void dm_bufio_mark_buffer_dirty(struct dm_buffer *b); void dm_bufio_mark_buffer_dirty(struct dm_buffer *b);
/*
* Mark a part of the buffer dirty.
*
* The specified part of the buffer is scheduled to be written. dm-bufio may
* write the specified part of the buffer or it may write a larger superset.
*/
void dm_bufio_mark_partial_buffer_dirty(struct dm_buffer *b,
unsigned start, unsigned end);
/* /*
* Initiate writing of dirty buffers, without waiting for completion. * Initiate writing of dirty buffers, without waiting for completion.
*/ */
......
...@@ -2306,7 +2306,7 @@ static void init_features(struct cache_features *cf) ...@@ -2306,7 +2306,7 @@ static void init_features(struct cache_features *cf)
static int parse_features(struct cache_args *ca, struct dm_arg_set *as, static int parse_features(struct cache_args *ca, struct dm_arg_set *as,
char **error) char **error)
{ {
static struct dm_arg _args[] = { static const struct dm_arg _args[] = {
{0, 2, "Invalid number of cache feature arguments"}, {0, 2, "Invalid number of cache feature arguments"},
}; };
...@@ -2348,7 +2348,7 @@ static int parse_features(struct cache_args *ca, struct dm_arg_set *as, ...@@ -2348,7 +2348,7 @@ static int parse_features(struct cache_args *ca, struct dm_arg_set *as,
static int parse_policy(struct cache_args *ca, struct dm_arg_set *as, static int parse_policy(struct cache_args *ca, struct dm_arg_set *as,
char **error) char **error)
{ {
static struct dm_arg _args[] = { static const struct dm_arg _args[] = {
{0, 1024, "Invalid number of policy arguments"}, {0, 1024, "Invalid number of policy arguments"},
}; };
......
...@@ -2529,7 +2529,7 @@ static int crypt_ctr_optional(struct dm_target *ti, unsigned int argc, char **ar ...@@ -2529,7 +2529,7 @@ static int crypt_ctr_optional(struct dm_target *ti, unsigned int argc, char **ar
{ {
struct crypt_config *cc = ti->private; struct crypt_config *cc = ti->private;
struct dm_arg_set as; struct dm_arg_set as;
static struct dm_arg _args[] = { static const struct dm_arg _args[] = {
{0, 6, "Invalid number of feature args"}, {0, 6, "Invalid number of feature args"},
}; };
unsigned int opt_params, val; unsigned int opt_params, val;
......
...@@ -51,7 +51,7 @@ static int parse_features(struct dm_arg_set *as, struct flakey_c *fc, ...@@ -51,7 +51,7 @@ static int parse_features(struct dm_arg_set *as, struct flakey_c *fc,
unsigned argc; unsigned argc;
const char *arg_name; const char *arg_name;
static struct dm_arg _args[] = { static const struct dm_arg _args[] = {
{0, 6, "Invalid number of feature args"}, {0, 6, "Invalid number of feature args"},
{1, UINT_MAX, "Invalid corrupt bio byte"}, {1, UINT_MAX, "Invalid corrupt bio byte"},
{0, 255, "Invalid corrupt value to write into bio byte (0-255)"}, {0, 255, "Invalid corrupt value to write into bio byte (0-255)"},
...@@ -178,7 +178,7 @@ static int parse_features(struct dm_arg_set *as, struct flakey_c *fc, ...@@ -178,7 +178,7 @@ static int parse_features(struct dm_arg_set *as, struct flakey_c *fc,
*/ */
static int flakey_ctr(struct dm_target *ti, unsigned int argc, char **argv) static int flakey_ctr(struct dm_target *ti, unsigned int argc, char **argv)
{ {
static struct dm_arg _args[] = { static const struct dm_arg _args[] = {
{0, UINT_MAX, "Invalid up interval"}, {0, UINT_MAX, "Invalid up interval"},
{0, UINT_MAX, "Invalid down interval"}, {0, UINT_MAX, "Invalid down interval"},
}; };
......
...@@ -225,6 +225,8 @@ struct dm_integrity_c { ...@@ -225,6 +225,8 @@ struct dm_integrity_c {
struct alg_spec internal_hash_alg; struct alg_spec internal_hash_alg;
struct alg_spec journal_crypt_alg; struct alg_spec journal_crypt_alg;
struct alg_spec journal_mac_alg; struct alg_spec journal_mac_alg;
atomic64_t number_of_mismatches;
}; };
struct dm_integrity_range { struct dm_integrity_range {
...@@ -298,7 +300,7 @@ static void __DEBUG_bytes(__u8 *bytes, size_t len, const char *msg, ...) ...@@ -298,7 +300,7 @@ static void __DEBUG_bytes(__u8 *bytes, size_t len, const char *msg, ...)
/* /*
* DM Integrity profile, protection is performed layer above (dm-crypt) * DM Integrity profile, protection is performed layer above (dm-crypt)
*/ */
static struct blk_integrity_profile dm_integrity_profile = { static const struct blk_integrity_profile dm_integrity_profile = {
.name = "DM-DIF-EXT-TAG", .name = "DM-DIF-EXT-TAG",
.generate_fn = NULL, .generate_fn = NULL,
.verify_fn = NULL, .verify_fn = NULL,
...@@ -310,6 +312,8 @@ static void dm_integrity_dtr(struct dm_target *ti); ...@@ -310,6 +312,8 @@ static void dm_integrity_dtr(struct dm_target *ti);
static void dm_integrity_io_error(struct dm_integrity_c *ic, const char *msg, int err) static void dm_integrity_io_error(struct dm_integrity_c *ic, const char *msg, int err)
{ {
if (err == -EILSEQ)
atomic64_inc(&ic->number_of_mismatches);
if (!cmpxchg(&ic->failed, 0, err)) if (!cmpxchg(&ic->failed, 0, err))
DMERR("Error on %s: %d", msg, err); DMERR("Error on %s: %d", msg, err);
} }
...@@ -770,13 +774,13 @@ static void write_journal(struct dm_integrity_c *ic, unsigned commit_start, unsi ...@@ -770,13 +774,13 @@ static void write_journal(struct dm_integrity_c *ic, unsigned commit_start, unsi
unsigned i; unsigned i;
io_comp.ic = ic; io_comp.ic = ic;
io_comp.comp = COMPLETION_INITIALIZER_ONSTACK(io_comp.comp); init_completion(&io_comp.comp);
if (commit_start + commit_sections <= ic->journal_sections) { if (commit_start + commit_sections <= ic->journal_sections) {
io_comp.in_flight = (atomic_t)ATOMIC_INIT(1); io_comp.in_flight = (atomic_t)ATOMIC_INIT(1);
if (ic->journal_io) { if (ic->journal_io) {
crypt_comp_1.ic = ic; crypt_comp_1.ic = ic;
crypt_comp_1.comp = COMPLETION_INITIALIZER_ONSTACK(crypt_comp_1.comp); init_completion(&crypt_comp_1.comp);
crypt_comp_1.in_flight = (atomic_t)ATOMIC_INIT(0); crypt_comp_1.in_flight = (atomic_t)ATOMIC_INIT(0);
encrypt_journal(ic, true, commit_start, commit_sections, &crypt_comp_1); encrypt_journal(ic, true, commit_start, commit_sections, &crypt_comp_1);
wait_for_completion_io(&crypt_comp_1.comp); wait_for_completion_io(&crypt_comp_1.comp);
...@@ -792,18 +796,18 @@ static void write_journal(struct dm_integrity_c *ic, unsigned commit_start, unsi ...@@ -792,18 +796,18 @@ static void write_journal(struct dm_integrity_c *ic, unsigned commit_start, unsi
to_end = ic->journal_sections - commit_start; to_end = ic->journal_sections - commit_start;
if (ic->journal_io) { if (ic->journal_io) {
crypt_comp_1.ic = ic; crypt_comp_1.ic = ic;
crypt_comp_1.comp = COMPLETION_INITIALIZER_ONSTACK(crypt_comp_1.comp); init_completion(&crypt_comp_1.comp);
crypt_comp_1.in_flight = (atomic_t)ATOMIC_INIT(0); crypt_comp_1.in_flight = (atomic_t)ATOMIC_INIT(0);
encrypt_journal(ic, true, commit_start, to_end, &crypt_comp_1); encrypt_journal(ic, true, commit_start, to_end, &crypt_comp_1);
if (try_wait_for_completion(&crypt_comp_1.comp)) { if (try_wait_for_completion(&crypt_comp_1.comp)) {
rw_journal(ic, REQ_OP_WRITE, REQ_FUA, commit_start, to_end, &io_comp); rw_journal(ic, REQ_OP_WRITE, REQ_FUA, commit_start, to_end, &io_comp);
crypt_comp_1.comp = COMPLETION_INITIALIZER_ONSTACK(crypt_comp_1.comp); reinit_completion(&crypt_comp_1.comp);
crypt_comp_1.in_flight = (atomic_t)ATOMIC_INIT(0); crypt_comp_1.in_flight = (atomic_t)ATOMIC_INIT(0);
encrypt_journal(ic, true, 0, commit_sections - to_end, &crypt_comp_1); encrypt_journal(ic, true, 0, commit_sections - to_end, &crypt_comp_1);
wait_for_completion_io(&crypt_comp_1.comp); wait_for_completion_io(&crypt_comp_1.comp);
} else { } else {
crypt_comp_2.ic = ic; crypt_comp_2.ic = ic;
crypt_comp_2.comp = COMPLETION_INITIALIZER_ONSTACK(crypt_comp_2.comp); init_completion(&crypt_comp_2.comp);
crypt_comp_2.in_flight = (atomic_t)ATOMIC_INIT(0); crypt_comp_2.in_flight = (atomic_t)ATOMIC_INIT(0);
encrypt_journal(ic, true, 0, commit_sections - to_end, &crypt_comp_2); encrypt_journal(ic, true, 0, commit_sections - to_end, &crypt_comp_2);
wait_for_completion_io(&crypt_comp_1.comp); wait_for_completion_io(&crypt_comp_1.comp);
...@@ -1041,7 +1045,7 @@ static int dm_integrity_rw_tag(struct dm_integrity_c *ic, unsigned char *tag, se ...@@ -1041,7 +1045,7 @@ static int dm_integrity_rw_tag(struct dm_integrity_c *ic, unsigned char *tag, se
memcpy(tag, dp, to_copy); memcpy(tag, dp, to_copy);
} else if (op == TAG_WRITE) { } else if (op == TAG_WRITE) {
memcpy(dp, tag, to_copy); memcpy(dp, tag, to_copy);
dm_bufio_mark_buffer_dirty(b); dm_bufio_mark_partial_buffer_dirty(b, *metadata_offset, *metadata_offset + to_copy);
} else { } else {
/* e.g.: op == TAG_CMP */ /* e.g.: op == TAG_CMP */
if (unlikely(memcmp(dp, tag, to_copy))) { if (unlikely(memcmp(dp, tag, to_copy))) {
...@@ -1275,6 +1279,7 @@ static void integrity_metadata(struct work_struct *w) ...@@ -1275,6 +1279,7 @@ static void integrity_metadata(struct work_struct *w)
DMERR("Checksum failed at sector 0x%llx", DMERR("Checksum failed at sector 0x%llx",
(unsigned long long)(sector - ((r + ic->tag_size - 1) / ic->tag_size))); (unsigned long long)(sector - ((r + ic->tag_size - 1) / ic->tag_size)));
r = -EILSEQ; r = -EILSEQ;
atomic64_inc(&ic->number_of_mismatches);
} }
if (likely(checksums != checksums_onstack)) if (likely(checksums != checksums_onstack))
kfree(checksums); kfree(checksums);
...@@ -1676,7 +1681,7 @@ static void dm_integrity_map_continue(struct dm_integrity_io *dio, bool from_map ...@@ -1676,7 +1681,7 @@ static void dm_integrity_map_continue(struct dm_integrity_io *dio, bool from_map
dio->in_flight = (atomic_t)ATOMIC_INIT(2); dio->in_flight = (atomic_t)ATOMIC_INIT(2);
if (need_sync_io) { if (need_sync_io) {
read_comp = COMPLETION_INITIALIZER_ONSTACK(read_comp); init_completion(&read_comp);
dio->completion = &read_comp; dio->completion = &read_comp;
} else } else
dio->completion = NULL; dio->completion = NULL;
...@@ -1700,7 +1705,11 @@ static void dm_integrity_map_continue(struct dm_integrity_io *dio, bool from_map ...@@ -1700,7 +1705,11 @@ static void dm_integrity_map_continue(struct dm_integrity_io *dio, bool from_map
if (need_sync_io) { if (need_sync_io) {
wait_for_completion_io(&read_comp); wait_for_completion_io(&read_comp);
if (likely(!bio->bi_status))
integrity_metadata(&dio->work); integrity_metadata(&dio->work);
else
dec_in_flight(dio);
} else { } else {
INIT_WORK(&dio->work, integrity_metadata); INIT_WORK(&dio->work, integrity_metadata);
queue_work(ic->metadata_wq, &dio->work); queue_work(ic->metadata_wq, &dio->work);
...@@ -1834,7 +1843,7 @@ static void do_journal_write(struct dm_integrity_c *ic, unsigned write_start, ...@@ -1834,7 +1843,7 @@ static void do_journal_write(struct dm_integrity_c *ic, unsigned write_start,
comp.ic = ic; comp.ic = ic;
comp.in_flight = (atomic_t)ATOMIC_INIT(1); comp.in_flight = (atomic_t)ATOMIC_INIT(1);
comp.comp = COMPLETION_INITIALIZER_ONSTACK(comp.comp); init_completion(&comp.comp);
i = write_start; i = write_start;
for (n = 0; n < write_sections; n++, i++, wraparound_section(ic, &i)) { for (n = 0; n < write_sections; n++, i++, wraparound_section(ic, &i)) {
...@@ -2061,7 +2070,7 @@ static void replay_journal(struct dm_integrity_c *ic) ...@@ -2061,7 +2070,7 @@ static void replay_journal(struct dm_integrity_c *ic)
if (ic->journal_io) { if (ic->journal_io) {
struct journal_completion crypt_comp; struct journal_completion crypt_comp;
crypt_comp.ic = ic; crypt_comp.ic = ic;
crypt_comp.comp = COMPLETION_INITIALIZER_ONSTACK(crypt_comp.comp); init_completion(&crypt_comp.comp);
crypt_comp.in_flight = (atomic_t)ATOMIC_INIT(0); crypt_comp.in_flight = (atomic_t)ATOMIC_INIT(0);
encrypt_journal(ic, false, 0, ic->journal_sections, &crypt_comp); encrypt_journal(ic, false, 0, ic->journal_sections, &crypt_comp);
wait_for_completion(&crypt_comp.comp); wait_for_completion(&crypt_comp.comp);
...@@ -2233,7 +2242,7 @@ static void dm_integrity_status(struct dm_target *ti, status_type_t type, ...@@ -2233,7 +2242,7 @@ static void dm_integrity_status(struct dm_target *ti, status_type_t type,
switch (type) { switch (type) {
case STATUSTYPE_INFO: case STATUSTYPE_INFO:
result[0] = '\0'; DMEMIT("%llu", (unsigned long long)atomic64_read(&ic->number_of_mismatches));
break; break;
case STATUSTYPE_TABLE: { case STATUSTYPE_TABLE: {
...@@ -2634,7 +2643,7 @@ static int create_journal(struct dm_integrity_c *ic, char **error) ...@@ -2634,7 +2643,7 @@ static int create_journal(struct dm_integrity_c *ic, char **error)
memset(iv, 0x00, ivsize); memset(iv, 0x00, ivsize);
skcipher_request_set_crypt(req, sg, sg, PAGE_SIZE * ic->journal_pages + sizeof ic->commit_ids, iv); skcipher_request_set_crypt(req, sg, sg, PAGE_SIZE * ic->journal_pages + sizeof ic->commit_ids, iv);
comp.comp = COMPLETION_INITIALIZER_ONSTACK(comp.comp); init_completion(&comp.comp);
comp.in_flight = (atomic_t)ATOMIC_INIT(1); comp.in_flight = (atomic_t)ATOMIC_INIT(1);
if (do_crypt(true, req, &comp)) if (do_crypt(true, req, &comp))
wait_for_completion(&comp.comp); wait_for_completion(&comp.comp);
...@@ -2691,7 +2700,7 @@ static int create_journal(struct dm_integrity_c *ic, char **error) ...@@ -2691,7 +2700,7 @@ static int create_journal(struct dm_integrity_c *ic, char **error)
sg_init_one(&sg, crypt_data, crypt_len); sg_init_one(&sg, crypt_data, crypt_len);
skcipher_request_set_crypt(req, &sg, &sg, crypt_len, iv); skcipher_request_set_crypt(req, &sg, &sg, crypt_len, iv);
comp.comp = COMPLETION_INITIALIZER_ONSTACK(comp.comp); init_completion(&comp.comp);
comp.in_flight = (atomic_t)ATOMIC_INIT(1); comp.in_flight = (atomic_t)ATOMIC_INIT(1);
if (do_crypt(true, req, &comp)) if (do_crypt(true, req, &comp))
wait_for_completion(&comp.comp); wait_for_completion(&comp.comp);
...@@ -2778,7 +2787,7 @@ static int dm_integrity_ctr(struct dm_target *ti, unsigned argc, char **argv) ...@@ -2778,7 +2787,7 @@ static int dm_integrity_ctr(struct dm_target *ti, unsigned argc, char **argv)
int r; int r;
unsigned extra_args; unsigned extra_args;
struct dm_arg_set as; struct dm_arg_set as;
static struct dm_arg _args[] = { static const struct dm_arg _args[] = {
{0, 9, "Invalid number of feature args"}, {0, 9, "Invalid number of feature args"},
}; };
unsigned journal_sectors, interleave_sectors, buffer_sectors, journal_watermark, sync_msec; unsigned journal_sectors, interleave_sectors, buffer_sectors, journal_watermark, sync_msec;
...@@ -2806,6 +2815,7 @@ static int dm_integrity_ctr(struct dm_target *ti, unsigned argc, char **argv) ...@@ -2806,6 +2815,7 @@ static int dm_integrity_ctr(struct dm_target *ti, unsigned argc, char **argv)
bio_list_init(&ic->flush_bio_list); bio_list_init(&ic->flush_bio_list);
init_waitqueue_head(&ic->copy_to_journal_wait); init_waitqueue_head(&ic->copy_to_journal_wait);
init_completion(&ic->crypto_backoff); init_completion(&ic->crypto_backoff);
atomic64_set(&ic->number_of_mismatches, 0);
r = dm_get_device(ti, argv[0], dm_table_get_mode(ti->table), &ic->dev); r = dm_get_device(ti, argv[0], dm_table_get_mode(ti->table), &ic->dev);
if (r) { if (r) {
...@@ -3202,7 +3212,7 @@ static void dm_integrity_dtr(struct dm_target *ti) ...@@ -3202,7 +3212,7 @@ static void dm_integrity_dtr(struct dm_target *ti)
static struct target_type integrity_target = { static struct target_type integrity_target = {
.name = "integrity", .name = "integrity",
.version = {1, 0, 0}, .version = {1, 1, 0},
.module = THIS_MODULE, .module = THIS_MODULE,
.features = DM_TARGET_SINGLETON | DM_TARGET_INTEGRITY, .features = DM_TARGET_SINGLETON | DM_TARGET_INTEGRITY,
.ctr = dm_integrity_ctr, .ctr = dm_integrity_ctr,
......
...@@ -1629,7 +1629,7 @@ static int target_message(struct file *filp, struct dm_ioctl *param, size_t para ...@@ -1629,7 +1629,7 @@ static int target_message(struct file *filp, struct dm_ioctl *param, size_t para
*---------------------------------------------------------------*/ *---------------------------------------------------------------*/
static ioctl_fn lookup_ioctl(unsigned int cmd, int *ioctl_flags) static ioctl_fn lookup_ioctl(unsigned int cmd, int *ioctl_flags)
{ {
static struct { static const struct {
int cmd; int cmd;
int flags; int flags;
ioctl_fn fn; ioctl_fn fn;
......
...@@ -184,20 +184,6 @@ static size_t linear_dax_copy_from_iter(struct dm_target *ti, pgoff_t pgoff, ...@@ -184,20 +184,6 @@ static size_t linear_dax_copy_from_iter(struct dm_target *ti, pgoff_t pgoff,
return dax_copy_from_iter(dax_dev, pgoff, addr, bytes, i); return dax_copy_from_iter(dax_dev, pgoff, addr, bytes, i);
} }
static void linear_dax_flush(struct dm_target *ti, pgoff_t pgoff, void *addr,
size_t size)
{
struct linear_c *lc = ti->private;
struct block_device *bdev = lc->dev->bdev;
struct dax_device *dax_dev = lc->dev->dax_dev;
sector_t dev_sector, sector = pgoff * PAGE_SECTORS;
dev_sector = linear_map_sector(ti, sector);
if (bdev_dax_pgoff(bdev, dev_sector, ALIGN(size, PAGE_SIZE), &pgoff))
return;
dax_flush(dax_dev, pgoff, addr, size);
}
static struct target_type linear_target = { static struct target_type linear_target = {
.name = "linear", .name = "linear",
.version = {1, 4, 0}, .version = {1, 4, 0},
...@@ -212,7 +198,6 @@ static struct target_type linear_target = { ...@@ -212,7 +198,6 @@ static struct target_type linear_target = {
.iterate_devices = linear_iterate_devices, .iterate_devices = linear_iterate_devices,
.direct_access = linear_dax_direct_access, .direct_access = linear_dax_direct_access,
.dax_copy_from_iter = linear_dax_copy_from_iter, .dax_copy_from_iter = linear_dax_copy_from_iter,
.dax_flush = linear_dax_flush,
}; };
int __init dm_linear_init(void) int __init dm_linear_init(void)
......
...@@ -100,6 +100,7 @@ struct log_writes_c { ...@@ -100,6 +100,7 @@ struct log_writes_c {
struct dm_dev *logdev; struct dm_dev *logdev;
u64 logged_entries; u64 logged_entries;
u32 sectorsize; u32 sectorsize;
u32 sectorshift;
atomic_t io_blocks; atomic_t io_blocks;
atomic_t pending_blocks; atomic_t pending_blocks;
sector_t next_sector; sector_t next_sector;
...@@ -128,6 +129,18 @@ struct per_bio_data { ...@@ -128,6 +129,18 @@ struct per_bio_data {
struct pending_block *block; struct pending_block *block;
}; };
static inline sector_t bio_to_dev_sectors(struct log_writes_c *lc,
sector_t sectors)
{
return sectors >> (lc->sectorshift - SECTOR_SHIFT);
}
static inline sector_t dev_to_bio_sectors(struct log_writes_c *lc,
sector_t sectors)
{
return sectors << (lc->sectorshift - SECTOR_SHIFT);
}
static void put_pending_block(struct log_writes_c *lc) static void put_pending_block(struct log_writes_c *lc)
{ {
if (atomic_dec_and_test(&lc->pending_blocks)) { if (atomic_dec_and_test(&lc->pending_blocks)) {
...@@ -253,7 +266,7 @@ static int log_one_block(struct log_writes_c *lc, ...@@ -253,7 +266,7 @@ static int log_one_block(struct log_writes_c *lc,
if (!block->vec_cnt) if (!block->vec_cnt)
goto out; goto out;
sector++; sector += dev_to_bio_sectors(lc, 1);
atomic_inc(&lc->io_blocks); atomic_inc(&lc->io_blocks);
bio = bio_alloc(GFP_KERNEL, min(block->vec_cnt, BIO_MAX_PAGES)); bio = bio_alloc(GFP_KERNEL, min(block->vec_cnt, BIO_MAX_PAGES));
...@@ -354,10 +367,9 @@ static int log_writes_kthread(void *arg) ...@@ -354,10 +367,9 @@ static int log_writes_kthread(void *arg)
goto next; goto next;
sector = lc->next_sector; sector = lc->next_sector;
if (block->flags & LOG_DISCARD_FLAG) if (!(block->flags & LOG_DISCARD_FLAG))
lc->next_sector++; lc->next_sector += dev_to_bio_sectors(lc, block->nr_sectors);
else lc->next_sector += dev_to_bio_sectors(lc, 1);
lc->next_sector += block->nr_sectors + 1;
/* /*
* Apparently the size of the device may not be known * Apparently the size of the device may not be known
...@@ -399,7 +411,7 @@ static int log_writes_kthread(void *arg) ...@@ -399,7 +411,7 @@ static int log_writes_kthread(void *arg)
if (!try_to_freeze()) { if (!try_to_freeze()) {
set_current_state(TASK_INTERRUPTIBLE); set_current_state(TASK_INTERRUPTIBLE);
if (!kthread_should_stop() && if (!kthread_should_stop() &&
!atomic_read(&lc->pending_blocks)) list_empty(&lc->logging_blocks))
schedule(); schedule();
__set_current_state(TASK_RUNNING); __set_current_state(TASK_RUNNING);
} }
...@@ -435,7 +447,6 @@ static int log_writes_ctr(struct dm_target *ti, unsigned int argc, char **argv) ...@@ -435,7 +447,6 @@ static int log_writes_ctr(struct dm_target *ti, unsigned int argc, char **argv)
INIT_LIST_HEAD(&lc->unflushed_blocks); INIT_LIST_HEAD(&lc->unflushed_blocks);
INIT_LIST_HEAD(&lc->logging_blocks); INIT_LIST_HEAD(&lc->logging_blocks);
init_waitqueue_head(&lc->wait); init_waitqueue_head(&lc->wait);
lc->sectorsize = 1 << SECTOR_SHIFT;
atomic_set(&lc->io_blocks, 0); atomic_set(&lc->io_blocks, 0);
atomic_set(&lc->pending_blocks, 0); atomic_set(&lc->pending_blocks, 0);
...@@ -455,6 +466,8 @@ static int log_writes_ctr(struct dm_target *ti, unsigned int argc, char **argv) ...@@ -455,6 +466,8 @@ static int log_writes_ctr(struct dm_target *ti, unsigned int argc, char **argv)
goto bad; goto bad;
} }
lc->sectorsize = bdev_logical_block_size(lc->dev->bdev);
lc->sectorshift = ilog2(lc->sectorsize);
lc->log_kthread = kthread_run(log_writes_kthread, lc, "log-write"); lc->log_kthread = kthread_run(log_writes_kthread, lc, "log-write");
if (IS_ERR(lc->log_kthread)) { if (IS_ERR(lc->log_kthread)) {
ret = PTR_ERR(lc->log_kthread); ret = PTR_ERR(lc->log_kthread);
...@@ -464,8 +477,12 @@ static int log_writes_ctr(struct dm_target *ti, unsigned int argc, char **argv) ...@@ -464,8 +477,12 @@ static int log_writes_ctr(struct dm_target *ti, unsigned int argc, char **argv)
goto bad; goto bad;
} }
/* We put the super at sector 0, start logging at sector 1 */ /*
lc->next_sector = 1; * next_sector is in 512b sectors to correspond to what bi_sector expects.
* The super starts at sector 0, and the next_sector is the next logical
* one based on the sectorsize of the device.
*/
lc->next_sector = lc->sectorsize >> SECTOR_SHIFT;
lc->logging_enabled = true; lc->logging_enabled = true;
lc->end_sector = logdev_last_sector(lc); lc->end_sector = logdev_last_sector(lc);
lc->device_supports_discard = true; lc->device_supports_discard = true;
...@@ -599,8 +616,8 @@ static int log_writes_map(struct dm_target *ti, struct bio *bio) ...@@ -599,8 +616,8 @@ static int log_writes_map(struct dm_target *ti, struct bio *bio)
if (discard_bio) if (discard_bio)
block->flags |= LOG_DISCARD_FLAG; block->flags |= LOG_DISCARD_FLAG;
block->sector = bio->bi_iter.bi_sector; block->sector = bio_to_dev_sectors(lc, bio->bi_iter.bi_sector);
block->nr_sectors = bio_sectors(bio); block->nr_sectors = bio_to_dev_sectors(lc, bio_sectors(bio));
/* We don't need the data, just submit */ /* We don't need the data, just submit */
if (discard_bio) { if (discard_bio) {
...@@ -767,9 +784,12 @@ static void log_writes_io_hints(struct dm_target *ti, struct queue_limits *limit ...@@ -767,9 +784,12 @@ static void log_writes_io_hints(struct dm_target *ti, struct queue_limits *limit
if (!q || !blk_queue_discard(q)) { if (!q || !blk_queue_discard(q)) {
lc->device_supports_discard = false; lc->device_supports_discard = false;
limits->discard_granularity = 1 << SECTOR_SHIFT; limits->discard_granularity = lc->sectorsize;
limits->max_discard_sectors = (UINT_MAX >> SECTOR_SHIFT); limits->max_discard_sectors = (UINT_MAX >> SECTOR_SHIFT);
} }
limits->logical_block_size = bdev_logical_block_size(lc->dev->bdev);
limits->physical_block_size = bdev_physical_block_size(lc->dev->bdev);
limits->io_min = limits->physical_block_size;
} }
static struct target_type log_writes_target = { static struct target_type log_writes_target = {
......
...@@ -632,6 +632,10 @@ static void process_queued_bios(struct work_struct *work) ...@@ -632,6 +632,10 @@ static void process_queued_bios(struct work_struct *work)
case DM_MAPIO_REMAPPED: case DM_MAPIO_REMAPPED:
generic_make_request(bio); generic_make_request(bio);
break; break;
case 0:
break;
default:
WARN_ONCE(true, "__multipath_map_bio() returned %d\n", r);
} }
} }
blk_finish_plug(&plug); blk_finish_plug(&plug);
...@@ -698,7 +702,7 @@ static int parse_path_selector(struct dm_arg_set *as, struct priority_group *pg, ...@@ -698,7 +702,7 @@ static int parse_path_selector(struct dm_arg_set *as, struct priority_group *pg,
struct path_selector_type *pst; struct path_selector_type *pst;
unsigned ps_argc; unsigned ps_argc;
static struct dm_arg _args[] = { static const struct dm_arg _args[] = {
{0, 1024, "invalid number of path selector args"}, {0, 1024, "invalid number of path selector args"},
}; };
...@@ -822,7 +826,7 @@ static struct pgpath *parse_path(struct dm_arg_set *as, struct path_selector *ps ...@@ -822,7 +826,7 @@ static struct pgpath *parse_path(struct dm_arg_set *as, struct path_selector *ps
static struct priority_group *parse_priority_group(struct dm_arg_set *as, static struct priority_group *parse_priority_group(struct dm_arg_set *as,
struct multipath *m) struct multipath *m)
{ {
static struct dm_arg _args[] = { static const struct dm_arg _args[] = {
{1, 1024, "invalid number of paths"}, {1, 1024, "invalid number of paths"},
{0, 1024, "invalid number of selector args"} {0, 1024, "invalid number of selector args"}
}; };
...@@ -898,7 +902,7 @@ static int parse_hw_handler(struct dm_arg_set *as, struct multipath *m) ...@@ -898,7 +902,7 @@ static int parse_hw_handler(struct dm_arg_set *as, struct multipath *m)
int ret; int ret;
struct dm_target *ti = m->ti; struct dm_target *ti = m->ti;
static struct dm_arg _args[] = { static const struct dm_arg _args[] = {
{0, 1024, "invalid number of hardware handler args"}, {0, 1024, "invalid number of hardware handler args"},
}; };
...@@ -950,7 +954,7 @@ static int parse_features(struct dm_arg_set *as, struct multipath *m) ...@@ -950,7 +954,7 @@ static int parse_features(struct dm_arg_set *as, struct multipath *m)
struct dm_target *ti = m->ti; struct dm_target *ti = m->ti;
const char *arg_name; const char *arg_name;
static struct dm_arg _args[] = { static const struct dm_arg _args[] = {
{0, 8, "invalid number of feature args"}, {0, 8, "invalid number of feature args"},
{1, 50, "pg_init_retries must be between 1 and 50"}, {1, 50, "pg_init_retries must be between 1 and 50"},
{0, 60000, "pg_init_delay_msecs must be between 0 and 60000"}, {0, 60000, "pg_init_delay_msecs must be between 0 and 60000"},
...@@ -1019,7 +1023,7 @@ static int parse_features(struct dm_arg_set *as, struct multipath *m) ...@@ -1019,7 +1023,7 @@ static int parse_features(struct dm_arg_set *as, struct multipath *m)
static int multipath_ctr(struct dm_target *ti, unsigned argc, char **argv) static int multipath_ctr(struct dm_target *ti, unsigned argc, char **argv)
{ {
/* target arguments */ /* target arguments */
static struct dm_arg _args[] = { static const struct dm_arg _args[] = {
{0, 1024, "invalid number of priority groups"}, {0, 1024, "invalid number of priority groups"},
{0, 1024, "invalid initial priority group number"}, {0, 1024, "invalid initial priority group number"},
}; };
...@@ -1379,6 +1383,7 @@ static void pg_init_done(void *data, int errors) ...@@ -1379,6 +1383,7 @@ static void pg_init_done(void *data, int errors)
case SCSI_DH_RETRY: case SCSI_DH_RETRY:
/* Wait before retrying. */ /* Wait before retrying. */
delay_retry = 1; delay_retry = 1;
/* fall through */
case SCSI_DH_IMM_RETRY: case SCSI_DH_IMM_RETRY:
case SCSI_DH_RES_TEMP_UNAVAIL: case SCSI_DH_RES_TEMP_UNAVAIL:
if (pg_init_limit_reached(m, pgpath)) if (pg_init_limit_reached(m, pgpath))
......
...@@ -117,9 +117,9 @@ static void end_clone_bio(struct bio *clone) ...@@ -117,9 +117,9 @@ static void end_clone_bio(struct bio *clone)
struct dm_rq_clone_bio_info *info = struct dm_rq_clone_bio_info *info =
container_of(clone, struct dm_rq_clone_bio_info, clone); container_of(clone, struct dm_rq_clone_bio_info, clone);
struct dm_rq_target_io *tio = info->tio; struct dm_rq_target_io *tio = info->tio;
struct bio *bio = info->orig;
unsigned int nr_bytes = info->orig->bi_iter.bi_size; unsigned int nr_bytes = info->orig->bi_iter.bi_size;
blk_status_t error = clone->bi_status; blk_status_t error = clone->bi_status;
bool is_last = !clone->bi_next;
bio_put(clone); bio_put(clone);
...@@ -137,28 +137,23 @@ static void end_clone_bio(struct bio *clone) ...@@ -137,28 +137,23 @@ static void end_clone_bio(struct bio *clone)
* when the request is completed. * when the request is completed.
*/ */
tio->error = error; tio->error = error;
return; goto exit;
} }
/* /*
* I/O for the bio successfully completed. * I/O for the bio successfully completed.
* Notice the data completion to the upper layer. * Notice the data completion to the upper layer.
*/ */
tio->completed += nr_bytes;
/*
* bios are processed from the head of the list.
* So the completing bio should always be rq->bio.
* If it's not, something wrong is happening.
*/
if (tio->orig->bio != bio)
DMERR("bio completion is going in the middle of the request");
/* /*
* Update the original request. * Update the original request.
* Do not use blk_end_request() here, because it may complete * Do not use blk_end_request() here, because it may complete
* the original request before the clone, and break the ordering. * the original request before the clone, and break the ordering.
*/ */
blk_update_request(tio->orig, BLK_STS_OK, nr_bytes); if (is_last)
exit:
blk_update_request(tio->orig, BLK_STS_OK, tio->completed);
} }
static struct dm_rq_target_io *tio_from_request(struct request *rq) static struct dm_rq_target_io *tio_from_request(struct request *rq)
...@@ -237,14 +232,14 @@ static void dm_end_request(struct request *clone, blk_status_t error) ...@@ -237,14 +232,14 @@ static void dm_end_request(struct request *clone, blk_status_t error)
/* /*
* Requeue the original request of a clone. * Requeue the original request of a clone.
*/ */
static void dm_old_requeue_request(struct request *rq) static void dm_old_requeue_request(struct request *rq, unsigned long delay_ms)
{ {
struct request_queue *q = rq->q; struct request_queue *q = rq->q;
unsigned long flags; unsigned long flags;
spin_lock_irqsave(q->queue_lock, flags); spin_lock_irqsave(q->queue_lock, flags);
blk_requeue_request(q, rq); blk_requeue_request(q, rq);
blk_run_queue_async(q); blk_delay_queue(q, delay_ms);
spin_unlock_irqrestore(q->queue_lock, flags); spin_unlock_irqrestore(q->queue_lock, flags);
} }
...@@ -270,6 +265,7 @@ static void dm_requeue_original_request(struct dm_rq_target_io *tio, bool delay_ ...@@ -270,6 +265,7 @@ static void dm_requeue_original_request(struct dm_rq_target_io *tio, bool delay_
struct mapped_device *md = tio->md; struct mapped_device *md = tio->md;
struct request *rq = tio->orig; struct request *rq = tio->orig;
int rw = rq_data_dir(rq); int rw = rq_data_dir(rq);
unsigned long delay_ms = delay_requeue ? 100 : 0;
rq_end_stats(md, rq); rq_end_stats(md, rq);
if (tio->clone) { if (tio->clone) {
...@@ -278,9 +274,9 @@ static void dm_requeue_original_request(struct dm_rq_target_io *tio, bool delay_ ...@@ -278,9 +274,9 @@ static void dm_requeue_original_request(struct dm_rq_target_io *tio, bool delay_
} }
if (!rq->q->mq_ops) if (!rq->q->mq_ops)
dm_old_requeue_request(rq); dm_old_requeue_request(rq, delay_ms);
else else
dm_mq_delay_requeue_request(rq, delay_requeue ? 100/*ms*/ : 0); dm_mq_delay_requeue_request(rq, delay_ms);
rq_completed(md, rw, false); rq_completed(md, rw, false);
} }
...@@ -455,6 +451,7 @@ static void init_tio(struct dm_rq_target_io *tio, struct request *rq, ...@@ -455,6 +451,7 @@ static void init_tio(struct dm_rq_target_io *tio, struct request *rq,
tio->clone = NULL; tio->clone = NULL;
tio->orig = rq; tio->orig = rq;
tio->error = 0; tio->error = 0;
tio->completed = 0;
/* /*
* Avoid initializing info for blk-mq; it passes * Avoid initializing info for blk-mq; it passes
* target-specific data through info.ptr * target-specific data through info.ptr
......
...@@ -29,6 +29,7 @@ struct dm_rq_target_io { ...@@ -29,6 +29,7 @@ struct dm_rq_target_io {
struct dm_stats_aux stats_aux; struct dm_stats_aux stats_aux;
unsigned long duration_jiffies; unsigned long duration_jiffies;
unsigned n_sectors; unsigned n_sectors;
unsigned completed;
}; };
/* /*
......
...@@ -351,25 +351,6 @@ static size_t stripe_dax_copy_from_iter(struct dm_target *ti, pgoff_t pgoff, ...@@ -351,25 +351,6 @@ static size_t stripe_dax_copy_from_iter(struct dm_target *ti, pgoff_t pgoff,
return dax_copy_from_iter(dax_dev, pgoff, addr, bytes, i); return dax_copy_from_iter(dax_dev, pgoff, addr, bytes, i);
} }
static void stripe_dax_flush(struct dm_target *ti, pgoff_t pgoff, void *addr,
size_t size)
{
sector_t dev_sector, sector = pgoff * PAGE_SECTORS;
struct stripe_c *sc = ti->private;
struct dax_device *dax_dev;
struct block_device *bdev;
uint32_t stripe;
stripe_map_sector(sc, sector, &stripe, &dev_sector);
dev_sector += sc->stripe[stripe].physical_start;
dax_dev = sc->stripe[stripe].dev->dax_dev;
bdev = sc->stripe[stripe].dev->bdev;
if (bdev_dax_pgoff(bdev, dev_sector, ALIGN(size, PAGE_SIZE), &pgoff))
return;
dax_flush(dax_dev, pgoff, addr, size);
}
/* /*
* Stripe status: * Stripe status:
* *
...@@ -489,7 +470,6 @@ static struct target_type stripe_target = { ...@@ -489,7 +470,6 @@ static struct target_type stripe_target = {
.io_hints = stripe_io_hints, .io_hints = stripe_io_hints,
.direct_access = stripe_dax_direct_access, .direct_access = stripe_dax_direct_access,
.dax_copy_from_iter = stripe_dax_copy_from_iter, .dax_copy_from_iter = stripe_dax_copy_from_iter,
.dax_flush = stripe_dax_flush,
}; };
int __init dm_stripe_init(void) int __init dm_stripe_init(void)
......
...@@ -251,7 +251,7 @@ static void switch_dtr(struct dm_target *ti) ...@@ -251,7 +251,7 @@ static void switch_dtr(struct dm_target *ti)
*/ */
static int switch_ctr(struct dm_target *ti, unsigned argc, char **argv) static int switch_ctr(struct dm_target *ti, unsigned argc, char **argv)
{ {
static struct dm_arg _args[] = { static const struct dm_arg _args[] = {
{1, (KMALLOC_MAX_SIZE - sizeof(struct switch_ctx)) / sizeof(struct switch_path), "Invalid number of paths"}, {1, (KMALLOC_MAX_SIZE - sizeof(struct switch_ctx)) / sizeof(struct switch_path), "Invalid number of paths"},
{1, UINT_MAX, "Invalid region size"}, {1, UINT_MAX, "Invalid region size"},
{0, 0, "Invalid number of optional args"}, {0, 0, "Invalid number of optional args"},
......
...@@ -806,7 +806,8 @@ int dm_table_add_target(struct dm_table *t, const char *type, ...@@ -806,7 +806,8 @@ int dm_table_add_target(struct dm_table *t, const char *type,
/* /*
* Target argument parsing helpers. * Target argument parsing helpers.
*/ */
static int validate_next_arg(struct dm_arg *arg, struct dm_arg_set *arg_set, static int validate_next_arg(const struct dm_arg *arg,
struct dm_arg_set *arg_set,
unsigned *value, char **error, unsigned grouped) unsigned *value, char **error, unsigned grouped)
{ {
const char *arg_str = dm_shift_arg(arg_set); const char *arg_str = dm_shift_arg(arg_set);
...@@ -824,14 +825,14 @@ static int validate_next_arg(struct dm_arg *arg, struct dm_arg_set *arg_set, ...@@ -824,14 +825,14 @@ static int validate_next_arg(struct dm_arg *arg, struct dm_arg_set *arg_set,
return 0; return 0;
} }
int dm_read_arg(struct dm_arg *arg, struct dm_arg_set *arg_set, int dm_read_arg(const struct dm_arg *arg, struct dm_arg_set *arg_set,
unsigned *value, char **error) unsigned *value, char **error)
{ {
return validate_next_arg(arg, arg_set, value, error, 0); return validate_next_arg(arg, arg_set, value, error, 0);
} }
EXPORT_SYMBOL(dm_read_arg); EXPORT_SYMBOL(dm_read_arg);
int dm_read_arg_group(struct dm_arg *arg, struct dm_arg_set *arg_set, int dm_read_arg_group(const struct dm_arg *arg, struct dm_arg_set *arg_set,
unsigned *value, char **error) unsigned *value, char **error)
{ {
return validate_next_arg(arg, arg_set, value, error, 1); return validate_next_arg(arg, arg_set, value, error, 1);
......
...@@ -3041,7 +3041,7 @@ static int parse_pool_features(struct dm_arg_set *as, struct pool_features *pf, ...@@ -3041,7 +3041,7 @@ static int parse_pool_features(struct dm_arg_set *as, struct pool_features *pf,
unsigned argc; unsigned argc;
const char *arg_name; const char *arg_name;
static struct dm_arg _args[] = { static const struct dm_arg _args[] = {
{0, 4, "Invalid number of pool feature arguments"}, {0, 4, "Invalid number of pool feature arguments"},
}; };
......
...@@ -839,7 +839,7 @@ static int verity_parse_opt_args(struct dm_arg_set *as, struct dm_verity *v) ...@@ -839,7 +839,7 @@ static int verity_parse_opt_args(struct dm_arg_set *as, struct dm_verity *v)
struct dm_target *ti = v->ti; struct dm_target *ti = v->ti;
const char *arg_name; const char *arg_name;
static struct dm_arg _args[] = { static const struct dm_arg _args[] = {
{0, DM_VERITY_OPTS_MAX, "Invalid number of feature args"}, {0, DM_VERITY_OPTS_MAX, "Invalid number of feature args"},
}; };
......
...@@ -987,24 +987,6 @@ static size_t dm_dax_copy_from_iter(struct dax_device *dax_dev, pgoff_t pgoff, ...@@ -987,24 +987,6 @@ static size_t dm_dax_copy_from_iter(struct dax_device *dax_dev, pgoff_t pgoff,
return ret; return ret;
} }
static void dm_dax_flush(struct dax_device *dax_dev, pgoff_t pgoff, void *addr,
size_t size)
{
struct mapped_device *md = dax_get_private(dax_dev);
sector_t sector = pgoff * PAGE_SECTORS;
struct dm_target *ti;
int srcu_idx;
ti = dm_dax_get_live_target(md, sector, &srcu_idx);
if (!ti)
goto out;
if (ti->type->dax_flush)
ti->type->dax_flush(ti, pgoff, addr, size);
out:
dm_put_live_table(md, srcu_idx);
}
/* /*
* A target may call dm_accept_partial_bio only from the map routine. It is * A target may call dm_accept_partial_bio only from the map routine. It is
* allowed for all bio types except REQ_PREFLUSH. * allowed for all bio types except REQ_PREFLUSH.
...@@ -2992,7 +2974,6 @@ static const struct block_device_operations dm_blk_dops = { ...@@ -2992,7 +2974,6 @@ static const struct block_device_operations dm_blk_dops = {
static const struct dax_operations dm_dax_ops = { static const struct dax_operations dm_dax_ops = {
.direct_access = dm_dax_direct_access, .direct_access = dm_dax_direct_access,
.copy_from_iter = dm_dax_copy_from_iter, .copy_from_iter = dm_dax_copy_from_iter,
.flush = dm_dax_flush,
}; };
/* /*
......
...@@ -262,16 +262,9 @@ static size_t pmem_copy_from_iter(struct dax_device *dax_dev, pgoff_t pgoff, ...@@ -262,16 +262,9 @@ static size_t pmem_copy_from_iter(struct dax_device *dax_dev, pgoff_t pgoff,
return copy_from_iter_flushcache(addr, bytes, i); return copy_from_iter_flushcache(addr, bytes, i);
} }
static void pmem_dax_flush(struct dax_device *dax_dev, pgoff_t pgoff,
void *addr, size_t size)
{
arch_wb_cache_pmem(addr, size);
}
static const struct dax_operations pmem_dax_ops = { static const struct dax_operations pmem_dax_ops = {
.direct_access = pmem_dax_direct_access, .direct_access = pmem_dax_direct_access,
.copy_from_iter = pmem_copy_from_iter, .copy_from_iter = pmem_copy_from_iter,
.flush = pmem_dax_flush,
}; };
static const struct attribute_group *pmem_attribute_groups[] = { static const struct attribute_group *pmem_attribute_groups[] = {
......
...@@ -734,7 +734,7 @@ static int dax_writeback_one(struct block_device *bdev, ...@@ -734,7 +734,7 @@ static int dax_writeback_one(struct block_device *bdev,
} }
dax_mapping_entry_mkclean(mapping, index, pfn_t_to_pfn(pfn)); dax_mapping_entry_mkclean(mapping, index, pfn_t_to_pfn(pfn));
dax_flush(dax_dev, pgoff, kaddr, size); dax_flush(dax_dev, kaddr, size);
/* /*
* After we have flushed the cache, we can clear the dirty tag. There * After we have flushed the cache, we can clear the dirty tag. There
* cannot be new dirty data in the pfn after the flush has completed as * cannot be new dirty data in the pfn after the flush has completed as
...@@ -929,7 +929,7 @@ int __dax_zero_page_range(struct block_device *bdev, ...@@ -929,7 +929,7 @@ int __dax_zero_page_range(struct block_device *bdev,
return rc; return rc;
} }
memset(kaddr + offset, 0, size); memset(kaddr + offset, 0, size);
dax_flush(dax_dev, pgoff, kaddr + offset, size); dax_flush(dax_dev, kaddr + offset, size);
dax_read_unlock(id); dax_read_unlock(id);
} }
return 0; return 0;
......
...@@ -19,8 +19,6 @@ struct dax_operations { ...@@ -19,8 +19,6 @@ struct dax_operations {
/* copy_from_iter: required operation for fs-dax direct-i/o */ /* copy_from_iter: required operation for fs-dax direct-i/o */
size_t (*copy_from_iter)(struct dax_device *, pgoff_t, void *, size_t, size_t (*copy_from_iter)(struct dax_device *, pgoff_t, void *, size_t,
struct iov_iter *); struct iov_iter *);
/* flush: optional driver-specific cache management after writes */
void (*flush)(struct dax_device *, pgoff_t, void *, size_t);
}; };
extern struct attribute_group dax_attribute_group; extern struct attribute_group dax_attribute_group;
...@@ -90,8 +88,7 @@ long dax_direct_access(struct dax_device *dax_dev, pgoff_t pgoff, long nr_pages, ...@@ -90,8 +88,7 @@ long dax_direct_access(struct dax_device *dax_dev, pgoff_t pgoff, long nr_pages,
void **kaddr, pfn_t *pfn); void **kaddr, pfn_t *pfn);
size_t dax_copy_from_iter(struct dax_device *dax_dev, pgoff_t pgoff, void *addr, size_t dax_copy_from_iter(struct dax_device *dax_dev, pgoff_t pgoff, void *addr,
size_t bytes, struct iov_iter *i); size_t bytes, struct iov_iter *i);
void dax_flush(struct dax_device *dax_dev, pgoff_t pgoff, void *addr, void dax_flush(struct dax_device *dax_dev, void *addr, size_t size);
size_t size);
void dax_write_cache(struct dax_device *dax_dev, bool wc); void dax_write_cache(struct dax_device *dax_dev, bool wc);
bool dax_write_cache_enabled(struct dax_device *dax_dev); bool dax_write_cache_enabled(struct dax_device *dax_dev);
......
...@@ -134,8 +134,6 @@ typedef long (*dm_dax_direct_access_fn) (struct dm_target *ti, pgoff_t pgoff, ...@@ -134,8 +134,6 @@ typedef long (*dm_dax_direct_access_fn) (struct dm_target *ti, pgoff_t pgoff,
long nr_pages, void **kaddr, pfn_t *pfn); long nr_pages, void **kaddr, pfn_t *pfn);
typedef size_t (*dm_dax_copy_from_iter_fn)(struct dm_target *ti, pgoff_t pgoff, typedef size_t (*dm_dax_copy_from_iter_fn)(struct dm_target *ti, pgoff_t pgoff,
void *addr, size_t bytes, struct iov_iter *i); void *addr, size_t bytes, struct iov_iter *i);
typedef void (*dm_dax_flush_fn)(struct dm_target *ti, pgoff_t pgoff, void *addr,
size_t size);
#define PAGE_SECTORS (PAGE_SIZE / 512) #define PAGE_SECTORS (PAGE_SIZE / 512)
void dm_error(const char *message); void dm_error(const char *message);
...@@ -186,7 +184,6 @@ struct target_type { ...@@ -186,7 +184,6 @@ struct target_type {
dm_io_hints_fn io_hints; dm_io_hints_fn io_hints;
dm_dax_direct_access_fn direct_access; dm_dax_direct_access_fn direct_access;
dm_dax_copy_from_iter_fn dax_copy_from_iter; dm_dax_copy_from_iter_fn dax_copy_from_iter;
dm_dax_flush_fn dax_flush;
/* For internal device-mapper use. */ /* For internal device-mapper use. */
struct list_head list; struct list_head list;
...@@ -387,7 +384,7 @@ struct dm_arg { ...@@ -387,7 +384,7 @@ struct dm_arg {
* Validate the next argument, either returning it as *value or, if invalid, * Validate the next argument, either returning it as *value or, if invalid,
* returning -EINVAL and setting *error. * returning -EINVAL and setting *error.
*/ */
int dm_read_arg(struct dm_arg *arg, struct dm_arg_set *arg_set, int dm_read_arg(const struct dm_arg *arg, struct dm_arg_set *arg_set,
unsigned *value, char **error); unsigned *value, char **error);
/* /*
...@@ -395,7 +392,7 @@ int dm_read_arg(struct dm_arg *arg, struct dm_arg_set *arg_set, ...@@ -395,7 +392,7 @@ int dm_read_arg(struct dm_arg *arg, struct dm_arg_set *arg_set,
* arg->min and arg->max further arguments. Either return the size as * arg->min and arg->max further arguments. Either return the size as
* *num_args or, if invalid, return -EINVAL and set *error. * *num_args or, if invalid, return -EINVAL and set *error.
*/ */
int dm_read_arg_group(struct dm_arg *arg, struct dm_arg_set *arg_set, int dm_read_arg_group(const struct dm_arg *arg, struct dm_arg_set *arg_set,
unsigned *num_args, char **error); unsigned *num_args, char **error);
/* /*
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment