Commit 08eee69f authored by Minchan Kim's avatar Minchan Kim Committed by Linus Torvalds

zram: remove init_lock in zram_make_request

Admin could reset zram during I/O operation going on so we have used
zram->init_lock as read-side lock in I/O path to prevent sudden zram
meta freeing.

However, the init_lock is really troublesome.  We can't do call
zram_meta_alloc under init_lock due to lockdep splat because
zram_rw_page is one of the function under reclaim path and hold it as
read_lock while other places in process context hold it as write_lock.
So, we have used allocation out of the lock to avoid lockdep warn but
it's not good for readability and fainally, I met another lockdep splat
between init_lock and cpu_hotplug from kmem_cache_destroy during working
zsmalloc compaction.  :(

Yes, the ideal is to remove horrible init_lock of zram in rw path.  This
patch removes it in rw path and instead, add atomic refcount for meta
lifetime management and completion to free meta in process context.
It's important to free meta in process context because some of resource
destruction needs mutex lock, which could be held if we releases the
resource in reclaim context so it's deadlock, again.

As a bonus, we could remove init_done check in rw path because
zram_meta_get will do a role for it, instead.
Signed-off-by: default avatarSergey Senozhatsky <sergey.senozhatsky@gmail.com>
Signed-off-by: default avatarMinchan Kim <minchan@kernel.org>
Cc: Nitin Gupta <ngupta@vflare.org>
Cc: Jerome Marchand <jmarchan@redhat.com>
Cc: Ganesh Mahendran <opensource.ganesh@gmail.com>
Signed-off-by: default avatarAndrew Morton <akpm@linux-foundation.org>
Signed-off-by: default avatarLinus Torvalds <torvalds@linux-foundation.org>
parent 2b269ce6
...@@ -53,9 +53,9 @@ static ssize_t name##_show(struct device *d, \ ...@@ -53,9 +53,9 @@ static ssize_t name##_show(struct device *d, \
} \ } \
static DEVICE_ATTR_RO(name); static DEVICE_ATTR_RO(name);
static inline int init_done(struct zram *zram) static inline bool init_done(struct zram *zram)
{ {
return zram->meta != NULL; return zram->disksize;
} }
static inline struct zram *dev_to_zram(struct device *dev) static inline struct zram *dev_to_zram(struct device *dev)
...@@ -356,6 +356,18 @@ static struct zram_meta *zram_meta_alloc(u64 disksize) ...@@ -356,6 +356,18 @@ static struct zram_meta *zram_meta_alloc(u64 disksize)
return NULL; return NULL;
} }
static inline bool zram_meta_get(struct zram *zram)
{
if (atomic_inc_not_zero(&zram->refcount))
return true;
return false;
}
static inline void zram_meta_put(struct zram *zram)
{
atomic_dec(&zram->refcount);
}
static void update_position(u32 *index, int *offset, struct bio_vec *bvec) static void update_position(u32 *index, int *offset, struct bio_vec *bvec)
{ {
if (*offset + bvec->bv_len >= PAGE_SIZE) if (*offset + bvec->bv_len >= PAGE_SIZE)
...@@ -717,6 +729,10 @@ static void zram_bio_discard(struct zram *zram, u32 index, ...@@ -717,6 +729,10 @@ static void zram_bio_discard(struct zram *zram, u32 index,
static void zram_reset_device(struct zram *zram) static void zram_reset_device(struct zram *zram)
{ {
struct zram_meta *meta;
struct zcomp *comp;
u64 disksize;
down_write(&zram->init_lock); down_write(&zram->init_lock);
zram->limit_pages = 0; zram->limit_pages = 0;
...@@ -726,16 +742,31 @@ static void zram_reset_device(struct zram *zram) ...@@ -726,16 +742,31 @@ static void zram_reset_device(struct zram *zram)
return; return;
} }
zcomp_destroy(zram->comp); meta = zram->meta;
zram->max_comp_streams = 1; comp = zram->comp;
zram_meta_free(zram->meta, zram->disksize); disksize = zram->disksize;
zram->meta = NULL; /*
* Refcount will go down to 0 eventually and r/w handler
* cannot handle further I/O so it will bail out by
* check zram_meta_get.
*/
zram_meta_put(zram);
/*
* We want to free zram_meta in process context to avoid
* deadlock between reclaim path and any other locks.
*/
wait_event(zram->io_done, atomic_read(&zram->refcount) == 0);
/* Reset stats */ /* Reset stats */
memset(&zram->stats, 0, sizeof(zram->stats)); memset(&zram->stats, 0, sizeof(zram->stats));
zram->disksize = 0; zram->disksize = 0;
zram->max_comp_streams = 1;
set_capacity(zram->disk, 0); set_capacity(zram->disk, 0);
up_write(&zram->init_lock); up_write(&zram->init_lock);
/* I/O operation under all of CPU are done so let's free */
zram_meta_free(meta, disksize);
zcomp_destroy(comp);
} }
static ssize_t disksize_store(struct device *dev, static ssize_t disksize_store(struct device *dev,
...@@ -771,6 +802,8 @@ static ssize_t disksize_store(struct device *dev, ...@@ -771,6 +802,8 @@ static ssize_t disksize_store(struct device *dev,
goto out_destroy_comp; goto out_destroy_comp;
} }
init_waitqueue_head(&zram->io_done);
atomic_set(&zram->refcount, 1);
zram->meta = meta; zram->meta = meta;
zram->comp = comp; zram->comp = comp;
zram->disksize = disksize; zram->disksize = disksize;
...@@ -901,23 +934,21 @@ static void zram_make_request(struct request_queue *queue, struct bio *bio) ...@@ -901,23 +934,21 @@ static void zram_make_request(struct request_queue *queue, struct bio *bio)
{ {
struct zram *zram = queue->queuedata; struct zram *zram = queue->queuedata;
down_read(&zram->init_lock); if (unlikely(!zram_meta_get(zram)))
if (unlikely(!init_done(zram)))
goto error; goto error;
if (!valid_io_request(zram, bio->bi_iter.bi_sector, if (!valid_io_request(zram, bio->bi_iter.bi_sector,
bio->bi_iter.bi_size)) { bio->bi_iter.bi_size)) {
atomic64_inc(&zram->stats.invalid_io); atomic64_inc(&zram->stats.invalid_io);
goto error; goto put_zram;
} }
__zram_make_request(zram, bio); __zram_make_request(zram, bio);
up_read(&zram->init_lock); zram_meta_put(zram);
return; return;
put_zram:
zram_meta_put(zram);
error: error:
up_read(&zram->init_lock);
bio_io_error(bio); bio_io_error(bio);
} }
...@@ -939,21 +970,19 @@ static void zram_slot_free_notify(struct block_device *bdev, ...@@ -939,21 +970,19 @@ static void zram_slot_free_notify(struct block_device *bdev,
static int zram_rw_page(struct block_device *bdev, sector_t sector, static int zram_rw_page(struct block_device *bdev, sector_t sector,
struct page *page, int rw) struct page *page, int rw)
{ {
int offset, err; int offset, err = -EIO;
u32 index; u32 index;
struct zram *zram; struct zram *zram;
struct bio_vec bv; struct bio_vec bv;
zram = bdev->bd_disk->private_data; zram = bdev->bd_disk->private_data;
if (unlikely(!zram_meta_get(zram)))
goto out;
if (!valid_io_request(zram, sector, PAGE_SIZE)) { if (!valid_io_request(zram, sector, PAGE_SIZE)) {
atomic64_inc(&zram->stats.invalid_io); atomic64_inc(&zram->stats.invalid_io);
return -EINVAL; err = -EINVAL;
} goto put_zram;
down_read(&zram->init_lock);
if (unlikely(!init_done(zram))) {
err = -EIO;
goto out_unlock;
} }
index = sector >> SECTORS_PER_PAGE_SHIFT; index = sector >> SECTORS_PER_PAGE_SHIFT;
...@@ -964,8 +993,9 @@ static int zram_rw_page(struct block_device *bdev, sector_t sector, ...@@ -964,8 +993,9 @@ static int zram_rw_page(struct block_device *bdev, sector_t sector,
bv.bv_offset = 0; bv.bv_offset = 0;
err = zram_bvec_rw(zram, &bv, index, offset, rw); err = zram_bvec_rw(zram, &bv, index, offset, rw);
out_unlock: put_zram:
up_read(&zram->init_lock); zram_meta_put(zram);
out:
/* /*
* If I/O fails, just return error(ie, non-zero) without * If I/O fails, just return error(ie, non-zero) without
* calling page_endio. * calling page_endio.
......
...@@ -100,24 +100,26 @@ struct zram_meta { ...@@ -100,24 +100,26 @@ struct zram_meta {
struct zram { struct zram {
struct zram_meta *meta; struct zram_meta *meta;
struct zcomp *comp;
struct request_queue *queue; struct request_queue *queue;
struct gendisk *disk; struct gendisk *disk;
struct zcomp *comp; /* Prevent concurrent execution of device init */
/* Prevent concurrent execution of device init, reset and R/W request */
struct rw_semaphore init_lock; struct rw_semaphore init_lock;
/* /*
* This is the limit on amount of *uncompressed* worth of data * the number of pages zram can consume for storing compressed data
* we can store in a disk.
*/ */
u64 disksize; /* bytes */ unsigned long limit_pages;
int max_comp_streams; int max_comp_streams;
struct zram_stats stats; struct zram_stats stats;
atomic_t refcount; /* refcount for zram_meta */
/* wait all IO under all of cpu are done */
wait_queue_head_t io_done;
/* /*
* the number of pages zram can consume for storing compressed data * This is the limit on amount of *uncompressed* worth of data
* we can store in a disk.
*/ */
unsigned long limit_pages; u64 disksize; /* bytes */
char compressor[10]; char compressor[10];
}; };
#endif #endif
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment