Commit b2dbff1b authored by Jens Axboe's avatar Jens Axboe

sbitmap: flush deferred clears for resize and shallow gets

We're missing a deferred clear off the shallow get, which can cause
a hang. Additionally, when we resize the sbitmap, we should also
flush deferred clears for good measure.

Ensure we have full coverage on batch clears, even for paths where
we would not be doing deferred clear. This makes it less error
prone for future additions.
Reported-by: default avatarBart Van Assche <bvanassche@acm.org>
Tested-by: default avatarMing Lei <ming.lei@redhat.com>
Signed-off-by: default avatarJens Axboe <axboe@kernel.dk>
parent 2c4d5356
...@@ -20,6 +20,47 @@ ...@@ -20,6 +20,47 @@
#include <linux/sbitmap.h> #include <linux/sbitmap.h>
#include <linux/seq_file.h> #include <linux/seq_file.h>
/*
* See if we have deferred clears that we can batch move
*/
static inline bool sbitmap_deferred_clear(struct sbitmap *sb, int index)
{
unsigned long mask, val;
unsigned long __maybe_unused flags;
bool ret = false;
/* Silence bogus lockdep warning */
#if defined(CONFIG_LOCKDEP)
local_irq_save(flags);
#endif
spin_lock(&sb->map[index].swap_lock);
if (!sb->map[index].cleared)
goto out_unlock;
/*
* First get a stable cleared mask, setting the old mask to 0.
*/
do {
mask = sb->map[index].cleared;
} while (cmpxchg(&sb->map[index].cleared, mask, 0) != mask);
/*
* Now clear the masked bits in our free word
*/
do {
val = sb->map[index].word;
} while (cmpxchg(&sb->map[index].word, val, val & ~mask) != val);
ret = true;
out_unlock:
spin_unlock(&sb->map[index].swap_lock);
#if defined(CONFIG_LOCKDEP)
local_irq_restore(flags);
#endif
return ret;
}
int sbitmap_init_node(struct sbitmap *sb, unsigned int depth, int shift, int sbitmap_init_node(struct sbitmap *sb, unsigned int depth, int shift,
gfp_t flags, int node) gfp_t flags, int node)
{ {
...@@ -70,6 +111,9 @@ void sbitmap_resize(struct sbitmap *sb, unsigned int depth) ...@@ -70,6 +111,9 @@ void sbitmap_resize(struct sbitmap *sb, unsigned int depth)
unsigned int bits_per_word = 1U << sb->shift; unsigned int bits_per_word = 1U << sb->shift;
unsigned int i; unsigned int i;
for (i = 0; i < sb->map_nr; i++)
sbitmap_deferred_clear(sb, i);
sb->depth = depth; sb->depth = depth;
sb->map_nr = DIV_ROUND_UP(sb->depth, bits_per_word); sb->map_nr = DIV_ROUND_UP(sb->depth, bits_per_word);
...@@ -112,47 +156,6 @@ static int __sbitmap_get_word(unsigned long *word, unsigned long depth, ...@@ -112,47 +156,6 @@ static int __sbitmap_get_word(unsigned long *word, unsigned long depth,
return nr; return nr;
} }
/*
* See if we have deferred clears that we can batch move
*/
static inline bool sbitmap_deferred_clear(struct sbitmap *sb, int index)
{
unsigned long mask, val;
unsigned long __maybe_unused flags;
bool ret = false;
/* Silence bogus lockdep warning */
#if defined(CONFIG_LOCKDEP)
local_irq_save(flags);
#endif
spin_lock(&sb->map[index].swap_lock);
if (!sb->map[index].cleared)
goto out_unlock;
/*
* First get a stable cleared mask, setting the old mask to 0.
*/
do {
mask = sb->map[index].cleared;
} while (cmpxchg(&sb->map[index].cleared, mask, 0) != mask);
/*
* Now clear the masked bits in our free word
*/
do {
val = sb->map[index].word;
} while (cmpxchg(&sb->map[index].word, val, val & ~mask) != val);
ret = true;
out_unlock:
spin_unlock(&sb->map[index].swap_lock);
#if defined(CONFIG_LOCKDEP)
local_irq_restore(flags);
#endif
return ret;
}
static int sbitmap_find_bit_in_index(struct sbitmap *sb, int index, static int sbitmap_find_bit_in_index(struct sbitmap *sb, int index,
unsigned int alloc_hint, bool round_robin) unsigned int alloc_hint, bool round_robin)
{ {
...@@ -215,6 +218,7 @@ int sbitmap_get_shallow(struct sbitmap *sb, unsigned int alloc_hint, ...@@ -215,6 +218,7 @@ int sbitmap_get_shallow(struct sbitmap *sb, unsigned int alloc_hint,
index = SB_NR_TO_INDEX(sb, alloc_hint); index = SB_NR_TO_INDEX(sb, alloc_hint);
for (i = 0; i < sb->map_nr; i++) { for (i = 0; i < sb->map_nr; i++) {
again:
nr = __sbitmap_get_word(&sb->map[index].word, nr = __sbitmap_get_word(&sb->map[index].word,
min(sb->map[index].depth, shallow_depth), min(sb->map[index].depth, shallow_depth),
SB_NR_TO_BIT(sb, alloc_hint), true); SB_NR_TO_BIT(sb, alloc_hint), true);
...@@ -223,6 +227,9 @@ int sbitmap_get_shallow(struct sbitmap *sb, unsigned int alloc_hint, ...@@ -223,6 +227,9 @@ int sbitmap_get_shallow(struct sbitmap *sb, unsigned int alloc_hint,
break; break;
} }
if (sbitmap_deferred_clear(sb, index))
goto again;
/* Jump to next index. */ /* Jump to next index. */
index++; index++;
alloc_hint = index << sb->shift; alloc_hint = index << sb->shift;
...@@ -242,7 +249,7 @@ bool sbitmap_any_bit_set(const struct sbitmap *sb) ...@@ -242,7 +249,7 @@ bool sbitmap_any_bit_set(const struct sbitmap *sb)
unsigned int i; unsigned int i;
for (i = 0; i < sb->map_nr; i++) { for (i = 0; i < sb->map_nr; i++) {
if (sb->map[i].word) if (sb->map[i].word & ~sb->map[i].cleared)
return true; return true;
} }
return false; return false;
...@@ -255,9 +262,10 @@ bool sbitmap_any_bit_clear(const struct sbitmap *sb) ...@@ -255,9 +262,10 @@ bool sbitmap_any_bit_clear(const struct sbitmap *sb)
for (i = 0; i < sb->map_nr; i++) { for (i = 0; i < sb->map_nr; i++) {
const struct sbitmap_word *word = &sb->map[i]; const struct sbitmap_word *word = &sb->map[i];
unsigned long mask = word->word & ~word->cleared;
unsigned long ret; unsigned long ret;
ret = find_first_zero_bit(&word->word, word->depth); ret = find_first_zero_bit(&mask, word->depth);
if (ret < word->depth) if (ret < word->depth)
return true; return true;
} }
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment