Commit 3db2776d authored by David Jeffery's avatar David Jeffery Committed by Mike Snitzer

dm snapshot: improve performance by switching out_of_order_list to rbtree

copy_complete()'s processing of out_of_order_list can result in
quadratic complexity in the worst case.  As such it was the source of
consuming too much cpu and the source of significant loss in
performance.

Fix this by converting out_of_order_list to an rbtree.  This improved
a dm-snapshot test copy workload from 32 seconds to 4 seconds.
Signed-off-by: default avatarDavid Jeffery <djeffery@redhat.com>
Signed-off-by: default avatarMikulas Patocka <mpatocka@redhat.com>
Tested-by: default avatarBrett Hull <bhull@redhat.com>
Signed-off-by: default avatarMike Snitzer <snitzer@redhat.com>
parent 784c9a29
...@@ -85,7 +85,7 @@ struct dm_snapshot { ...@@ -85,7 +85,7 @@ struct dm_snapshot {
* A list of pending exceptions that completed out of order. * A list of pending exceptions that completed out of order.
* Protected by kcopyd single-threaded callback. * Protected by kcopyd single-threaded callback.
*/ */
struct list_head out_of_order_list; struct rb_root out_of_order_tree;
mempool_t pending_pool; mempool_t pending_pool;
...@@ -200,7 +200,7 @@ struct dm_snap_pending_exception { ...@@ -200,7 +200,7 @@ struct dm_snap_pending_exception {
/* A sequence number, it is used for in-order completion. */ /* A sequence number, it is used for in-order completion. */
sector_t exception_sequence; sector_t exception_sequence;
struct list_head out_of_order_entry; struct rb_node out_of_order_node;
/* /*
* For writing a complete chunk, bypassing the copy. * For writing a complete chunk, bypassing the copy.
...@@ -1173,7 +1173,7 @@ static int snapshot_ctr(struct dm_target *ti, unsigned int argc, char **argv) ...@@ -1173,7 +1173,7 @@ static int snapshot_ctr(struct dm_target *ti, unsigned int argc, char **argv)
atomic_set(&s->pending_exceptions_count, 0); atomic_set(&s->pending_exceptions_count, 0);
s->exception_start_sequence = 0; s->exception_start_sequence = 0;
s->exception_complete_sequence = 0; s->exception_complete_sequence = 0;
INIT_LIST_HEAD(&s->out_of_order_list); s->out_of_order_tree = RB_ROOT;
mutex_init(&s->lock); mutex_init(&s->lock);
INIT_LIST_HEAD(&s->list); INIT_LIST_HEAD(&s->list);
spin_lock_init(&s->pe_lock); spin_lock_init(&s->pe_lock);
...@@ -1539,28 +1539,41 @@ static void copy_callback(int read_err, unsigned long write_err, void *context) ...@@ -1539,28 +1539,41 @@ static void copy_callback(int read_err, unsigned long write_err, void *context)
pe->copy_error = read_err || write_err; pe->copy_error = read_err || write_err;
if (pe->exception_sequence == s->exception_complete_sequence) { if (pe->exception_sequence == s->exception_complete_sequence) {
struct rb_node *next;
s->exception_complete_sequence++; s->exception_complete_sequence++;
complete_exception(pe); complete_exception(pe);
while (!list_empty(&s->out_of_order_list)) { next = rb_first(&s->out_of_order_tree);
pe = list_entry(s->out_of_order_list.next, while (next) {
struct dm_snap_pending_exception, out_of_order_entry); pe = rb_entry(next, struct dm_snap_pending_exception,
out_of_order_node);
if (pe->exception_sequence != s->exception_complete_sequence) if (pe->exception_sequence != s->exception_complete_sequence)
break; break;
next = rb_next(next);
s->exception_complete_sequence++; s->exception_complete_sequence++;
list_del(&pe->out_of_order_entry); rb_erase(&pe->out_of_order_node, &s->out_of_order_tree);
complete_exception(pe); complete_exception(pe);
cond_resched();
} }
} else { } else {
struct list_head *lh; struct rb_node *parent = NULL;
struct rb_node **p = &s->out_of_order_tree.rb_node;
struct dm_snap_pending_exception *pe2; struct dm_snap_pending_exception *pe2;
list_for_each_prev(lh, &s->out_of_order_list) { while (*p) {
pe2 = list_entry(lh, struct dm_snap_pending_exception, out_of_order_entry); pe2 = rb_entry(*p, struct dm_snap_pending_exception, out_of_order_node);
if (pe2->exception_sequence < pe->exception_sequence) parent = *p;
break;
BUG_ON(pe->exception_sequence == pe2->exception_sequence);
if (pe->exception_sequence < pe2->exception_sequence)
p = &((*p)->rb_left);
else
p = &((*p)->rb_right);
} }
list_add(&pe->out_of_order_entry, lh);
rb_link_node(&pe->out_of_order_node, parent, p);
rb_insert_color(&pe->out_of_order_node, &s->out_of_order_tree);
} }
} }
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment