From 30525f68633740e071c0960c11c4380f1f6851af Mon Sep 17 00:00:00 2001 From: Kent Overstreet <kent.overstreet@gmail.com> Date: Sat, 21 May 2022 13:10:39 -0400 Subject: [PATCH] bcachefs: Fix journal_keys_search() overhead Previously, on every btree_iter_peek() operation we were searching the journal keys, doing a full binary search - which was slow. This patch fixes that by saving our position in the journal keys, so that we only do a full binary search when moving our position backwards or a large jump forwards. Signed-off-by: Kent Overstreet <kent.overstreet@gmail.com> --- fs/bcachefs/btree_iter.c | 28 ++++++++++++++++---- fs/bcachefs/btree_iter.h | 3 +++ fs/bcachefs/btree_types.h | 4 +++ fs/bcachefs/recovery.c | 54 ++++++++++++++++++++++++++------------- fs/bcachefs/recovery.h | 4 +-- 5 files changed, 67 insertions(+), 26 deletions(-) diff --git a/fs/bcachefs/btree_iter.c b/fs/bcachefs/btree_iter.c index b840035dca55..3ce0571651b5 100644 --- a/fs/bcachefs/btree_iter.c +++ b/fs/bcachefs/btree_iter.c @@ -2242,13 +2242,30 @@ static inline struct bkey_i *btree_trans_peek_updates(struct btree_iter *iter) : NULL; } +struct bkey_i *bch2_btree_journal_peek(struct btree_trans *trans, + struct btree_iter *iter, + struct bpos end_pos) +{ + struct bkey_i *k; + + if (bpos_cmp(iter->path->pos, iter->journal_pos) < 0) + iter->journal_idx = 0; + + k = bch2_journal_keys_peek_upto(trans->c, iter->btree_id, + iter->path->level, + iter->path->pos, + end_pos, + &iter->journal_idx); + + iter->journal_pos = k ? k->k.p : end_pos; + return k; +} + static noinline struct bkey_s_c btree_trans_peek_slot_journal(struct btree_trans *trans, struct btree_iter *iter) { - struct bkey_i *k = bch2_journal_keys_peek_slot(trans->c, iter->btree_id, - iter->path->level, - iter->path->pos); + struct bkey_i *k = bch2_btree_journal_peek(trans, iter, iter->path->pos); if (k) { iter->k = k->k; @@ -2264,8 +2281,7 @@ struct bkey_s_c btree_trans_peek_journal(struct btree_trans *trans, struct bkey_s_c k) { struct bkey_i *next_journal = - bch2_journal_keys_peek_upto(trans->c, iter->btree_id, 0, - iter->path->pos, + bch2_btree_journal_peek(trans, iter, k.k ? k.k->p : iter->path->l[0].b->key.k.p); if (next_journal) { @@ -3072,6 +3088,8 @@ static void __bch2_trans_iter_init(struct btree_trans *trans, iter->k.type = KEY_TYPE_deleted; iter->k.p = pos; iter->k.size = 0; + iter->journal_idx = 0; + iter->journal_pos = POS_MIN; iter->path = bch2_path_get(trans, btree_id, iter->pos, locks_want, depth, flags); diff --git a/fs/bcachefs/btree_iter.h b/fs/bcachefs/btree_iter.h index dc6f07492bc9..83587383a41f 100644 --- a/fs/bcachefs/btree_iter.h +++ b/fs/bcachefs/btree_iter.h @@ -175,6 +175,9 @@ struct btree_path *bch2_path_get(struct btree_trans *, enum btree_id, struct bpo unsigned, unsigned, unsigned); inline struct bkey_s_c bch2_btree_path_peek_slot(struct btree_path *, struct bkey *); +struct bkey_i *bch2_btree_journal_peek_slot(struct btree_trans *, + struct btree_iter *, struct bpos); + #ifdef CONFIG_BCACHEFS_DEBUG void bch2_trans_verify_paths(struct btree_trans *); void bch2_trans_verify_locks(struct btree_trans *); diff --git a/fs/bcachefs/btree_types.h b/fs/bcachefs/btree_types.h index 4f359ff79334..82c8c148c4bc 100644 --- a/fs/bcachefs/btree_types.h +++ b/fs/bcachefs/btree_types.h @@ -292,6 +292,10 @@ struct btree_iter { * bch2_btree_iter_next_slot() can correctly advance pos. */ struct bkey k; + + /* BTREE_ITER_WITH_JOURNAL: */ + size_t journal_idx; + struct bpos journal_pos; }; struct btree_key_cache { diff --git a/fs/bcachefs/recovery.c b/fs/bcachefs/recovery.c index 2e782d5d968e..edb04f65a148 100644 --- a/fs/bcachefs/recovery.c +++ b/fs/bcachefs/recovery.c @@ -86,9 +86,9 @@ static inline struct journal_key *idx_to_key(struct journal_keys *keys, size_t i return keys->d + idx_to_pos(keys, idx); } -size_t bch2_journal_key_search(struct journal_keys *keys, - enum btree_id id, unsigned level, - struct bpos pos) +static size_t __bch2_journal_key_search(struct journal_keys *keys, + enum btree_id id, unsigned level, + struct bpos pos) { size_t l = 0, r = keys->nr, m; @@ -106,26 +106,42 @@ size_t bch2_journal_key_search(struct journal_keys *keys, BUG_ON(l && __journal_key_cmp(id, level, pos, idx_to_key(keys, l - 1)) <= 0); - return idx_to_pos(keys, l); + return l; +} + +static size_t bch2_journal_key_search(struct journal_keys *keys, + enum btree_id id, unsigned level, + struct bpos pos) +{ + return idx_to_pos(keys, __bch2_journal_key_search(keys, id, level, pos)); } struct bkey_i *bch2_journal_keys_peek_upto(struct bch_fs *c, enum btree_id btree_id, unsigned level, struct bpos pos, - struct bpos end_pos) + struct bpos end_pos, size_t *idx) { struct journal_keys *keys = &c->journal_keys; - size_t idx = bch2_journal_key_search(keys, btree_id, level, pos); - - while (idx < keys->size && - keys->d[idx].btree_id == btree_id && - keys->d[idx].level == level && - bpos_cmp(keys->d[idx].k->k.p, end_pos) <= 0) { - if (!keys->d[idx].overwritten) - return keys->d[idx].k; - - idx++; - if (idx == keys->gap) - idx += keys->size - keys->nr; + unsigned iters = 0; + struct journal_key *k; +search: + if (!*idx) + *idx = __bch2_journal_key_search(keys, btree_id, level, pos); + + while (*idx < keys->nr && + (k = idx_to_key(keys, *idx), + k->btree_id == btree_id && + k->level == level && + bpos_cmp(k->k->k.p, end_pos) <= 0)) { + if (bpos_cmp(k->k->k.p, pos) >= 0 && + !k->overwritten) + return k->k; + + (*idx)++; + iters++; + if (iters == 10) { + *idx = 0; + goto search; + } } return NULL; @@ -134,7 +150,9 @@ struct bkey_i *bch2_journal_keys_peek_upto(struct bch_fs *c, enum btree_id btree struct bkey_i *bch2_journal_keys_peek_slot(struct bch_fs *c, enum btree_id btree_id, unsigned level, struct bpos pos) { - return bch2_journal_keys_peek_upto(c, btree_id, level, pos, pos); + size_t idx = 0; + + return bch2_journal_keys_peek_upto(c, btree_id, level, pos, pos, &idx); } static void journal_iters_fix(struct bch_fs *c) diff --git a/fs/bcachefs/recovery.h b/fs/bcachefs/recovery.h index e05aac64185d..52db06b29310 100644 --- a/fs/bcachefs/recovery.h +++ b/fs/bcachefs/recovery.h @@ -28,10 +28,8 @@ struct btree_and_journal_iter { } last; }; -size_t bch2_journal_key_search(struct journal_keys *, enum btree_id, - unsigned, struct bpos); struct bkey_i *bch2_journal_keys_peek_upto(struct bch_fs *, enum btree_id, - unsigned, struct bpos, struct bpos); + unsigned, struct bpos, struct bpos, size_t *); struct bkey_i *bch2_journal_keys_peek_slot(struct bch_fs *, enum btree_id, unsigned, struct bpos); -- 2.30.9