bcachefs: Throttle updates when btree key cache is too dirty

This is needed to ensure we don't deadlock because journal reclaim and
thus memory reclaim isn't making forward progress.
Signed-off-by: Kent Overstreet <kent.overstreet@gmail.com>
Signed-off-by: Kent Overstreet <kent.overstreet@linux.dev>

fs/bcachefs/btree_key_cache.h

@@ -5,11 +5,20 @@ static inline size_t bch2_nr_btree_keys_need_flush(struct bch_fs *c)
 {
 	size_t nr_dirty = READ_ONCE(c->btree_key_cache.nr_dirty);
 	size_t nr_keys = READ_ONCE(c->btree_key_cache.nr_dirty);
-	size_t max_dirty = 1024 + (nr_keys * 3) / 4;
+	size_t max_dirty = 4096 + nr_keys  / 2;
 
 	return max_t(ssize_t, 0, nr_dirty - max_dirty);
 }
 
+static inline bool bch2_btree_key_cache_must_wait(struct bch_fs *c)
+{
+	size_t nr_dirty = READ_ONCE(c->btree_key_cache.nr_dirty);
+	size_t nr_keys = READ_ONCE(c->btree_key_cache.nr_dirty);
+	size_t max_dirty = 4096 + (nr_keys * 3) / 4;
+
+	return nr_dirty > max_dirty;
+}
+
 struct bkey_cached *
 bch2_btree_key_cache_find(struct bch_fs *, enum btree_id, struct bpos);
 

fs/bcachefs/btree_types.h

@@ -649,6 +649,7 @@ enum btree_insert_ret {
 	BTREE_INSERT_ENOSPC,
 	BTREE_INSERT_NEED_MARK_REPLICAS,
 	BTREE_INSERT_NEED_JOURNAL_RES,
+	BTREE_INSERT_NEED_JOURNAL_RECLAIM,
 };
 
 enum btree_gc_coalesce_fail_reason {

fs/bcachefs/btree_update_leaf.c

@@ -286,6 +286,10 @@ btree_key_can_insert_cached(struct btree_trans *trans,
 
 	BUG_ON(iter->level);
 
+	if (!test_bit(BKEY_CACHED_DIRTY, &ck->flags) &&
+	    bch2_btree_key_cache_must_wait(trans->c))
+		return BTREE_INSERT_NEED_JOURNAL_RECLAIM;
+
 	if (u64s <= ck->u64s)
 		return BTREE_INSERT_OK;
 
@@ -652,6 +656,21 @@ int bch2_trans_commit_error(struct btree_trans *trans,
 		trace_trans_restart_journal_res_get(trans->ip);
 		ret = -EINTR;
 		break;
+	case BTREE_INSERT_NEED_JOURNAL_RECLAIM:
+		bch2_trans_unlock(trans);
+
+		while (bch2_btree_key_cache_must_wait(c)) {
+			mutex_lock(&c->journal.reclaim_lock);
+			bch2_journal_reclaim(&c->journal);
+			mutex_unlock(&c->journal.reclaim_lock);
+		}
+
+		if (bch2_trans_relock(trans))
+			return 0;
+
+		trace_trans_restart_journal_reclaim(trans->ip);
+		ret = -EINTR;
+		break;
 	default:
 		BUG_ON(ret >= 0);
 		break;

fs/bcachefs/trace.h

@@ -681,6 +681,11 @@ DEFINE_EVENT(transaction_restart,	trans_restart_journal_preres_get,
 	TP_ARGS(ip)
 );
 
+DEFINE_EVENT(transaction_restart,	trans_restart_journal_reclaim,
+	TP_PROTO(unsigned long ip),
+	TP_ARGS(ip)
+);
+
 DEFINE_EVENT(transaction_restart,	trans_restart_mark_replicas,
 	TP_PROTO(unsigned long ip),
 	TP_ARGS(ip)