bcachefs: Fix btree updates when mixing cached and non cached iterators

There was a bug where bch2_trans_update() would incorrectly delete a
pending update where the new update did not actually overwrite the
existing update, because we were incorrectly using BTREE_ITER_TYPE when
sorting pending btree updates.

This affects the pending patch to use cached iterators for inode
updates.
Signed-off-by: Kent Overstreet <kent.overstreet@gmail.com>
Signed-off-by: Kent Overstreet <kent.overstreet@linux.dev>

fs/bcachefs/btree_cache.c

@@ -949,7 +949,7 @@ struct btree *bch2_btree_node_get_sibling(struct bch_fs *c,
 		 * holding other locks that would cause us to deadlock:
 		 */
 		trans_for_each_iter(trans, linked)
-			if (btree_iter_cmp(iter, linked) < 0)
+			if (btree_iter_lock_cmp(iter, linked) < 0)
 				__bch2_btree_iter_unlock(linked);
 
 		if (sib == btree_prev_sib)

fs/bcachefs/btree_iter.c

@@ -1104,7 +1104,7 @@ static int __btree_iter_traverse_all(struct btree_trans *trans, int ret)
 		sorted[nr_sorted++] = iter->idx;
 
 #define btree_iter_cmp_by_idx(_l, _r)				\
-		btree_iter_cmp(&trans->iters[_l], &trans->iters[_r])
+		btree_iter_lock_cmp(&trans->iters[_l], &trans->iters[_r])
 
 	bubble_sort(sorted, nr_sorted, btree_iter_cmp_by_idx);
 #undef btree_iter_cmp_by_idx

fs/bcachefs/btree_iter.h

@@ -177,8 +177,9 @@ void bch2_btree_iter_set_pos_same_leaf(struct btree_iter *, struct bpos);
 void __bch2_btree_iter_set_pos(struct btree_iter *, struct bpos, bool);
 void bch2_btree_iter_set_pos(struct btree_iter *, struct bpos);
 
-static inline int btree_iter_cmp(const struct btree_iter *l,
-				 const struct btree_iter *r)
+/* Sort order for locking btree iterators: */
+static inline int btree_iter_lock_cmp(const struct btree_iter *l,
+				      const struct btree_iter *r)
 {
 	return   cmp_int(l->btree_id, r->btree_id) ?:
 		-cmp_int(btree_iter_type(l), btree_iter_type(r)) ?:

fs/bcachefs/btree_update_leaf.c

@@ -690,6 +690,13 @@ bch2_trans_commit_get_rw_cold(struct btree_trans *trans)
 	return 0;
 }
 
+static inline int btree_iter_pos_cmp(const struct btree_iter *l,
+				     const struct btree_iter *r)
+{
+	return   cmp_int(l->btree_id, r->btree_id) ?:
+		 bkey_cmp(l->pos, r->pos);
+}
+
 static void bch2_trans_update2(struct btree_trans *trans,
 			       struct btree_iter *iter,
 			       struct bkey_i *insert)
@@ -707,12 +714,12 @@ static void bch2_trans_update2(struct btree_trans *trans,
 	iter->flags |= BTREE_ITER_KEEP_UNTIL_COMMIT;
 
 	trans_for_each_update2(trans, i) {
-		if (btree_iter_cmp(n.iter, i->iter) == 0) {
+		if (btree_iter_pos_cmp(n.iter, i->iter) == 0) {
 			*i = n;
 			return;
 		}
 
-		if (btree_iter_cmp(n.iter, i->iter) <= 0)
+		if (btree_iter_pos_cmp(n.iter, i->iter) <= 0)
 			break;
 	}
 
@@ -996,7 +1003,7 @@ int bch2_trans_update(struct btree_trans *trans, struct btree_iter *iter,
 	 * Pending updates are kept sorted: first, find position of new update:
 	 */
 	trans_for_each_update(trans, i)
-		if (btree_iter_cmp(iter, i->iter) <= 0)
+		if (btree_iter_pos_cmp(iter, i->iter) <= 0)
 			break;
 
 	/*