buckets.c 58.7 KB
Newer Older
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66
// SPDX-License-Identifier: GPL-2.0
/*
 * Code for manipulating bucket marks for garbage collection.
 *
 * Copyright 2014 Datera, Inc.
 *
 * Bucket states:
 * - free bucket: mark == 0
 *   The bucket contains no data and will not be read
 *
 * - allocator bucket: owned_by_allocator == 1
 *   The bucket is on a free list, or it is an open bucket
 *
 * - cached bucket: owned_by_allocator == 0 &&
 *                  dirty_sectors == 0 &&
 *                  cached_sectors > 0
 *   The bucket contains data but may be safely discarded as there are
 *   enough replicas of the data on other cache devices, or it has been
 *   written back to the backing device
 *
 * - dirty bucket: owned_by_allocator == 0 &&
 *                 dirty_sectors > 0
 *   The bucket contains data that we must not discard (either only copy,
 *   or one of the 'main copies' for data requiring multiple replicas)
 *
 * - metadata bucket: owned_by_allocator == 0 && is_metadata == 1
 *   This is a btree node, journal or gen/prio bucket
 *
 * Lifecycle:
 *
 * bucket invalidated => bucket on freelist => open bucket =>
 *     [dirty bucket =>] cached bucket => bucket invalidated => ...
 *
 * Note that cache promotion can skip the dirty bucket step, as data
 * is copied from a deeper tier to a shallower tier, onto a cached
 * bucket.
 * Note also that a cached bucket can spontaneously become dirty --
 * see below.
 *
 * Only a traversal of the key space can determine whether a bucket is
 * truly dirty or cached.
 *
 * Transitions:
 *
 * - free => allocator: bucket was invalidated
 * - cached => allocator: bucket was invalidated
 *
 * - allocator => dirty: open bucket was filled up
 * - allocator => cached: open bucket was filled up
 * - allocator => metadata: metadata was allocated
 *
 * - dirty => cached: dirty sectors were copied to a deeper tier
 * - dirty => free: dirty sectors were overwritten or moved (copy gc)
 * - cached => free: cached sectors were overwritten
 *
 * - metadata => free: metadata was freed
 *
 * Oddities:
 * - cached => dirty: a device was removed so formerly replicated data
 *                    is no longer sufficiently replicated
 * - free => cached: cannot happen
 * - free => dirty: cannot happen
 * - free => metadata: cannot happen
 */

#include "bcachefs.h"
67
#include "alloc_background.h"
68
#include "bset.h"
69
#include "btree_gc.h"
70
#include "btree_update.h"
71
#include "buckets.h"
72
#include "ec.h"
73 74
#include "error.h"
#include "movinggc.h"
75
#include "replicas.h"
76 77 78 79
#include "trace.h"

#include <linux/preempt.h>

80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99
static inline void fs_usage_data_type_to_base(struct bch_fs_usage *fs_usage,
					      enum bch_data_type data_type,
					      s64 sectors)
{
	switch (data_type) {
	case BCH_DATA_btree:
		fs_usage->btree		+= sectors;
		break;
	case BCH_DATA_user:
	case BCH_DATA_parity:
		fs_usage->data		+= sectors;
		break;
	case BCH_DATA_cached:
		fs_usage->cached	+= sectors;
		break;
	default:
		break;
	}
}

100 101 102 103 104 105
/*
 * Clear journal_seq_valid for buckets for which it's not needed, to prevent
 * wraparound:
 */
void bch2_bucket_seq_cleanup(struct bch_fs *c)
{
106
	u64 journal_seq = atomic64_read(&c->journal.seq);
107 108 109 110 111 112 113
	u16 last_seq_ondisk = c->journal.last_seq_ondisk;
	struct bch_dev *ca;
	struct bucket_array *buckets;
	struct bucket *g;
	struct bucket_mark m;
	unsigned i;

114 115 116 117 118 119
	if (journal_seq - c->last_bucket_seq_cleanup <
	    (1U << (BUCKET_JOURNAL_SEQ_BITS - 2)))
		return;

	c->last_bucket_seq_cleanup = journal_seq;

120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136
	for_each_member_device(ca, c, i) {
		down_read(&ca->bucket_lock);
		buckets = bucket_array(ca);

		for_each_bucket(g, buckets) {
			bucket_cmpxchg(g, m, ({
				if (!m.journal_seq_valid ||
				    bucket_needs_journal_commit(m, last_seq_ondisk))
					break;

				m.journal_seq_valid = 0;
			}));
		}
		up_read(&ca->bucket_lock);
	}
}

137 138 139
void bch2_fs_usage_initialize(struct bch_fs *c)
{
	struct bch_fs_usage *usage;
140
	struct bch_dev *ca;
141
	unsigned i;
142 143

	percpu_down_write(&c->mark_lock);
144 145 146 147
	usage = c->usage_base;

	for (i = 0; i < ARRAY_SIZE(c->usage); i++)
		bch2_fs_usage_acc_to_base(c, i);
148

149
	for (i = 0; i < BCH_REPLICAS_MAX; i++)
150
		usage->reserved += usage->persistent_reserved[i];
151

152 153 154 155
	for (i = 0; i < c->replicas.nr; i++) {
		struct bch_replicas_entry *e =
			cpu_replicas_entry(&c->replicas, i);

156
		fs_usage_data_type_to_base(usage, e->data_type, usage->replicas[i]);
157 158
	}

159 160 161 162 163 164 165 166
	for_each_member_device(ca, c, i) {
		struct bch_dev_usage dev = bch2_dev_usage_read(ca);

		usage->hidden += (dev.d[BCH_DATA_sb].buckets +
				  dev.d[BCH_DATA_journal].buckets) *
			ca->mi.bucket_size;
	}

167 168 169
	percpu_up_write(&c->mark_lock);
}

170 171 172 173 174 175 176 177 178
static inline struct bch_dev_usage *dev_usage_ptr(struct bch_dev *ca,
						  unsigned journal_seq,
						  bool gc)
{
	return this_cpu_ptr(gc
			    ? ca->usage_gc
			    : ca->usage[journal_seq & JOURNAL_BUF_MASK]);
}

179
struct bch_dev_usage bch2_dev_usage_read(struct bch_dev *ca)
180
{
181
	struct bch_fs *c = ca->fs;
182
	struct bch_dev_usage ret;
183
	unsigned seq, i, u64s = dev_usage_u64s();
184

185 186 187 188 189 190
	do {
		seq = read_seqcount_begin(&c->usage_lock);
		memcpy(&ret, ca->usage_base, u64s * sizeof(u64));
		for (i = 0; i < ARRAY_SIZE(ca->usage); i++)
			acc_u64s_percpu((u64 *) &ret, (u64 __percpu *) ca->usage[i], u64s);
	} while (read_seqcount_retry(&c->usage_lock, seq));
191 192

	return ret;
193 194
}

195 196 197
static inline struct bch_fs_usage *fs_usage_ptr(struct bch_fs *c,
						unsigned journal_seq,
						bool gc)
198
{
199 200
	return this_cpu_ptr(gc
			    ? c->usage_gc
201
			    : c->usage[journal_seq & JOURNAL_BUF_MASK]);
202 203 204 205 206
}

u64 bch2_fs_usage_read_one(struct bch_fs *c, u64 *v)
{
	ssize_t offset = v - (u64 *) c->usage_base;
207
	unsigned i, seq;
208 209 210 211 212 213 214
	u64 ret;

	BUG_ON(offset < 0 || offset >= fs_usage_u64s(c));
	percpu_rwsem_assert_held(&c->mark_lock);

	do {
		seq = read_seqcount_begin(&c->usage_lock);
215 216 217 218
		ret = *v;

		for (i = 0; i < ARRAY_SIZE(c->usage); i++)
			ret += percpu_u64_get((u64 __percpu *) c->usage[i] + offset);
219 220 221 222 223 224 225 226
	} while (read_seqcount_retry(&c->usage_lock, seq));

	return ret;
}

struct bch_fs_usage_online *bch2_fs_usage_read(struct bch_fs *c)
{
	struct bch_fs_usage_online *ret;
227 228 229 230 231
	unsigned seq, i, v, u64s = fs_usage_u64s(c);
retry:
	ret = kmalloc(u64s * sizeof(u64), GFP_NOFS);
	if (unlikely(!ret))
		return NULL;
232 233 234

	percpu_down_read(&c->mark_lock);

235 236 237
	v = fs_usage_u64s(c);
	if (unlikely(u64s != v)) {
		u64s = v;
238
		percpu_up_read(&c->mark_lock);
239 240
		kfree(ret);
		goto retry;
241 242
	}

243 244 245 246
	ret->online_reserved = percpu_u64_get(c->online_reserved);

	do {
		seq = read_seqcount_begin(&c->usage_lock);
247
		memcpy(&ret->u, c->usage_base, u64s * sizeof(u64));
248 249 250
		for (i = 0; i < ARRAY_SIZE(c->usage); i++)
			acc_u64s_percpu((u64 *) &ret->u, (u64 __percpu *) c->usage[i], u64s);
	} while (read_seqcount_retry(&c->usage_lock, seq));
251 252

	return ret;
253 254
}

255 256
void bch2_fs_usage_acc_to_base(struct bch_fs *c, unsigned idx)
{
257 258
	struct bch_dev *ca;
	unsigned i, u64s = fs_usage_u64s(c);
259 260 261 262 263 264 265 266 267 268

	BUG_ON(idx >= ARRAY_SIZE(c->usage));

	preempt_disable();
	write_seqcount_begin(&c->usage_lock);

	acc_u64s_percpu((u64 *) c->usage_base,
			(u64 __percpu *) c->usage[idx], u64s);
	percpu_memset(c->usage[idx], 0, u64s * sizeof(u64));

269 270 271 272 273 274 275 276 277 278
	rcu_read_lock();
	for_each_member_device_rcu(ca, c, i, NULL) {
		u64s = dev_usage_u64s();

		acc_u64s_percpu((u64 *) ca->usage_base,
				(u64 __percpu *) ca->usage[idx], u64s);
		percpu_memset(ca->usage[idx], 0, u64s * sizeof(u64));
	}
	rcu_read_unlock();

279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 321
	write_seqcount_end(&c->usage_lock);
	preempt_enable();
}

void bch2_fs_usage_to_text(struct printbuf *out,
			   struct bch_fs *c,
			   struct bch_fs_usage_online *fs_usage)
{
	unsigned i;

	pr_buf(out, "capacity:\t\t\t%llu\n", c->capacity);

	pr_buf(out, "hidden:\t\t\t\t%llu\n",
	       fs_usage->u.hidden);
	pr_buf(out, "data:\t\t\t\t%llu\n",
	       fs_usage->u.data);
	pr_buf(out, "cached:\t\t\t\t%llu\n",
	       fs_usage->u.cached);
	pr_buf(out, "reserved:\t\t\t%llu\n",
	       fs_usage->u.reserved);
	pr_buf(out, "nr_inodes:\t\t\t%llu\n",
	       fs_usage->u.nr_inodes);
	pr_buf(out, "online reserved:\t\t%llu\n",
	       fs_usage->online_reserved);

	for (i = 0;
	     i < ARRAY_SIZE(fs_usage->u.persistent_reserved);
	     i++) {
		pr_buf(out, "%u replicas:\n", i + 1);
		pr_buf(out, "\treserved:\t\t%llu\n",
		       fs_usage->u.persistent_reserved[i]);
	}

	for (i = 0; i < c->replicas.nr; i++) {
		struct bch_replicas_entry *e =
			cpu_replicas_entry(&c->replicas, i);

		pr_buf(out, "\t");
		bch2_replicas_entry_to_text(out, e);
		pr_buf(out, ":\t%llu\n", fs_usage->u.replicas[i]);
	}
}

322 323 324 325 326 327 328 329 330
#define RESERVE_FACTOR	6

static u64 reserve_factor(u64 r)
{
	return r + (round_up(r, (1 << RESERVE_FACTOR)) >> RESERVE_FACTOR);
}

static u64 avail_factor(u64 r)
{
331
	return div_u64(r << RESERVE_FACTOR, (1 << RESERVE_FACTOR) + 1);
332 333
}

334
u64 bch2_fs_sectors_used(struct bch_fs *c, struct bch_fs_usage_online *fs_usage)
335
{
336 337 338 339
	return min(fs_usage->u.hidden +
		   fs_usage->u.btree +
		   fs_usage->u.data +
		   reserve_factor(fs_usage->u.reserved +
340
				  fs_usage->online_reserved),
341
		   c->capacity);
342 343
}

344 345 346 347 348 349 350
static struct bch_fs_usage_short
__bch2_fs_usage_read_short(struct bch_fs *c)
{
	struct bch_fs_usage_short ret;
	u64 data, reserved;

	ret.capacity = c->capacity -
351
		bch2_fs_usage_read_one(c, &c->usage_base->hidden);
352

353 354 355 356
	data		= bch2_fs_usage_read_one(c, &c->usage_base->data) +
		bch2_fs_usage_read_one(c, &c->usage_base->btree);
	reserved	= bch2_fs_usage_read_one(c, &c->usage_base->reserved) +
		percpu_u64_get(c->online_reserved);
357 358 359 360

	ret.used	= min(ret.capacity, data + reserve_factor(reserved));
	ret.free	= ret.capacity - ret.used;

361
	ret.nr_inodes	= bch2_fs_usage_read_one(c, &c->usage_base->nr_inodes);
362 363 364 365

	return ret;
}

366 367 368 369 370
struct bch_fs_usage_short
bch2_fs_usage_read_short(struct bch_fs *c)
{
	struct bch_fs_usage_short ret;

371 372 373
	percpu_down_read(&c->mark_lock);
	ret = __bch2_fs_usage_read_short(c);
	percpu_up_read(&c->mark_lock);
374 375

	return ret;
376 377 378 379 380 381 382
}

static inline int is_unavailable_bucket(struct bucket_mark m)
{
	return !is_available_bucket(m);
}

383 384
static inline int bucket_sectors_fragmented(struct bch_dev *ca,
					    struct bucket_mark m)
385
{
386 387 388
	return bucket_sectors_used(m)
		? max(0, (int) ca->mi.bucket_size - (int) bucket_sectors_used(m))
		: 0;
389 390
}

391 392 393 394 395
static inline int is_stripe_data_bucket(struct bucket_mark m)
{
	return m.stripe && m.data_type != BCH_DATA_parity;
}

396 397 398
static inline enum bch_data_type bucket_type(struct bucket_mark m)
{
	return m.cached_sectors && !m.dirty_sectors
399
		? BCH_DATA_cached
400 401 402
		: m.data_type;
}

403
static bool bucket_became_unavailable(struct bucket_mark old,
404 405 406
				      struct bucket_mark new)
{
	return is_available_bucket(old) &&
407
	       !is_available_bucket(new);
408 409
}

410 411 412 413 414
static inline void account_bucket(struct bch_fs_usage *fs_usage,
				  struct bch_dev_usage *dev_usage,
				  enum bch_data_type type,
				  int nr, s64 size)
{
415
	if (type == BCH_DATA_sb || type == BCH_DATA_journal)
416
		fs_usage->hidden	+= size;
417

418
	dev_usage->d[type].buckets	+= nr;
419 420
}

421
static void bch2_dev_usage_update(struct bch_fs *c, struct bch_dev *ca,
422 423
				  struct bch_fs_usage *fs_usage,
				  struct bucket_mark old, struct bucket_mark new,
424
				  u64 journal_seq, bool gc)
425
{
426
	struct bch_dev_usage *u;
427

428
	percpu_rwsem_assert_held(&c->mark_lock);
429 430

	preempt_disable();
431 432
	if (!fs_usage)
		fs_usage = fs_usage_ptr(c, journal_seq, gc);
433
	u = dev_usage_ptr(ca, journal_seq, gc);
434

435
	if (bucket_type(old))
436
		account_bucket(fs_usage, u, bucket_type(old),
437
			       -1, -ca->mi.bucket_size);
438

439
	if (bucket_type(new))
440
		account_bucket(fs_usage, u, bucket_type(new),
441
			       1, ca->mi.bucket_size);
442

443
	u->buckets_unavailable +=
444 445
		is_unavailable_bucket(new) - is_unavailable_bucket(old);

446 447 448
	u->d[old.data_type].sectors -= old.dirty_sectors;
	u->d[new.data_type].sectors += new.dirty_sectors;
	u->d[BCH_DATA_cached].sectors +=
449
		(int) new.cached_sectors - (int) old.cached_sectors;
450 451 452 453

	u->d[old.data_type].fragmented -= bucket_sectors_fragmented(ca, old);
	u->d[new.data_type].fragmented += bucket_sectors_fragmented(ca, new);

454 455 456 457 458 459
	preempt_enable();

	if (!is_available_bucket(old) && is_available_bucket(new))
		bch2_wake_allocator(ca);
}

460 461 462 463
static inline void update_replicas(struct bch_fs *c,
				   struct bch_fs_usage *fs_usage,
				   struct bch_replicas_entry *r,
				   s64 sectors)
464 465 466
{
	int idx = bch2_replicas_entry_idx(c, r);

467
	BUG_ON(idx < 0);
468

469
	fs_usage_data_type_to_base(fs_usage, r->data_type, sectors);
470
	fs_usage->replicas[idx]		+= sectors;
471 472 473 474 475 476 477 478 479 480 481 482 483
}

static inline void update_cached_sectors(struct bch_fs *c,
					 struct bch_fs_usage *fs_usage,
					 unsigned dev, s64 sectors)
{
	struct bch_replicas_padded r;

	bch2_replicas_entry_cached(&r.e, dev);

	update_replicas(c, fs_usage, &r.e, sectors);
}

484 485 486 487 488
static struct replicas_delta_list *
replicas_deltas_realloc(struct btree_trans *trans, unsigned more)
{
	struct replicas_delta_list *d = trans->fs_usage_deltas;
	unsigned new_size = d ? (d->size + more) * 2 : 128;
489 490 491
	unsigned alloc_size = sizeof(*d) + new_size;

	WARN_ON_ONCE(alloc_size > REPLICAS_DELTA_LIST_MAX);
492 493

	if (!d || d->used + more > d->size) {
494 495 496 497 498 499 500 501 502 503 504 505 506 507 508
		d = krealloc(d, alloc_size, GFP_NOIO|__GFP_ZERO);

		BUG_ON(!d && alloc_size > REPLICAS_DELTA_LIST_MAX);

		if (!d) {
			d = mempool_alloc(&trans->c->replicas_delta_pool, GFP_NOIO);
			memset(d, 0, REPLICAS_DELTA_LIST_MAX);

			if (trans->fs_usage_deltas)
				memcpy(d, trans->fs_usage_deltas,
				       trans->fs_usage_deltas->size + sizeof(*d));

			new_size = REPLICAS_DELTA_LIST_MAX - sizeof(*d);
			kfree(trans->fs_usage_deltas);
		}
509 510 511 512 513 514 515 516 517 518 519 520 521

		d->size = new_size;
		trans->fs_usage_deltas = d;
	}
	return d;
}

static inline void update_replicas_list(struct btree_trans *trans,
					struct bch_replicas_entry *r,
					s64 sectors)
{
	struct replicas_delta_list *d;
	struct replicas_delta *n;
522 523 524 525
	unsigned b;

	if (!sectors)
		return;
526

527
	b = replicas_entry_bytes(r) + 8;
528 529 530 531 532 533
	d = replicas_deltas_realloc(trans, b);

	n = (void *) d->d + d->used;
	n->delta = sectors;
	memcpy((void *) n + offsetof(struct replicas_delta, r),
	       r, replicas_entry_bytes(r));
534
	bch2_replicas_entry_sort(&n->r);
535 536 537 538 539 540 541 542 543 544 545 546 547
	d->used += b;
}

static inline void update_cached_sectors_list(struct btree_trans *trans,
					      unsigned dev, s64 sectors)
{
	struct bch_replicas_padded r;

	bch2_replicas_entry_cached(&r.e, dev);

	update_replicas_list(trans, &r.e, sectors);
}

548 549 550 551 552 553 554
#define do_mark_fn(fn, c, pos, flags, ...)				\
({									\
	int gc, ret = 0;						\
									\
	percpu_rwsem_assert_held(&c->mark_lock);			\
									\
	for (gc = 0; gc < 2 && !ret; gc++)				\
555
		if (!gc == !(flags & BTREE_TRIGGER_GC) ||		\
556 557 558 559 560 561 562 563
		    (gc && gc_visited(c, pos)))				\
			ret = fn(c, __VA_ARGS__, gc);			\
	ret;								\
})

static int __bch2_mark_alloc_bucket(struct bch_fs *c, struct bch_dev *ca,
				    size_t b, bool owned_by_allocator,
				    bool gc)
564
{
565
	struct bucket *g = __bucket(ca, b, gc);
566 567
	struct bucket_mark old, new;

568
	old = bucket_cmpxchg(g, new, ({
569 570 571
		new.owned_by_allocator	= owned_by_allocator;
	}));

572 573
	BUG_ON(!gc &&
	       !owned_by_allocator && !old.owned_by_allocator);
574 575

	return 0;
576 577 578 579 580 581
}

void bch2_mark_alloc_bucket(struct bch_fs *c, struct bch_dev *ca,
			    size_t b, bool owned_by_allocator,
			    struct gc_pos pos, unsigned flags)
{
582 583
	preempt_disable();

584 585
	do_mark_fn(__bch2_mark_alloc_bucket, c, pos, flags,
		   ca, b, owned_by_allocator);
586 587

	preempt_enable();
588 589
}

590 591
static int bch2_mark_alloc(struct bch_fs *c,
			   struct bkey_s_c old, struct bkey_s_c new,
592
			   struct bch_fs_usage *fs_usage,
593
			   u64 journal_seq, unsigned flags)
594
{
595
	bool gc = flags & BTREE_TRIGGER_GC;
596 597 598
	struct bkey_alloc_unpacked u;
	struct bch_dev *ca;
	struct bucket *g;
599 600 601
	struct bucket_mark old_m, m;

	/* We don't do anything for deletions - do we?: */
602 603
	if (new.k->type != KEY_TYPE_alloc &&
	    new.k->type != KEY_TYPE_alloc_v2)
604
		return 0;
605 606 607 608

	/*
	 * alloc btree is read in by bch2_alloc_read, not gc:
	 */
609 610
	if ((flags & BTREE_TRIGGER_GC) &&
	    !(flags & BTREE_TRIGGER_BUCKET_INVALIDATE))
611 612
		return 0;

613
	ca = bch_dev_bkey_exists(c, new.k->p.inode);
614

615
	if (new.k->p.offset >= ca->mi.nbuckets)
616 617
		return 0;

618 619
	g = __bucket(ca, new.k->p.offset, gc);
	u = bch2_alloc_unpack(new);
620

621
	old_m = bucket_cmpxchg(g, m, ({
622 623 624 625
		m.gen			= u.gen;
		m.data_type		= u.data_type;
		m.dirty_sectors		= u.dirty_sectors;
		m.cached_sectors	= u.cached_sectors;
626
		m.stripe		= u.stripe != 0;
627

628
		if (journal_seq) {
629 630 631
			m.journal_seq_valid	= 1;
			m.journal_seq		= journal_seq;
		}
632 633
	}));

634
	bch2_dev_usage_update(c, ca, fs_usage, old_m, m, journal_seq, gc);
635

636 637 638 639
	g->io_time[READ]	= u.read_time;
	g->io_time[WRITE]	= u.write_time;
	g->oldest_gen		= u.oldest_gen;
	g->gen_valid		= 1;
640 641
	g->stripe		= u.stripe;
	g->stripe_redundancy	= u.stripe_redundancy;
642

643 644 645 646 647
	/*
	 * need to know if we're getting called from the invalidate path or
	 * not:
	 */

648
	if ((flags & BTREE_TRIGGER_BUCKET_INVALIDATE) &&
649
	    old_m.cached_sectors) {
650
		update_cached_sectors(c, fs_usage, ca->dev_idx,
651 652 653
				      -old_m.cached_sectors);
		trace_invalidate(ca, bucket_to_sector(ca, new.k->p.offset),
				 old_m.cached_sectors);
654 655 656 657 658
	}

	return 0;
}

659
#define checked_add(a, b)					\
660
({								\
661
	unsigned _res = (unsigned) (a) + (b);			\
662 663 664
	bool overflow = _res > U16_MAX;				\
	if (overflow)						\
		_res = U16_MAX;					\
665
	(a) = _res;						\
666 667
	overflow;						\
})
668

669
static int __bch2_mark_metadata_bucket(struct bch_fs *c, struct bch_dev *ca,
670
				       size_t b, enum bch_data_type data_type,
671
				       unsigned sectors, bool gc)
672 673
{
	struct bucket *g = __bucket(ca, b, gc);
674 675
	struct bucket_mark old, new;
	bool overflow;
676

677 678
	BUG_ON(data_type != BCH_DATA_sb &&
	       data_type != BCH_DATA_journal);
679

680
	old = bucket_cmpxchg(g, new, ({
681
		new.data_type	= data_type;
682
		overflow = checked_add(new.dirty_sectors, sectors);
683
	}));
684

685
	bch2_fs_inconsistent_on(old.data_type &&
686
				old.data_type != data_type, c,
687 688
		"different types of data in same bucket: %s, %s",
		bch2_data_types[old.data_type],
689
		bch2_data_types[data_type]);
690

691
	bch2_fs_inconsistent_on(overflow, c,
692 693 694
		"bucket %u:%zu gen %u data type %s sector count overflow: %u + %u > U16_MAX",
		ca->dev_idx, b, new.gen,
		bch2_data_types[old.data_type ?: data_type],
695 696 697
		old.dirty_sectors, sectors);

	if (c)
698
		bch2_dev_usage_update(c, ca, fs_usage_ptr(c, 0, gc),
699
				      old, new, 0, gc);
700

701
	return 0;
702 703
}

704 705 706 707 708
void bch2_mark_metadata_bucket(struct bch_fs *c, struct bch_dev *ca,
			       size_t b, enum bch_data_type type,
			       unsigned sectors, struct gc_pos pos,
			       unsigned flags)
{
709 710
	BUG_ON(type != BCH_DATA_sb &&
	       type != BCH_DATA_journal);
711

712 713
	preempt_disable();

714
	if (likely(c)) {
715 716
		do_mark_fn(__bch2_mark_metadata_bucket, c, pos, flags,
			   ca, b, type, sectors);
717
	} else {
718
		__bch2_mark_metadata_bucket(c, ca, b, type, sectors, 0);
719
	}
720

721
	preempt_enable();
722 723
}

724 725 726 727 728 729 730 731 732
static s64 disk_sectors_scaled(unsigned n, unsigned d, unsigned sectors)
{
	return DIV_ROUND_UP(sectors * n, d);
}

static s64 __ptr_disk_sectors_delta(unsigned old_size,
				    unsigned offset, s64 delta,
				    unsigned flags,
				    unsigned n, unsigned d)
733
{
734 735
	BUG_ON(!n || !d);

736
	if (flags & BTREE_TRIGGER_OVERWRITE_SPLIT) {
737
		BUG_ON(offset + -delta > old_size);
738

739 740 741
		return -disk_sectors_scaled(n, d, old_size) +
			disk_sectors_scaled(n, d, offset) +
			disk_sectors_scaled(n, d, old_size - offset + delta);
742
	} else if (flags & BTREE_TRIGGER_OVERWRITE) {
743
		BUG_ON(offset + -delta > old_size);
744

745 746
		return -disk_sectors_scaled(n, d, old_size) +
			disk_sectors_scaled(n, d, old_size + delta);
747
	} else {
748
		return  disk_sectors_scaled(n, d, delta);
749
	}
750 751
}

752 753 754 755 756 757 758 759 760 761
static s64 ptr_disk_sectors_delta(struct extent_ptr_decoded p,
				  unsigned offset, s64 delta,
				  unsigned flags)
{
	return __ptr_disk_sectors_delta(p.crc.live_size,
					offset, delta, flags,
					p.crc.compressed_size,
					p.crc.uncompressed_size);
}

762 763 764 765 766
static int check_bucket_ref(struct bch_fs *c, struct bkey_s_c k,
			    const struct bch_extent_ptr *ptr,
			    s64 sectors, enum bch_data_type ptr_data_type,
			    u8 bucket_gen, u8 bucket_data_type,
			    u16 dirty_sectors, u16 cached_sectors)
767
{
768 769
	size_t bucket_nr = PTR_BUCKET_NR(bch_dev_bkey_exists(c, ptr->dev), ptr);
	u16 bucket_sectors = !ptr->cached
770 771 772 773
		? dirty_sectors
		: cached_sectors;
	char buf[200];

774
	if (gen_after(ptr->gen, bucket_gen)) {
775 776 777
		bch2_fsck_err(c, FSCK_CAN_IGNORE|FSCK_NEED_FSCK,
			"bucket %u:%zu gen %u data type %s: ptr gen %u newer than bucket gen\n"
			"while marking %s",
778 779 780
			ptr->dev, bucket_nr, bucket_gen,
			bch2_data_types[bucket_data_type ?: ptr_data_type],
			ptr->gen,
781 782 783 784
			(bch2_bkey_val_to_text(&PBUF(buf), c, k), buf));
		return -EIO;
	}

785
	if (gen_cmp(bucket_gen, ptr->gen) > BUCKET_GC_GEN_MAX) {
786 787 788
		bch2_fsck_err(c, FSCK_CAN_IGNORE|FSCK_NEED_FSCK,
			"bucket %u:%zu gen %u data type %s: ptr gen %u too stale\n"
			"while marking %s",
789 790 791
			ptr->dev, bucket_nr, bucket_gen,
			bch2_data_types[bucket_data_type ?: ptr_data_type],
			ptr->gen,
792 793 794 795
			(bch2_bkey_val_to_text(&PBUF(buf), c, k), buf));
		return -EIO;
	}

796
	if (bucket_gen != ptr->gen && !ptr->cached) {
797 798 799
		bch2_fsck_err(c, FSCK_CAN_IGNORE|FSCK_NEED_FSCK,
			"bucket %u:%zu gen %u data type %s: stale dirty ptr (gen %u)\n"
			"while marking %s",
800 801 802
			ptr->dev, bucket_nr, bucket_gen,
			bch2_data_types[bucket_data_type ?: ptr_data_type],
			ptr->gen,
803 804 805 806
			(bch2_bkey_val_to_text(&PBUF(buf), c, k), buf));
		return -EIO;
	}

807
	if (bucket_gen != ptr->gen)
808 809
		return 1;

810 811
	if (bucket_data_type && ptr_data_type &&
	    bucket_data_type != ptr_data_type) {
812 813 814
		bch2_fsck_err(c, FSCK_CAN_IGNORE|FSCK_NEED_FSCK,
			"bucket %u:%zu gen %u different types of data in same bucket: %s, %s\n"
			"while marking %s",
815 816
			ptr->dev, bucket_nr, bucket_gen,
			bch2_data_types[bucket_data_type],
817 818 819 820 821
			bch2_data_types[ptr_data_type],
			(bch2_bkey_val_to_text(&PBUF(buf), c, k), buf));
		return -EIO;
	}

822
	if ((unsigned) (bucket_sectors + sectors) > U16_MAX) {
823 824 825
		bch2_fsck_err(c, FSCK_CAN_IGNORE|FSCK_NEED_FSCK,
			"bucket %u:%zu gen %u data type %s sector count overflow: %u + %lli > U16_MAX\n"
			"while marking %s",
826 827 828
			ptr->dev, bucket_nr, bucket_gen,
			bch2_data_types[bucket_data_type ?: ptr_data_type],
			bucket_sectors, sectors,
829 830 831 832
			(bch2_bkey_val_to_text(&PBUF(buf), c, k), buf));
		return -EIO;
	}

833 834 835
	return 0;
}

836
static int mark_stripe_bucket(struct bch_fs *c, struct bkey_s_c k,
837
			     unsigned ptr_idx,
838
			     struct bch_fs_usage *fs_usage,
839
			     u64 journal_seq, unsigned flags)
840
{
841 842 843 844
	const struct bch_stripe *s = bkey_s_c_to_stripe(k).v;
	unsigned nr_data = s->nr_blocks - s->nr_redundant;
	bool parity = ptr_idx >= nr_data;
	const struct bch_extent_ptr *ptr = s->ptrs + ptr_idx;
845 846 847 848 849 850 851
	bool gc = flags & BTREE_TRIGGER_GC;
	struct bch_dev *ca = bch_dev_bkey_exists(c, ptr->dev);
	struct bucket *g = PTR_BUCKET(ca, ptr, gc);
	struct bucket_mark new, old;
	char buf[200];
	int ret;

852 853 854 855 856 857 858
	if (g->stripe && g->stripe != k.k->p.offset) {
		bch2_fs_inconsistent(c,
			      "bucket %u:%zu gen %u: multiple stripes using same bucket\n%s",
			      ptr->dev, PTR_BUCKET_NR(ca, ptr), new.gen,
			      (bch2_bkey_val_to_text(&PBUF(buf), c, k), buf));
		return -EINVAL;
	}
859

860 861 862 863 864 865
	old = bucket_cmpxchg(g, new, ({
		ret = check_bucket_ref(c, k, ptr, 0, 0, new.gen, new.data_type,
				       new.dirty_sectors, new.cached_sectors);
		if (ret)
			return ret;

866 867 868
		if (parity) {
			new.data_type		= BCH_DATA_parity;
			new.dirty_sectors	= le16_to_cpu(s->sectors);
869 870
		}

871 872 873 874 875 876
		if (journal_seq) {
			new.journal_seq_valid	= 1;
			new.journal_seq		= journal_seq;
		}
	}));

877 878
	g->stripe		= k.k->p.offset;
	g->stripe_redundancy	= s->nr_redundant;
879

880
	bch2_dev_usage_update(c, ca, fs_usage, old, new, journal_seq, gc);
881 882 883 884 885 886 887 888 889 890 891 892 893 894 895 896 897 898 899 900
	return 0;
}

static int __mark_pointer(struct bch_fs *c, struct bkey_s_c k,
			  const struct bch_extent_ptr *ptr,
			  s64 sectors, enum bch_data_type ptr_data_type,
			  u8 bucket_gen, u8 *bucket_data_type,
			  u16 *dirty_sectors, u16 *cached_sectors)
{
	u16 *dst_sectors = !ptr->cached
		? dirty_sectors
		: cached_sectors;
	int ret = check_bucket_ref(c, k, ptr, sectors, ptr_data_type,
				   bucket_gen, *bucket_data_type,
				   *dirty_sectors, *cached_sectors);

	if (ret)
		return ret;

	*dst_sectors += sectors;
901 902 903 904 905 906 907 908 909 910
	*bucket_data_type = *dirty_sectors || *cached_sectors
		? ptr_data_type : 0;
	return 0;
}

static int bch2_mark_pointer(struct bch_fs *c, struct bkey_s_c k,
			     struct extent_ptr_decoded p,
			     s64 sectors, enum bch_data_type data_type,
			     struct bch_fs_usage *fs_usage,
			     u64 journal_seq, unsigned flags)
911
{
912
	bool gc = flags & BTREE_TRIGGER_GC;
913 914
	struct bucket_mark old, new;
	struct bch_dev *ca = bch_dev_bkey_exists(c, p.ptr.dev);
915
	struct bucket *g = PTR_BUCKET(ca, &p.ptr, gc);
916
	u8 bucket_data_type;
917
	u64 v;
918
	int ret;
919

920 921 922
	v = atomic64_read(&g->_mark.v);
	do {
		new.v.counter = old.v.counter = v;
923
		bucket_data_type = new.data_type;
924

925
		ret = __mark_pointer(c, k, &p.ptr, sectors, data_type, new.gen,
926 927 928 929 930
				     &bucket_data_type,
				     &new.dirty_sectors,
				     &new.cached_sectors);
		if (ret)
			return ret;
931

932
		new.data_type = bucket_data_type;
933

934 935 936
		if (journal_seq) {
			new.journal_seq_valid = 1;
			new.journal_seq = journal_seq;
937 938
		}

939
		if (flags & BTREE_TRIGGER_NOATOMIC) {
940 941 942 943 944 945 946
			g->_mark = new;
			break;
		}
	} while ((v = atomic64_cmpxchg(&g->_mark.v,
			      old.v.counter,
			      new.v.counter)) != old.v.counter);

947
	bch2_dev_usage_update(c, ca, fs_usage, old, new, journal_seq, gc);
948

949
	BUG_ON(!gc && bucket_became_unavailable(old, new));
950

951
	return 0;
952 953
}

954 955
static int bch2_mark_stripe_ptr(struct bch_fs *c,
				struct bch_extent_stripe_ptr p,
956 957
				enum bch_data_type data_type,
				struct bch_fs_usage *fs_usage,
958
				s64 sectors, unsigned flags)
959
{
960
	bool gc = flags & BTREE_TRIGGER_GC;
961
	struct bch_replicas_padded r;
962
	struct stripe *m;
963
	unsigned i, blocks_nonempty = 0;
964

965
	m = genradix_ptr(&c->stripes[gc], p.idx);
966

967 968
	spin_lock(&c->ec_stripes_heap_lock);

969
	if (!m || !m->alive) {
970
		spin_unlock(&c->ec_stripes_heap_lock);
971 972
		bch_err_ratelimited(c, "pointer to nonexistent stripe %llu",
				    (u64) p.idx);
Kent Overstreet's avatar
Kent Overstreet committed
973
		return -EIO;
974
	}
975

976
	m->block_sectors[p.block] += sectors;
977

978 979
	r = m->r;

980 981
	for (i = 0; i < m->nr_blocks; i++)
		blocks_nonempty += m->block_sectors[i] != 0;
982

983 984
	if (m->blocks_nonempty != blocks_nonempty) {
		m->blocks_nonempty = blocks_nonempty;
985 986 987
		if (!gc)
			bch2_stripes_heap_update(c, m, p.idx);
	}
988

989
	spin_unlock(&c->ec_stripes_heap_lock);
990

991 992 993
	r.e.data_type = data_type;
	update_replicas(c, fs_usage, &r.e, sectors);

994
	return 0;
995 996
}

997 998
static int bch2_mark_extent(struct bch_fs *c,
			    struct bkey_s_c old, struct bkey_s_c new,
999 1000
			    unsigned offset, s64 sectors,
			    enum bch_data_type data_type,
1001
			    struct bch_fs_usage *fs_usage,
1002
			    unsigned journal_seq, unsigned flags)
1003
{
1004
	struct bkey_s_c k = flags & BTREE_TRIGGER_INSERT ? new : old;
1005 1006 1007
	struct bkey_ptrs_c ptrs = bch2_bkey_ptrs_c(k);
	const union bch_extent_entry *entry;
	struct extent_ptr_decoded p;
1008 1009
	struct bch_replicas_padded r;
	s64 dirty_sectors = 0;
1010
	bool stale;
1011 1012
	int ret;

1013 1014 1015 1016
	r.e.data_type	= data_type;
	r.e.nr_devs	= 0;
	r.e.nr_required	= 1;

1017 1018
	BUG_ON(!sectors);

1019
	bkey_for_each_ptr_decode(k.k, ptrs, p, entry) {
1020
		s64 disk_sectors = data_type == BCH_DATA_btree
1021
			? sectors
1022
			: ptr_disk_sectors_delta(p, offset, sectors, flags);
1023 1024 1025 1026 1027 1028 1029

		ret = bch2_mark_pointer(c, k, p, disk_sectors, data_type,
					fs_usage, journal_seq, flags);
		if (ret < 0)
			return ret;

		stale = ret > 0;
1030

1031
		if (p.ptr.cached) {
1032
			if (!stale)
1033 1034
				update_cached_sectors(c, fs_usage, p.ptr.dev,
						      disk_sectors);
1035
		} else if (!p.has_ec) {
1036 1037 1038
			dirty_sectors	       += disk_sectors;
			r.e.devs[r.e.nr_devs++]	= p.ptr.dev;
		} else {
1039
			ret = bch2_mark_stripe_ptr(c, p.ec, data_type,
1040
					fs_usage, disk_sectors, flags);
1041 1042
			if (ret)
				return ret;
1043

1044 1045 1046 1047 1048
			/*
			 * There may be other dirty pointers in this extent, but
			 * if so they're not required for mounting if we have an
			 * erasure coded pointer in this extent:
			 */
1049 1050
			r.e.nr_required = 0;
		}
1051
	}
1052

1053 1054
	if (r.e.nr_devs)
		update_replicas(c, fs_usage, &r.e, dirty_sectors);
1055 1056

	return 0;
1057
}
1058

1059 1060
static int bch2_mark_stripe(struct bch_fs *c,
			    struct bkey_s_c old, struct bkey_s_c new,
1061
			    struct bch_fs_usage *fs_usage,
1062
			    u64 journal_seq, unsigned flags)
1063
{
1064
	bool gc = flags & BTREE_TRIGGER_GC;
1065 1066 1067 1068 1069
	size_t idx = new.k->p.offset;
	const struct bch_stripe *old_s = old.k->type == KEY_TYPE_stripe
		? bkey_s_c_to_stripe(old).v : NULL;
	const struct bch_stripe *new_s = new.k->type == KEY_TYPE_stripe
		? bkey_s_c_to_stripe(new).v : NULL;
1070 1071
	struct stripe *m = genradix_ptr(&c->stripes[gc], idx);
	unsigned i;
1072
	int ret;
1073

1074 1075
	BUG_ON(gc && old_s);

1076
	if (!m || (old_s && !m->alive)) {
1077 1078 1079 1080
		bch_err_ratelimited(c, "error marking nonexistent stripe %zu",
				    idx);
		return -1;
	}
1081

1082
	if (!new_s) {
1083 1084 1085
		spin_lock(&c->ec_stripes_heap_lock);
		bch2_stripes_heap_del(c, m, idx);
		spin_unlock(&c->ec_stripes_heap_lock);
1086

1087 1088
		memset(m, 0, sizeof(*m));
	} else {
1089
		m->alive	= true;
1090 1091 1092 1093
		m->sectors	= le16_to_cpu(new_s->sectors);
		m->algorithm	= new_s->algorithm;
		m->nr_blocks	= new_s->nr_blocks;
		m->nr_redundant	= new_s->nr_redundant;
1094 1095 1096
		m->blocks_nonempty = 0;

		for (i = 0; i < new_s->nr_blocks; i++) {
1097 1098 1099
			m->block_sectors[i] =
				stripe_blockcount_get(new_s, i);
			m->blocks_nonempty += !!m->block_sectors[i];
1100 1101

			m->ptrs[i] = new_s->ptrs[i];
1102
		}
1103 1104

		bch2_bkey_to_replicas(&m->r.e, new);
1105

1106 1107
		if (!gc) {
			spin_lock(&c->ec_stripes_heap_lock);
1108
			bch2_stripes_heap_update(c, m, idx);
1109 1110
			spin_unlock(&c->ec_stripes_heap_lock);
		}
1111
	}
1112

1113 1114 1115 1116 1117 1118 1119 1120 1121 1122 1123 1124 1125 1126 1127 1128 1129 1130 1131
	if (gc) {
		/*
		 * gc recalculates this field from stripe ptr
		 * references:
		 */
		memset(m->block_sectors, 0, sizeof(m->block_sectors));
		m->blocks_nonempty = 0;

		for (i = 0; i < new_s->nr_blocks; i++) {
			ret = mark_stripe_bucket(c, new, i, fs_usage,
						 journal_seq, flags);
			if (ret)
				return ret;
		}

		update_replicas(c, fs_usage, &m->r.e,
				((s64) m->sectors * m->nr_redundant));
	}

1132
	return 0;
1133 1134
}

1135
static int bch2_mark_key_locked(struct bch_fs *c,
1136 1137
		   struct bkey_s_c old,
		   struct bkey_s_c new,
1138
		   unsigned offset, s64 sectors,
1139 1140
		   struct bch_fs_usage *fs_usage,
		   u64 journal_seq, unsigned flags)
1141
{
1142
	struct bkey_s_c k = flags & BTREE_TRIGGER_INSERT ? new : old;
1143 1144
	int ret = 0;

1145 1146
	BUG_ON(!(flags & (BTREE_TRIGGER_INSERT|BTREE_TRIGGER_OVERWRITE)));

1147 1148
	preempt_disable();

1149
	if (!fs_usage || (flags & BTREE_TRIGGER_GC))
1150
		fs_usage = fs_usage_ptr(c, journal_seq,
1151
					flags & BTREE_TRIGGER_GC);
1152

1153
	switch (k.k->type) {
1154
	case KEY_TYPE_alloc:
1155
	case KEY_TYPE_alloc_v2:
1156
		ret = bch2_mark_alloc(c, old, new, fs_usage, journal_seq, flags);
1157
		break;
1158
	case KEY_TYPE_btree_ptr:
Kent Overstreet's avatar
Kent Overstreet committed
1159
	case KEY_TYPE_btree_ptr_v2:
1160
		sectors = !(flags & BTREE_TRIGGER_OVERWRITE)
1161 1162 1163
			?  c->opts.btree_node_size
			: -c->opts.btree_node_size;

1164
		ret = bch2_mark_extent(c, old, new, offset, sectors,
1165
				BCH_DATA_btree, fs_usage, journal_seq, flags);
1166
		break;
1167
	case KEY_TYPE_extent:
Kent Overstreet's avatar
Kent Overstreet committed
1168
	case KEY_TYPE_reflink_v:
1169
		ret = bch2_mark_extent(c, old, new, offset, sectors,
1170
				BCH_DATA_user, fs_usage, journal_seq, flags);
1171
		break;
1172
	case KEY_TYPE_stripe:
1173
		ret = bch2_mark_stripe(c, old, new, fs_usage, journal_seq, flags);
1174
		break;
1175
	case KEY_TYPE_inode:
1176 1177
		fs_usage->nr_inodes += new.k->type == KEY_TYPE_inode;
		fs_usage->nr_inodes -= old.k->type == KEY_TYPE_inode;
1178
		break;
1179 1180 1181 1182
	case KEY_TYPE_reservation: {
		unsigned replicas = bkey_s_c_to_reservation(k).v->nr_replicas;

		sectors *= replicas;
1183 1184
		replicas = clamp_t(unsigned, replicas, 1,
				   ARRAY_SIZE(fs_usage->persistent_reserved));
1185

1186
		fs_usage->reserved				+= sectors;
1187
		fs_usage->persistent_reserved[replicas - 1]	+= sectors;
1188
		break;
1189
	}
1190
	}
1191 1192 1193 1194

	preempt_enable();

	return ret;
1195 1196
}

1197
int bch2_mark_key(struct bch_fs *c, struct bkey_s_c new,
1198
		  unsigned offset, s64 sectors,
1199
		  struct bch_fs_usage *fs_usage,
1200 1201
		  u64 journal_seq, unsigned flags)
{
1202 1203
	struct bkey deleted;
	struct bkey_s_c old = (struct bkey_s_c) { &deleted, NULL };
1204 1205
	int ret;

1206 1207
	bkey_init(&deleted);

1208
	percpu_down_read(&c->mark_lock);
1209 1210 1211
	ret = bch2_mark_key_locked(c, old, new, offset, sectors,
				   fs_usage, journal_seq,
				   BTREE_TRIGGER_INSERT|flags);
1212
	percpu_up_read(&c->mark_lock);
1213 1214

	return ret;
1215 1216
}

1217
int bch2_mark_update(struct btree_trans *trans,
1218
		     struct btree_iter *iter,
1219
		     struct bkey_i *new,
1220 1221
		     struct bch_fs_usage *fs_usage,
		     unsigned flags)
1222 1223
{
	struct bch_fs		*c = trans->c;
1224 1225
	struct bkey_s_c		old;
	struct bkey		unpacked;
1226
	int ret = 0;
1227

1228 1229 1230
	if (unlikely(flags & BTREE_TRIGGER_NORUN))
		return 0;

1231
	if (!btree_node_type_needs_gc(iter->btree_id))
1232
		return 0;
1233

1234 1235
	bkey_init(&unpacked);
	old = (struct bkey_s_c) { &unpacked, NULL };
1236

1237
	if (!btree_node_type_is_extents(iter->btree_id)) {
1238
		/* iterators should be uptodate, shouldn't get errors here: */
1239
		if (btree_iter_type(iter) != BTREE_ITER_CACHED) {
1240 1241
			old = bch2_btree_iter_peek_slot(iter);
			BUG_ON(bkey_err(old));
1242 1243
		} else {
			struct bkey_cached *ck = (void *) iter->l[0].b;
1244

1245 1246 1247
			if (ck->valid)
				old = bkey_i_to_s_c(ck->k);
		}
1248

1249 1250 1251 1252 1253 1254 1255 1256 1257 1258 1259 1260 1261 1262
		if (old.k->type == new->k.type) {
			bch2_mark_key_locked(c, old, bkey_i_to_s_c(new), 0, 0,
				fs_usage, trans->journal_res.seq,
				BTREE_TRIGGER_INSERT|BTREE_TRIGGER_OVERWRITE|flags);

		} else {
			bch2_mark_key_locked(c, old, bkey_i_to_s_c(new), 0, 0,
				fs_usage, trans->journal_res.seq,
				BTREE_TRIGGER_INSERT|flags);
			bch2_mark_key_locked(c, old, bkey_i_to_s_c(new), 0, 0,
				fs_usage, trans->journal_res.seq,
				BTREE_TRIGGER_OVERWRITE|flags);
		}
	} else {
1263 1264
		struct btree_iter *copy;

1265 1266 1267 1268 1269 1270
		BUG_ON(btree_iter_type(iter) == BTREE_ITER_CACHED);
		bch2_mark_key_locked(c, old, bkey_i_to_s_c(new),
			0, new->k.size,
			fs_usage, trans->journal_res.seq,
			BTREE_TRIGGER_INSERT|flags);

1271
		copy = bch2_trans_copy_iter(trans, iter);
1272

1273 1274 1275
		for_each_btree_key_continue(copy, 0, old, ret) {
			unsigned offset = 0;
			s64 sectors = -((s64) old.k->size);
1276 1277 1278 1279

			flags |= BTREE_TRIGGER_OVERWRITE;

			if (bkey_cmp(new->k.p, bkey_start_pos(old.k)) <= 0)
1280
				break;
1281 1282 1283 1284 1285 1286 1287 1288 1289 1290 1291 1292 1293 1294 1295 1296 1297 1298 1299 1300 1301 1302 1303 1304 1305 1306

			switch (bch2_extent_overlap(&new->k, old.k)) {
			case BCH_EXTENT_OVERLAP_ALL:
				offset = 0;
				sectors = -((s64) old.k->size);
				break;
			case BCH_EXTENT_OVERLAP_BACK:
				offset = bkey_start_offset(&new->k) -
					bkey_start_offset(old.k);
				sectors = bkey_start_offset(&new->k) -
					old.k->p.offset;
				break;
			case BCH_EXTENT_OVERLAP_FRONT:
				offset = 0;
				sectors = bkey_start_offset(old.k) -
					new->k.p.offset;
				break;
			case BCH_EXTENT_OVERLAP_MIDDLE:
				offset = bkey_start_offset(&new->k) -
					bkey_start_offset(old.k);
				sectors = -((s64) new->k.size);
				flags |= BTREE_TRIGGER_OVERWRITE_SPLIT;
				break;
			}

			BUG_ON(sectors >= 0);
1307

1308 1309 1310 1311 1312 1313
			ret = bch2_mark_key_locked(c, old, bkey_i_to_s_c(new),
					offset, sectors, fs_usage,
					trans->journal_res.seq, flags) ?: 1;
			if (ret <= 0)
				break;
		}
1314
		bch2_trans_iter_put(trans, copy);
1315
	}
1316 1317

	return ret;
1318
}
1319

1320 1321 1322
static noinline __cold
void fs_usage_apply_warn(struct btree_trans *trans,
			 unsigned disk_res_sectors)
1323 1324 1325 1326 1327
{
	struct bch_fs *c = trans->c;
	struct btree_insert_entry *i;
	char buf[200];

1328
	bch_err(c, "disk usage increased more than %u sectors reserved",
Kent Overstreet's avatar
Kent Overstreet committed
1329
		disk_res_sectors);
1330

1331
	trans_for_each_update(trans, i) {
1332
		pr_err("while inserting");
1333
		bch2_bkey_val_to_text(&PBUF(buf), c, bkey_i_to_s_c(i->k));
1334 1335 1336
		pr_err("%s", buf);
		pr_err("overlapping with");

1337
		if (btree_iter_type(i->iter) != BTREE_ITER_CACHED) {
1338 1339 1340
			struct btree_iter *copy = bch2_trans_copy_iter(trans, i->iter);
			struct bkey_s_c k;
			int ret;
1341

1342 1343
			for_each_btree_key_continue(copy, 0, k, ret) {
				if (btree_node_type_is_extents(i->iter->btree_id)
1344 1345 1346 1347 1348 1349 1350
				    ? bkey_cmp(i->k->k.p, bkey_start_pos(k.k)) <= 0
				    : bkey_cmp(i->k->k.p, k.k->p))
					break;

				bch2_bkey_val_to_text(&PBUF(buf), c, k);
				pr_err("%s", buf);
			}
1351
			bch2_trans_iter_put(trans, copy);
1352 1353 1354
		} else {
			struct bkey_cached *ck = (void *) i->iter->l[0].b;

1355 1356 1357 1358
			if (ck->valid) {
				bch2_bkey_val_to_text(&PBUF(buf), c, bkey_i_to_s_c(ck->k));
				pr_err("%s", buf);
			}
1359 1360
		}
	}
1361 1362
}

1363 1364 1365 1366 1367 1368 1369 1370 1371 1372 1373 1374 1375 1376 1377 1378 1379 1380 1381 1382 1383 1384 1385 1386 1387 1388 1389 1390 1391 1392 1393 1394 1395 1396 1397 1398 1399 1400 1401 1402 1403 1404 1405 1406 1407 1408 1409 1410 1411 1412 1413 1414 1415 1416 1417 1418 1419 1420 1421
void bch2_trans_fs_usage_apply(struct btree_trans *trans,
			       struct replicas_delta_list *deltas)
{
	struct bch_fs *c = trans->c;
	static int warned_disk_usage = 0;
	bool warn = false;
	unsigned disk_res_sectors = trans->disk_res ? trans->disk_res->sectors : 0;
	struct replicas_delta *d = deltas->d;
	struct replicas_delta *top = (void *) deltas->d + deltas->used;
	struct bch_fs_usage *dst;
	s64 added = 0, should_not_have_added;
	unsigned i;

	percpu_rwsem_assert_held(&c->mark_lock);

	preempt_disable();
	dst = fs_usage_ptr(c, trans->journal_res.seq, false);

	for (d = deltas->d; d != top; d = replicas_delta_next(d)) {
		switch (d->r.data_type) {
		case BCH_DATA_btree:
		case BCH_DATA_user:
		case BCH_DATA_parity:
			added += d->delta;
		}

		update_replicas(c, dst, &d->r, d->delta);
	}

	dst->nr_inodes += deltas->nr_inodes;

	for (i = 0; i < BCH_REPLICAS_MAX; i++) {
		added				+= deltas->persistent_reserved[i];
		dst->reserved			+= deltas->persistent_reserved[i];
		dst->persistent_reserved[i]	+= deltas->persistent_reserved[i];
	}

	/*
	 * Not allowed to reduce sectors_available except by getting a
	 * reservation:
	 */
	should_not_have_added = added - (s64) disk_res_sectors;
	if (unlikely(should_not_have_added > 0)) {
		atomic64_sub(should_not_have_added, &c->sectors_available);
		added -= should_not_have_added;
		warn = true;
	}

	if (added > 0) {
		trans->disk_res->sectors -= added;
		this_cpu_sub(*c->online_reserved, added);
	}

	preempt_enable();

	if (unlikely(warn) && !xchg(&warned_disk_usage, 1))
		fs_usage_apply_warn(trans, disk_res_sectors);
}

1422 1423
/* trans_mark: */

1424 1425 1426
static struct btree_iter *trans_get_update(struct btree_trans *trans,
			    enum btree_id btree_id, struct bpos pos,
			    struct bkey_s_c *k)
1427
{
1428
	struct btree_insert_entry *i;
1429

1430
	trans_for_each_update(trans, i)
1431
		if (i->iter->btree_id == btree_id &&
1432 1433 1434 1435
		    (btree_node_type_is_extents(btree_id)
		     ? bkey_cmp(pos, bkey_start_pos(&i->k->k)) >= 0 &&
		       bkey_cmp(pos, i->k->k.p) < 0
		     : !bkey_cmp(pos, i->iter->pos))) {
1436
			*k = bkey_i_to_s_c(i->k);
1437 1438 1439 1440

			/* ugly hack.. */
			BUG_ON(btree_iter_live(trans, i->iter));
			trans->iters_live |= 1ULL << i->iter->idx;
1441
			return i->iter;
1442 1443
		}

1444 1445 1446 1447 1448 1449 1450 1451
	return NULL;
}

static int trans_get_key(struct btree_trans *trans,
			 enum btree_id btree_id, struct bpos pos,
			 struct btree_iter **iter,
			 struct bkey_s_c *k)
{
1452
	unsigned flags = btree_id != BTREE_ID_alloc
1453 1454 1455 1456 1457 1458 1459 1460
		? BTREE_ITER_SLOTS
		: BTREE_ITER_CACHED;
	int ret;

	*iter = trans_get_update(trans, btree_id, pos, k);
	if (*iter)
		return 1;

1461
	*iter = bch2_trans_get_iter(trans, btree_id, pos,
1462 1463
				    flags|BTREE_ITER_INTENT);
	*k = __bch2_btree_iter_peek(*iter, flags);
1464 1465 1466 1467 1468 1469
	ret = bkey_err(*k);
	if (ret)
		bch2_trans_iter_put(trans, *iter);
	return ret;
}

1470 1471 1472 1473
static struct bkey_alloc_buf *
bch2_trans_start_alloc_update(struct btree_trans *trans, struct btree_iter **_iter,
			      const struct bch_extent_ptr *ptr,
			      struct bkey_alloc_unpacked *u)
1474 1475
{
	struct bch_fs *c = trans->c;
1476 1477
	struct bch_dev *ca = bch_dev_bkey_exists(c, ptr->dev);
	struct bpos pos = POS(ptr->dev, PTR_BUCKET_NR(ca, ptr));
1478
	struct bucket *g;
1479 1480
	struct btree_iter *iter;
	struct bkey_s_c k;
1481
	struct bkey_alloc_buf *a;
1482 1483
	int ret;

1484 1485 1486 1487
	a = bch2_trans_kmalloc(trans, sizeof(struct bkey_alloc_buf));
	if (IS_ERR(a))
		return a;

1488
	iter = trans_get_update(trans, BTREE_ID_alloc, pos, &k);
1489
	if (iter) {
1490
		*u = bch2_alloc_unpack(k);
1491
	} else {
1492
		iter = bch2_trans_get_iter(trans, BTREE_ID_alloc, pos,
1493 1494 1495 1496
					   BTREE_ITER_CACHED|
					   BTREE_ITER_CACHED_NOFILL|
					   BTREE_ITER_INTENT);
		ret = bch2_btree_iter_traverse(iter);
1497 1498
		if (ret) {
			bch2_trans_iter_put(trans, iter);
1499
			return ERR_PTR(ret);
1500
		}
1501

1502 1503
		percpu_down_read(&c->mark_lock);
		g = bucket(ca, pos.offset);
1504
		*u = alloc_mem_to_key(iter, g, READ_ONCE(g->mark));
1505 1506
		percpu_up_read(&c->mark_lock);
	}
1507

1508
	*_iter = iter;
1509
	return a;
1510 1511 1512 1513 1514 1515 1516 1517 1518
}

static int bch2_trans_mark_pointer(struct btree_trans *trans,
			struct bkey_s_c k, struct extent_ptr_decoded p,
			s64 sectors, enum bch_data_type data_type)
{
	struct bch_fs *c = trans->c;
	struct btree_iter *iter;
	struct bkey_alloc_unpacked u;
1519
	struct bkey_alloc_buf *a;
1520 1521
	int ret;

1522 1523 1524
	a = bch2_trans_start_alloc_update(trans, &iter, &p.ptr, &u);
	if (IS_ERR(a))
		return PTR_ERR(a);
1525 1526

	ret = __mark_pointer(c, k, &p.ptr, sectors, data_type, u.gen, &u.data_type,
1527 1528
			     &u.dirty_sectors, &u.cached_sectors);
	if (ret)
1529
		goto out;
1530

1531 1532
	bch2_alloc_pack(c, a, u);
	bch2_trans_update(trans, iter, &a->k, 0);
1533 1534 1535 1536 1537 1538
out:
	bch2_trans_iter_put(trans, iter);
	return ret;
}

static int bch2_trans_mark_stripe_ptr(struct btree_trans *trans,
1539
			struct extent_ptr_decoded p,
1540
			s64 sectors, enum bch_data_type data_type)
1541
{
Kent Overstreet's avatar
Kent Overstreet committed
1542
	struct bch_fs *c = trans->c;
1543 1544
	struct btree_iter *iter;
	struct bkey_s_c k;
1545
	struct bkey_i_stripe *s;
1546
	struct bch_replicas_padded r;
1547 1548
	int ret = 0;

1549
	ret = trans_get_key(trans, BTREE_ID_stripes, POS(0, p.ec.idx), &iter, &k);
1550
	if (ret < 0)
1551 1552 1553
		return ret;

	if (k.k->type != KEY_TYPE_stripe) {
Kent Overstreet's avatar
Kent Overstreet committed
1554 1555
		bch2_fs_inconsistent(c,
			"pointer to nonexistent stripe %llu",
1556 1557 1558 1559 1560 1561 1562 1563 1564
			(u64) p.ec.idx);
		ret = -EIO;
		goto out;
	}

	if (!bch2_ptr_matches_stripe(bkey_s_c_to_stripe(k).v, p)) {
		bch2_fs_inconsistent(c,
			"stripe pointer doesn't match stripe %llu",
			(u64) p.ec.idx);
Kent Overstreet's avatar
Kent Overstreet committed
1565
		ret = -EIO;
1566 1567 1568
		goto out;
	}

1569 1570
	s = bch2_trans_kmalloc(trans, bkey_bytes(k.k));
	ret = PTR_ERR_OR_ZERO(s);
1571 1572 1573
	if (ret)
		goto out;

1574
	bkey_reassemble(&s->k_i, k);
1575 1576
	stripe_blockcount_set(&s->v, p.ec.block,
		stripe_blockcount_get(&s->v, p.ec.block) +
1577
		sectors);
1578
	bch2_trans_update(trans, iter, &s->k_i, 0);
1579 1580 1581 1582

	bch2_bkey_to_replicas(&r.e, bkey_i_to_s_c(&s->k_i));
	r.e.data_type = data_type;
	update_replicas_list(trans, &r.e, sectors);
1583 1584 1585 1586 1587 1588
out:
	bch2_trans_iter_put(trans, iter);
	return ret;
}

static int bch2_trans_mark_extent(struct btree_trans *trans,
1589 1590 1591
			struct bkey_s_c k, unsigned offset,
			s64 sectors, unsigned flags,
			enum bch_data_type data_type)
1592 1593 1594 1595 1596 1597 1598 1599 1600 1601 1602 1603 1604 1605 1606 1607
{
	struct bkey_ptrs_c ptrs = bch2_bkey_ptrs_c(k);
	const union bch_extent_entry *entry;
	struct extent_ptr_decoded p;
	struct bch_replicas_padded r;
	s64 dirty_sectors = 0;
	bool stale;
	int ret;

	r.e.data_type	= data_type;
	r.e.nr_devs	= 0;
	r.e.nr_required	= 1;

	BUG_ON(!sectors);

	bkey_for_each_ptr_decode(k.k, ptrs, p, entry) {
1608
		s64 disk_sectors = data_type == BCH_DATA_btree
1609
			? sectors
1610
			: ptr_disk_sectors_delta(p, offset, sectors, flags);
1611

1612
		ret = bch2_trans_mark_pointer(trans, k, p, disk_sectors,
1613
					      data_type);
1614 1615 1616 1617 1618 1619
		if (ret < 0)
			return ret;

		stale = ret > 0;

		if (p.ptr.cached) {
1620
			if (!stale)
1621
				update_cached_sectors_list(trans, p.ptr.dev,
1622
							   disk_sectors);
1623
		} else if (!p.has_ec) {
1624 1625 1626
			dirty_sectors	       += disk_sectors;
			r.e.devs[r.e.nr_devs++]	= p.ptr.dev;
		} else {
1627
			ret = bch2_trans_mark_stripe_ptr(trans, p,
1628
					disk_sectors, data_type);
1629 1630
			if (ret)
				return ret;
1631 1632 1633 1634 1635

			r.e.nr_required = 0;
		}
	}

1636 1637
	if (r.e.nr_devs)
		update_replicas_list(trans, &r.e, dirty_sectors);
1638 1639 1640 1641

	return 0;
}

1642
static int bch2_trans_mark_stripe_alloc_ref(struct btree_trans *trans,
1643 1644
					    struct bkey_s_c_stripe s,
					    unsigned idx, bool deleting)
1645
{
1646 1647 1648
	struct bch_fs *c = trans->c;
	const struct bch_extent_ptr *ptr = &s.v->ptrs[idx];
	struct bkey_alloc_buf *a;
1649 1650
	struct btree_iter *iter;
	struct bkey_alloc_unpacked u;
1651 1652
	bool parity = idx >= s.v->nr_blocks - s.v->nr_redundant;
	int ret = 0;
1653

1654 1655 1656
	a = bch2_trans_start_alloc_update(trans, &iter, ptr, &u);
	if (IS_ERR(a))
		return PTR_ERR(a);
1657 1658

	if (parity) {
1659 1660 1661 1662 1663
		s64 sectors = le16_to_cpu(s.v->sectors);

		if (deleting)
			sectors = -sectors;

1664 1665 1666 1667 1668 1669
		u.dirty_sectors += sectors;
		u.data_type = u.dirty_sectors
			? BCH_DATA_parity
			: 0;
	}

1670 1671 1672 1673 1674 1675 1676 1677
	if (!deleting) {
		if (bch2_fs_inconsistent_on(u.stripe && u.stripe != s.k->p.offset, c,
				"bucket %llu:%llu gen %u: multiple stripes using same bucket (%u, %llu)",
				iter->pos.inode, iter->pos.offset, u.gen,
				u.stripe, s.k->p.offset)) {
			ret = -EIO;
			goto err;
		}
1678

1679 1680 1681 1682 1683 1684 1685 1686 1687
		u.stripe		= s.k->p.offset;
		u.stripe_redundancy	= s.v->nr_redundant;
	} else {
		u.stripe		= 0;
		u.stripe_redundancy	= 0;
	}

	bch2_alloc_pack(c, a, u);
	bch2_trans_update(trans, iter, &a->k, 0);
1688 1689 1690 1691 1692
err:
	bch2_trans_iter_put(trans, iter);
	return ret;
}

1693
static int bch2_trans_mark_stripe(struct btree_trans *trans,
1694
				  struct bkey_s_c old, struct bkey_s_c new,
1695
				  unsigned flags)
1696
{
1697 1698
	struct bkey_s_c_stripe old_s = { NULL };
	struct bkey_s_c_stripe new_s = { NULL };
1699
	struct bch_replicas_padded r;
1700 1701 1702
	unsigned i;
	int ret = 0;

1703 1704 1705 1706 1707
	if (old.k->type == KEY_TYPE_stripe)
		old_s = bkey_s_c_to_stripe(old);
	if (new.k->type == KEY_TYPE_stripe)
		new_s = bkey_s_c_to_stripe(new);

1708
	/*
1709
	 * If the pointers aren't changing, we don't need to do anything:
1710
	 */
1711 1712 1713 1714 1715
	if (new_s.k && old_s.k &&
	    new_s.v->nr_blocks		== old_s.v->nr_blocks &&
	    new_s.v->nr_redundant	== old_s.v->nr_redundant &&
	    !memcmp(old_s.v->ptrs, new_s.v->ptrs,
		    new_s.v->nr_blocks * sizeof(struct bch_extent_ptr)))
1716
		return 0;
1717

1718 1719
	if (new_s.k) {
		s64 sectors = le16_to_cpu(new_s.v->sectors);
1720

1721
		bch2_bkey_to_replicas(&r.e, new);
1722
		update_replicas_list(trans, &r.e, sectors * new_s.v->nr_redundant);
1723

1724 1725 1726
		for (i = 0; i < new_s.v->nr_blocks; i++) {
			ret = bch2_trans_mark_stripe_alloc_ref(trans, new_s,
							       i, false);
1727 1728
			if (ret)
				return ret;
1729
		}
1730
	}
1731

1732 1733
	if (old_s.k) {
		s64 sectors = -((s64) le16_to_cpu(old_s.v->sectors));
1734 1735

		bch2_bkey_to_replicas(&r.e, old);
1736
		update_replicas_list(trans, &r.e, sectors * old_s.v->nr_redundant);
1737

1738 1739 1740
		for (i = 0; i < old_s.v->nr_blocks; i++) {
			ret = bch2_trans_mark_stripe_alloc_ref(trans, old_s,
							       i, true);
1741 1742 1743
			if (ret)
				return ret;
		}
1744 1745 1746 1747 1748
	}

	return ret;
}

1749 1750 1751 1752 1753 1754 1755 1756 1757 1758 1759 1760
static __le64 *bkey_refcount(struct bkey_i *k)
{
	switch (k->k.type) {
	case KEY_TYPE_reflink_v:
		return &bkey_i_to_reflink_v(k)->v.refcount;
	case KEY_TYPE_indirect_inline_data:
		return &bkey_i_to_indirect_inline_data(k)->v.refcount;
	default:
		return NULL;
	}
}

Kent Overstreet's avatar
Kent Overstreet committed
1761 1762 1763 1764 1765 1766 1767 1768
static int __bch2_trans_mark_reflink_p(struct btree_trans *trans,
			struct bkey_s_c_reflink_p p,
			u64 idx, unsigned sectors,
			unsigned flags)
{
	struct bch_fs *c = trans->c;
	struct btree_iter *iter;
	struct bkey_s_c k;
1769 1770
	struct bkey_i *n;
	__le64 *refcount;
Kent Overstreet's avatar
Kent Overstreet committed
1771 1772
	s64 ret;

1773
	ret = trans_get_key(trans, BTREE_ID_reflink,
Kent Overstreet's avatar
Kent Overstreet committed
1774
			    POS(0, idx), &iter, &k);
1775
	if (ret < 0)
Kent Overstreet's avatar
Kent Overstreet committed
1776 1777
		return ret;

1778
	if ((flags & BTREE_TRIGGER_OVERWRITE) &&
Kent Overstreet's avatar
Kent Overstreet committed
1779 1780 1781 1782
	    (bkey_start_offset(k.k) < idx ||
	     k.k->p.offset > idx + sectors))
		goto out;

1783
	sectors = k.k->p.offset - idx;
Kent Overstreet's avatar
Kent Overstreet committed
1784

1785 1786
	n = bch2_trans_kmalloc(trans, bkey_bytes(k.k));
	ret = PTR_ERR_OR_ZERO(n);
Kent Overstreet's avatar
Kent Overstreet committed
1787 1788 1789
	if (ret)
		goto err;

1790 1791 1792 1793 1794 1795 1796 1797 1798 1799
	bkey_reassemble(n, k);

	refcount = bkey_refcount(n);
	if (!refcount) {
		bch2_fs_inconsistent(c,
			"%llu:%llu len %u points to nonexistent indirect extent %llu",
			p.k->p.inode, p.k->p.offset, p.k->size, idx);
		ret = -EIO;
		goto err;
	}
Kent Overstreet's avatar
Kent Overstreet committed
1800

1801
	le64_add_cpu(refcount, !(flags & BTREE_TRIGGER_OVERWRITE) ? 1 : -1);
Kent Overstreet's avatar
Kent Overstreet committed
1802

1803 1804 1805
	if (!*refcount) {
		n->k.type = KEY_TYPE_deleted;
		set_bkey_val_u64s(&n->k, 0);
Kent Overstreet's avatar
Kent Overstreet committed
1806
	}
1807

1808
	bch2_btree_iter_set_pos(iter, bkey_start_pos(k.k));
1809
	bch2_trans_update(trans, iter, n, 0);
Kent Overstreet's avatar
Kent Overstreet committed
1810
out:
1811
	ret = sectors;
Kent Overstreet's avatar
Kent Overstreet committed
1812 1813 1814 1815 1816 1817 1818 1819 1820 1821 1822 1823 1824 1825 1826 1827 1828 1829 1830 1831 1832 1833 1834 1835 1836 1837 1838 1839
err:
	bch2_trans_iter_put(trans, iter);
	return ret;
}

static int bch2_trans_mark_reflink_p(struct btree_trans *trans,
			struct bkey_s_c_reflink_p p, unsigned offset,
			s64 sectors, unsigned flags)
{
	u64 idx = le64_to_cpu(p.v->idx) + offset;
	s64 ret = 0;

	sectors = abs(sectors);
	BUG_ON(offset + sectors > p.k->size);

	while (sectors) {
		ret = __bch2_trans_mark_reflink_p(trans, p, idx, sectors, flags);
		if (ret < 0)
			break;

		idx += ret;
		sectors = max_t(s64, 0LL, sectors - ret);
		ret = 0;
	}

	return ret;
}

1840 1841 1842
int bch2_trans_mark_key(struct btree_trans *trans,
			struct bkey_s_c old,
			struct bkey_s_c new,
1843
			unsigned offset, s64 sectors, unsigned flags)
1844 1845
{
	struct bch_fs *c = trans->c;
1846 1847 1848 1849
	struct bkey_s_c k = flags & BTREE_TRIGGER_INSERT ? new : old;
	struct replicas_delta_list *d;

	BUG_ON(!(flags & (BTREE_TRIGGER_INSERT|BTREE_TRIGGER_OVERWRITE)));
1850 1851 1852

	switch (k.k->type) {
	case KEY_TYPE_btree_ptr:
Kent Overstreet's avatar
Kent Overstreet committed
1853
	case KEY_TYPE_btree_ptr_v2:
1854
		sectors = !(flags & BTREE_TRIGGER_OVERWRITE)
1855 1856 1857
			?  c->opts.btree_node_size
			: -c->opts.btree_node_size;

1858
		return bch2_trans_mark_extent(trans, k, offset, sectors,
1859
					      flags, BCH_DATA_btree);
1860
	case KEY_TYPE_extent:
Kent Overstreet's avatar
Kent Overstreet committed
1861
	case KEY_TYPE_reflink_v:
1862
		return bch2_trans_mark_extent(trans, k, offset, sectors,
1863
					      flags, BCH_DATA_user);
1864
	case KEY_TYPE_stripe:
1865 1866 1867 1868 1869 1870 1871 1872 1873
		return bch2_trans_mark_stripe(trans, old, new, flags);
	case KEY_TYPE_inode: {
		int nr = (new.k->type == KEY_TYPE_inode) -
			 (old.k->type == KEY_TYPE_inode);

		if (nr) {
			d = replicas_deltas_realloc(trans, 0);
			d->nr_inodes += nr;
		}
1874

1875
		return 0;
1876
	}
1877 1878 1879
	case KEY_TYPE_reservation: {
		unsigned replicas = bkey_s_c_to_reservation(k).v->nr_replicas;

1880
		d = replicas_deltas_realloc(trans, 0);
1881

1882 1883
		sectors *= replicas;
		replicas = clamp_t(unsigned, replicas, 1,
1884
				   ARRAY_SIZE(d->persistent_reserved));
1885

1886
		d->persistent_reserved[replicas - 1] += sectors;
1887 1888
		return 0;
	}
Kent Overstreet's avatar
Kent Overstreet committed
1889 1890 1891 1892
	case KEY_TYPE_reflink_p:
		return bch2_trans_mark_reflink_p(trans,
					bkey_s_c_to_reflink_p(k),
					offset, sectors, flags);
1893 1894 1895 1896 1897 1898
	default:
		return 0;
	}
}

int bch2_trans_mark_update(struct btree_trans *trans,
1899
			   struct btree_iter *iter,
1900
			   struct bkey_i *new,
1901
			   unsigned flags)
1902
{
1903
	struct bkey_s_c	old;
1904 1905
	int ret;

1906 1907 1908
	if (unlikely(flags & BTREE_TRIGGER_NORUN))
		return 0;

1909 1910 1911
	if (!btree_node_type_needs_gc(iter->btree_id))
		return 0;

1912 1913 1914 1915 1916 1917 1918
	if (!btree_node_type_is_extents(iter->btree_id)) {
		/* iterators should be uptodate, shouldn't get errors here: */
		if (btree_iter_type(iter) != BTREE_ITER_CACHED) {
			old = bch2_btree_iter_peek_slot(iter);
			BUG_ON(bkey_err(old));
		} else {
			struct bkey_cached *ck = (void *) iter->l[0].b;
1919

1920 1921 1922
			BUG_ON(!ck->valid);
			old = bkey_i_to_s_c(ck->k);
		}
1923

1924 1925 1926 1927 1928 1929 1930 1931 1932 1933
		if (old.k->type == new->k.type) {
			ret   = bch2_trans_mark_key(trans, old, bkey_i_to_s_c(new), 0, 0,
					BTREE_TRIGGER_INSERT|BTREE_TRIGGER_OVERWRITE|flags);
		} else {
			ret   = bch2_trans_mark_key(trans, old, bkey_i_to_s_c(new), 0, 0,
					BTREE_TRIGGER_INSERT|flags) ?:
				bch2_trans_mark_key(trans, old, bkey_i_to_s_c(new), 0, 0,
					BTREE_TRIGGER_OVERWRITE|flags);
		}
	} else {
1934 1935
		struct btree_iter *copy;
		struct bkey _old;
1936

1937
		EBUG_ON(btree_iter_type(iter) == BTREE_ITER_CACHED);
1938

1939 1940
		bkey_init(&_old);
		old = (struct bkey_s_c) { &_old, NULL };
1941 1942 1943 1944 1945 1946 1947

		ret = bch2_trans_mark_key(trans, old, bkey_i_to_s_c(new),
					  0, new->k.size,
					  BTREE_TRIGGER_INSERT);
		if (ret)
			return ret;

1948
		copy = bch2_trans_copy_iter(trans, iter);
1949

1950 1951 1952
		for_each_btree_key_continue(copy, 0, old, ret) {
			unsigned offset = 0;
			s64 sectors = -((s64) old.k->size);
1953 1954 1955 1956

			flags |= BTREE_TRIGGER_OVERWRITE;

			if (bkey_cmp(new->k.p, bkey_start_pos(old.k)) <= 0)
1957
				break;
1958

1959
			switch (bch2_extent_overlap(&new->k, old.k)) {
1960
			case BCH_EXTENT_OVERLAP_ALL:
1961
				offset = 0;
1962
				sectors = -((s64) old.k->size);
1963 1964
				break;
			case BCH_EXTENT_OVERLAP_BACK:
1965 1966 1967 1968
				offset = bkey_start_offset(&new->k) -
					bkey_start_offset(old.k);
				sectors = bkey_start_offset(&new->k) -
					old.k->p.offset;
1969 1970
				break;
			case BCH_EXTENT_OVERLAP_FRONT:
1971
				offset = 0;
1972 1973
				sectors = bkey_start_offset(old.k) -
					new->k.p.offset;
1974 1975
				break;
			case BCH_EXTENT_OVERLAP_MIDDLE:
1976 1977 1978
				offset = bkey_start_offset(&new->k) -
					bkey_start_offset(old.k);
				sectors = -((s64) new->k.size);
1979
				flags |= BTREE_TRIGGER_OVERWRITE_SPLIT;
1980 1981 1982 1983 1984
				break;
			}

			BUG_ON(sectors >= 0);

1985 1986 1987
			ret = bch2_trans_mark_key(trans, old, bkey_i_to_s_c(new),
					offset, sectors, flags);
			if (ret)
1988
				break;
1989
		}
1990
		bch2_trans_iter_put(trans, copy);
1991 1992
	}

1993
	return ret;
1994 1995
}

1996 1997 1998 1999 2000 2001 2002 2003
static int __bch2_trans_mark_metadata_bucket(struct btree_trans *trans,
				    struct bch_dev *ca, size_t b,
				    enum bch_data_type type,
				    unsigned sectors)
{
	struct bch_fs *c = trans->c;
	struct btree_iter *iter;
	struct bkey_alloc_unpacked u;
2004
	struct bkey_alloc_buf *a;
2005 2006 2007 2008 2009 2010
	struct bch_extent_ptr ptr = {
		.dev = ca->dev_idx,
		.offset = bucket_to_sector(ca, b),
	};
	int ret = 0;

2011 2012 2013
	a = bch2_trans_start_alloc_update(trans, &iter, &ptr, &u);
	if (IS_ERR(a))
		return PTR_ERR(a);
2014 2015 2016 2017 2018 2019 2020 2021 2022 2023 2024 2025 2026 2027 2028 2029 2030 2031 2032 2033 2034 2035 2036 2037 2038 2039 2040 2041 2042 2043 2044 2045

	if (u.data_type && u.data_type != type) {
		bch2_fsck_err(c, FSCK_CAN_IGNORE|FSCK_NEED_FSCK,
			"bucket %llu:%llu gen %u different types of data in same bucket: %s, %s\n"
			"while marking %s",
			iter->pos.inode, iter->pos.offset, u.gen,
			bch2_data_types[u.data_type],
			bch2_data_types[type],
			bch2_data_types[type]);
		ret = -EIO;
		goto out;
	}

	if ((unsigned) (u.dirty_sectors + sectors) > ca->mi.bucket_size) {
		bch2_fsck_err(c, FSCK_CAN_IGNORE|FSCK_NEED_FSCK,
			"bucket %llu:%llu gen %u data type %s sector count overflow: %u + %u > %u\n"
			"while marking %s",
			iter->pos.inode, iter->pos.offset, u.gen,
			bch2_data_types[u.data_type ?: type],
			u.dirty_sectors, sectors, ca->mi.bucket_size,
			bch2_data_types[type]);
		ret = -EIO;
		goto out;
	}

	if (u.data_type		== type &&
	    u.dirty_sectors	== sectors)
		goto out;

	u.data_type	= type;
	u.dirty_sectors	= sectors;

2046 2047
	bch2_alloc_pack(c, a, u);
	bch2_trans_update(trans, iter, &a->k, 0);
2048 2049 2050 2051 2052 2053 2054 2055 2056 2057 2058 2059 2060 2061 2062 2063 2064 2065 2066 2067 2068 2069 2070 2071 2072 2073 2074 2075 2076 2077 2078 2079 2080 2081 2082 2083 2084 2085 2086 2087 2088 2089 2090 2091 2092 2093 2094 2095 2096 2097 2098 2099 2100 2101 2102 2103 2104 2105 2106 2107 2108 2109 2110 2111 2112 2113 2114 2115 2116 2117 2118 2119 2120 2121 2122 2123 2124 2125 2126 2127 2128 2129 2130 2131 2132 2133 2134 2135 2136 2137 2138 2139 2140 2141 2142 2143 2144 2145 2146 2147 2148 2149 2150
out:
	bch2_trans_iter_put(trans, iter);
	return ret;
}

int bch2_trans_mark_metadata_bucket(struct btree_trans *trans,
				    struct disk_reservation *res,
				    struct bch_dev *ca, size_t b,
				    enum bch_data_type type,
				    unsigned sectors)
{
	return __bch2_trans_do(trans, res, NULL, 0,
			__bch2_trans_mark_metadata_bucket(trans, ca, b, BCH_DATA_journal,
							ca->mi.bucket_size));

}

static int bch2_trans_mark_metadata_sectors(struct btree_trans *trans,
					    struct disk_reservation *res,
					    struct bch_dev *ca,
					    u64 start, u64 end,
					    enum bch_data_type type,
					    u64 *bucket, unsigned *bucket_sectors)
{
	int ret;

	do {
		u64 b = sector_to_bucket(ca, start);
		unsigned sectors =
			min_t(u64, bucket_to_sector(ca, b + 1), end) - start;

		if (b != *bucket) {
			if (*bucket_sectors) {
				ret = bch2_trans_mark_metadata_bucket(trans, res, ca,
						*bucket, type, *bucket_sectors);
				if (ret)
					return ret;
			}

			*bucket		= b;
			*bucket_sectors	= 0;
		}

		*bucket_sectors	+= sectors;
		start += sectors;
	} while (!ret && start < end);

	return 0;
}

static int __bch2_trans_mark_dev_sb(struct btree_trans *trans,
			     struct disk_reservation *res,
			     struct bch_dev *ca)
{
	struct bch_sb_layout *layout = &ca->disk_sb.sb->layout;
	u64 bucket = 0;
	unsigned i, bucket_sectors = 0;
	int ret;

	for (i = 0; i < layout->nr_superblocks; i++) {
		u64 offset = le64_to_cpu(layout->sb_offset[i]);

		if (offset == BCH_SB_SECTOR) {
			ret = bch2_trans_mark_metadata_sectors(trans, res, ca,
						0, BCH_SB_SECTOR,
						BCH_DATA_sb, &bucket, &bucket_sectors);
			if (ret)
				return ret;
		}

		ret = bch2_trans_mark_metadata_sectors(trans, res, ca, offset,
				      offset + (1 << layout->sb_max_size_bits),
				      BCH_DATA_sb, &bucket, &bucket_sectors);
		if (ret)
			return ret;
	}

	if (bucket_sectors) {
		ret = bch2_trans_mark_metadata_bucket(trans, res, ca,
				bucket, BCH_DATA_sb, bucket_sectors);
		if (ret)
			return ret;
	}

	for (i = 0; i < ca->journal.nr; i++) {
		ret = bch2_trans_mark_metadata_bucket(trans, res, ca,
				ca->journal.buckets[i],
				BCH_DATA_journal, ca->mi.bucket_size);
		if (ret)
			return ret;
	}

	return 0;
}

int bch2_trans_mark_dev_sb(struct bch_fs *c,
			   struct disk_reservation *res,
			   struct bch_dev *ca)
{
	return bch2_trans_do(c, res, NULL, 0,
			__bch2_trans_mark_dev_sb(&trans, res, ca));
}

2151 2152 2153 2154 2155
/* Disk reservations: */

#define SECTORS_CACHE	1024

int bch2_disk_reservation_add(struct bch_fs *c, struct disk_reservation *res,
2156
			      u64 sectors, int flags)
2157
{
2158
	struct bch_fs_pcpu *pcpu;
2159 2160 2161 2162
	u64 old, v, get;
	s64 sectors_available;
	int ret;

2163
	percpu_down_read(&c->mark_lock);
2164
	preempt_disable();
2165
	pcpu = this_cpu_ptr(c->pcpu);
2166

2167
	if (sectors <= pcpu->sectors_available)
2168 2169 2170 2171 2172 2173 2174 2175 2176 2177 2178 2179 2180 2181
		goto out;

	v = atomic64_read(&c->sectors_available);
	do {
		old = v;
		get = min((u64) sectors + SECTORS_CACHE, old);

		if (get < sectors) {
			preempt_enable();
			goto recalculate;
		}
	} while ((v = atomic64_cmpxchg(&c->sectors_available,
				       old, old - get)) != old);

2182
	pcpu->sectors_available		+= get;
2183 2184

out:
2185
	pcpu->sectors_available		-= sectors;
2186
	this_cpu_add(*c->online_reserved, sectors);
2187
	res->sectors			+= sectors;
2188 2189

	preempt_enable();
2190
	percpu_up_read(&c->mark_lock);
2191 2192 2193
	return 0;

recalculate:
2194
	mutex_lock(&c->sectors_available_lock);
2195

2196 2197
	percpu_u64_set(&c->pcpu->sectors_available, 0);
	sectors_available = avail_factor(__bch2_fs_usage_read_short(c).free);
2198 2199 2200 2201 2202

	if (sectors <= sectors_available ||
	    (flags & BCH_DISK_RESERVATION_NOFAIL)) {
		atomic64_set(&c->sectors_available,
			     max_t(s64, 0, sectors_available - sectors));
2203
		this_cpu_add(*c->online_reserved, sectors);
2204
		res->sectors			+= sectors;
2205 2206 2207 2208 2209 2210
		ret = 0;
	} else {
		atomic64_set(&c->sectors_available, sectors_available);
		ret = -ENOSPC;
	}

2211 2212
	mutex_unlock(&c->sectors_available_lock);
	percpu_up_read(&c->mark_lock);
2213 2214 2215 2216 2217 2218 2219 2220 2221 2222 2223 2224 2225 2226 2227 2228 2229 2230 2231

	return ret;
}

/* Startup/shutdown: */

static void buckets_free_rcu(struct rcu_head *rcu)
{
	struct bucket_array *buckets =
		container_of(rcu, struct bucket_array, rcu);

	kvpfree(buckets,
		sizeof(struct bucket_array) +
		buckets->nbuckets * sizeof(struct bucket));
}

int bch2_dev_buckets_resize(struct bch_fs *c, struct bch_dev *ca, u64 nbuckets)
{
	struct bucket_array *buckets = NULL, *old_buckets = NULL;
2232
	unsigned long *buckets_nouse = NULL;
2233 2234 2235 2236 2237 2238 2239
	alloc_fifo	free[RESERVE_NR];
	alloc_fifo	free_inc;
	alloc_heap	alloc_heap;

	size_t btree_reserve	= DIV_ROUND_UP(BTREE_NODE_RESERVE,
			     ca->mi.bucket_size / c->opts.btree_node_size);
	/* XXX: these should be tunable */
2240
	size_t reserve_none	= max_t(size_t, 1, nbuckets >> 9);
2241
	size_t copygc_reserve	= max_t(size_t, 2, nbuckets >> 6);
2242
	size_t free_inc_nr	= max(max_t(size_t, 1, nbuckets >> 12),
2243
				      btree_reserve * 2);
2244
	bool resize = ca->buckets[0] != NULL;
2245 2246 2247 2248 2249 2250 2251 2252 2253 2254
	int ret = -ENOMEM;
	unsigned i;

	memset(&free,		0, sizeof(free));
	memset(&free_inc,	0, sizeof(free_inc));
	memset(&alloc_heap,	0, sizeof(alloc_heap));

	if (!(buckets		= kvpmalloc(sizeof(struct bucket_array) +
					    nbuckets * sizeof(struct bucket),
					    GFP_KERNEL|__GFP_ZERO)) ||
2255
	    !(buckets_nouse	= kvpmalloc(BITS_TO_LONGS(nbuckets) *
2256 2257 2258 2259 2260
					    sizeof(unsigned long),
					    GFP_KERNEL|__GFP_ZERO)) ||
	    !init_fifo(&free[RESERVE_MOVINGGC],
		       copygc_reserve, GFP_KERNEL) ||
	    !init_fifo(&free[RESERVE_NONE], reserve_none, GFP_KERNEL) ||
2261
	    !init_fifo(&free_inc,	free_inc_nr, GFP_KERNEL) ||
2262
	    !init_heap(&alloc_heap,	ALLOC_SCAN_BATCH(ca) << 1, GFP_KERNEL))
2263 2264 2265 2266 2267
		goto err;

	buckets->first_bucket	= ca->mi.first_bucket;
	buckets->nbuckets	= nbuckets;

2268
	bch2_copygc_stop(c);
2269 2270

	if (resize) {
2271
		down_write(&c->gc_lock);
2272
		down_write(&ca->bucket_lock);
2273
		percpu_down_write(&c->mark_lock);
2274 2275 2276 2277 2278 2279 2280 2281 2282 2283
	}

	old_buckets = bucket_array(ca);

	if (resize) {
		size_t n = min(buckets->nbuckets, old_buckets->nbuckets);

		memcpy(buckets->b,
		       old_buckets->b,
		       n * sizeof(struct bucket));
2284 2285
		memcpy(buckets_nouse,
		       ca->buckets_nouse,
2286 2287 2288
		       BITS_TO_LONGS(n) * sizeof(unsigned long));
	}

2289
	rcu_assign_pointer(ca->buckets[0], buckets);
2290 2291
	buckets = old_buckets;

2292
	swap(ca->buckets_nouse, buckets_nouse);
2293

2294
	if (resize) {
2295
		percpu_up_write(&c->mark_lock);
2296 2297
		up_write(&c->gc_lock);
	}
2298 2299 2300 2301 2302 2303 2304 2305 2306 2307 2308 2309 2310 2311 2312

	spin_lock(&c->freelist_lock);
	for (i = 0; i < RESERVE_NR; i++) {
		fifo_move(&free[i], &ca->free[i]);
		swap(ca->free[i], free[i]);
	}
	fifo_move(&free_inc, &ca->free_inc);
	swap(ca->free_inc, free_inc);
	spin_unlock(&c->freelist_lock);

	/* with gc lock held, alloc_heap can't be in use: */
	swap(ca->alloc_heap, alloc_heap);

	nbuckets = ca->mi.nbuckets;

2313
	if (resize)
2314 2315 2316 2317 2318 2319 2320 2321
		up_write(&ca->bucket_lock);

	ret = 0;
err:
	free_heap(&alloc_heap);
	free_fifo(&free_inc);
	for (i = 0; i < RESERVE_NR; i++)
		free_fifo(&free[i]);
2322
	kvpfree(buckets_nouse,
2323 2324 2325 2326 2327 2328 2329 2330 2331 2332 2333 2334 2335 2336 2337
		BITS_TO_LONGS(nbuckets) * sizeof(unsigned long));
	if (buckets)
		call_rcu(&old_buckets->rcu, buckets_free_rcu);

	return ret;
}

void bch2_dev_buckets_free(struct bch_dev *ca)
{
	unsigned i;

	free_heap(&ca->alloc_heap);
	free_fifo(&ca->free_inc);
	for (i = 0; i < RESERVE_NR; i++)
		free_fifo(&ca->free[i]);
2338
	kvpfree(ca->buckets_nouse,
2339
		BITS_TO_LONGS(ca->mi.nbuckets) * sizeof(unsigned long));
2340
	kvpfree(rcu_dereference_protected(ca->buckets[0], 1),
2341 2342 2343
		sizeof(struct bucket_array) +
		ca->mi.nbuckets * sizeof(struct bucket));

2344 2345 2346
	for (i = 0; i < ARRAY_SIZE(ca->usage); i++)
		free_percpu(ca->usage[i]);
	kfree(ca->usage_base);
2347 2348 2349 2350
}

int bch2_dev_buckets_alloc(struct bch_fs *c, struct bch_dev *ca)
{
2351 2352 2353 2354
	unsigned i;

	ca->usage_base = kzalloc(sizeof(struct bch_dev_usage), GFP_KERNEL);
	if (!ca->usage_base)
2355 2356
		return -ENOMEM;

2357 2358 2359 2360 2361 2362
	for (i = 0; i < ARRAY_SIZE(ca->usage); i++) {
		ca->usage[i] = alloc_percpu(struct bch_dev_usage);
		if (!ca->usage[i])
			return -ENOMEM;
	}

2363 2364
	return bch2_dev_buckets_resize(c, ca, ca->mi.nbuckets);;
}