writeback.c 12.6 KB
Newer Older
1 2 3 4 5 6 7 8 9 10 11
/*
 * background writeback - scan btree for dirty data and write it to the backing
 * device
 *
 * Copyright 2010, 2011 Kent Overstreet <kent.overstreet@gmail.com>
 * Copyright 2012 Google, Inc.
 */

#include "bcache.h"
#include "btree.h"
#include "debug.h"
12
#include "writeback.h"
13

14 15 16
#include <linux/delay.h>
#include <linux/freezer.h>
#include <linux/kthread.h>
17 18
#include <trace/events/bcache.h>

19 20 21 22 23 24 25 26 27 28 29 30 31 32
/* Rate limiting */

static void __update_writeback_rate(struct cached_dev *dc)
{
	struct cache_set *c = dc->disk.c;
	uint64_t cache_sectors = c->nbuckets * c->sb.bucket_size;
	uint64_t cache_dirty_target =
		div_u64(cache_sectors * dc->writeback_percent, 100);

	int64_t target = div64_u64(cache_dirty_target * bdev_sectors(dc->bdev),
				   c->cached_dev_sectors);

	/* PD controller */

33
	int64_t dirty = bcache_dev_sectors_dirty(&dc->disk);
34
	int64_t derivative = dirty - dc->disk.sectors_dirty_last;
35 36
	int64_t proportional = dirty - target;
	int64_t change;
37 38 39

	dc->disk.sectors_dirty_last = dirty;

40
	/* Scale to sectors per second */
41

42 43
	proportional *= dc->writeback_rate_update_seconds;
	proportional = div_s64(proportional, dc->writeback_rate_p_term_inverse);
44

45
	derivative = div_s64(derivative, dc->writeback_rate_update_seconds);
46

47 48 49 50 51 52
	derivative = ewma_add(dc->disk.sectors_dirty_derivative, derivative,
			      (dc->writeback_rate_d_term /
			       dc->writeback_rate_update_seconds) ?: 1, 0);

	derivative *= dc->writeback_rate_d_term;
	derivative = div_s64(derivative, dc->writeback_rate_p_term_inverse);
53

54
	change = proportional + derivative;
55 56 57 58

	/* Don't increase writeback rate if the device isn't keeping up */
	if (change > 0 &&
	    time_after64(local_clock(),
59
			 dc->writeback_rate.next + NSEC_PER_MSEC))
60 61 62
		change = 0;

	dc->writeback_rate.rate =
63
		clamp_t(int64_t, (int64_t) dc->writeback_rate.rate + change,
64
			1, NSEC_PER_MSEC);
65 66

	dc->writeback_rate_proportional = proportional;
67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84
	dc->writeback_rate_derivative = derivative;
	dc->writeback_rate_change = change;
	dc->writeback_rate_target = target;
}

static void update_writeback_rate(struct work_struct *work)
{
	struct cached_dev *dc = container_of(to_delayed_work(work),
					     struct cached_dev,
					     writeback_rate_update);

	down_read(&dc->writeback_lock);

	if (atomic_read(&dc->has_dirty) &&
	    dc->writeback_percent)
		__update_writeback_rate(dc);

	up_read(&dc->writeback_lock);
85 86 87

	schedule_delayed_work(&dc->writeback_rate_update,
			      dc->writeback_rate_update_seconds * HZ);
88 89 90 91
}

static unsigned writeback_delay(struct cached_dev *dc, unsigned sectors)
{
92
	if (test_bit(BCACHE_DEV_DETACHING, &dc->disk.flags) ||
93 94 95
	    !dc->writeback_percent)
		return 0;

96
	return bch_next_delay(&dc->writeback_rate, sectors);
97 98
}

99 100 101 102 103
struct dirty_io {
	struct closure		cl;
	struct cached_dev	*dc;
	struct bio		bio;
};
104

105 106 107 108 109 110 111 112 113
static void dirty_init(struct keybuf_key *w)
{
	struct dirty_io *io = w->private;
	struct bio *bio = &io->bio;

	bio_init(bio);
	if (!io->dc->writeback_percent)
		bio_set_prio(bio, IOPRIO_PRIO_VALUE(IOPRIO_CLASS_IDLE, 0));

114
	bio->bi_iter.bi_size	= KEY_SIZE(&w->key) << 9;
115 116 117
	bio->bi_max_vecs	= DIV_ROUND_UP(KEY_SIZE(&w->key), PAGE_SECTORS);
	bio->bi_private		= w;
	bio->bi_io_vec		= bio->bi_inline_vecs;
118
	bch_bio_map(bio, NULL);
119 120 121 122 123 124 125 126 127 128 129 130 131
}

static void dirty_io_destructor(struct closure *cl)
{
	struct dirty_io *io = container_of(cl, struct dirty_io, cl);
	kfree(io);
}

static void write_dirty_finish(struct closure *cl)
{
	struct dirty_io *io = container_of(cl, struct dirty_io, cl);
	struct keybuf_key *w = io->bio.bi_private;
	struct cached_dev *dc = io->dc;
132 133
	struct bio_vec *bv;
	int i;
134

135
	bio_for_each_segment_all(bv, &io->bio, i)
136 137 138 139
		__free_page(bv->bv_page);

	/* This is kind of a dumb way of signalling errors. */
	if (KEY_DIRTY(&w->key)) {
140
		int ret;
141
		unsigned i;
142 143 144
		struct keylist keys;

		bch_keylist_init(&keys);
145

Kent Overstreet's avatar
Kent Overstreet committed
146 147 148
		bkey_copy(keys.top, &w->key);
		SET_KEY_DIRTY(keys.top, false);
		bch_keylist_push(&keys);
149 150 151 152

		for (i = 0; i < KEY_PTRS(&w->key); i++)
			atomic_inc(&PTR_BUCKET(dc->disk.c, &w->key, i)->pin);

153
		ret = bch_btree_insert(dc->disk.c, &keys, NULL, &w->key);
154

155
		if (ret)
156 157
			trace_bcache_writeback_collision(&w->key);

158
		atomic_long_inc(ret
159 160 161 162 163
				? &dc->disk.c->writeback_keys_failed
				: &dc->disk.c->writeback_keys_done);
	}

	bch_keybuf_del(&dc->writeback_keys, w);
164
	up(&dc->in_flight);
165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186

	closure_return_with_destructor(cl, dirty_io_destructor);
}

static void dirty_endio(struct bio *bio, int error)
{
	struct keybuf_key *w = bio->bi_private;
	struct dirty_io *io = w->private;

	if (error)
		SET_KEY_DIRTY(&w->key, false);

	closure_put(&io->cl);
}

static void write_dirty(struct closure *cl)
{
	struct dirty_io *io = container_of(cl, struct dirty_io, cl);
	struct keybuf_key *w = io->bio.bi_private;

	dirty_init(w);
	io->bio.bi_rw		= WRITE;
187
	io->bio.bi_iter.bi_sector = KEY_START(&w->key);
188 189 190 191 192
	io->bio.bi_bdev		= io->dc->bdev;
	io->bio.bi_end_io	= dirty_endio;

	closure_bio_submit(&io->bio, cl, &io->dc->disk);

193
	continue_at(cl, write_dirty_finish, system_wq);
194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212
}

static void read_dirty_endio(struct bio *bio, int error)
{
	struct keybuf_key *w = bio->bi_private;
	struct dirty_io *io = w->private;

	bch_count_io_errors(PTR_CACHE(io->dc->disk.c, &w->key, 0),
			    error, "reading dirty data from cache");

	dirty_endio(bio, error);
}

static void read_dirty_submit(struct closure *cl)
{
	struct dirty_io *io = container_of(cl, struct dirty_io, cl);

	closure_bio_submit(&io->bio, cl, &io->dc->disk);

213
	continue_at(cl, write_dirty, system_wq);
214 215
}

216
static void read_dirty(struct cached_dev *dc)
217
{
218
	unsigned delay = 0;
219 220
	struct keybuf_key *w;
	struct dirty_io *io;
221 222 223
	struct closure cl;

	closure_init_stack(&cl);
224 225 226 227 228 229

	/*
	 * XXX: if we error, background writeback just spins. Should use some
	 * mempools.
	 */

230 231 232
	while (!kthread_should_stop()) {
		try_to_freeze();

233 234 235 236 237 238
		w = bch_keybuf_next(&dc->writeback_keys);
		if (!w)
			break;

		BUG_ON(ptr_stale(dc->disk.c, &w->key, 0));

239 240 241
		if (KEY_START(&w->key) != dc->last_read ||
		    jiffies_to_msecs(delay) > 50)
			while (!kthread_should_stop() && delay)
242
				delay = schedule_timeout_interruptible(delay);
243 244 245 246 247 248 249 250 251 252 253 254 255

		dc->last_read	= KEY_OFFSET(&w->key);

		io = kzalloc(sizeof(struct dirty_io) + sizeof(struct bio_vec)
			     * DIV_ROUND_UP(KEY_SIZE(&w->key), PAGE_SECTORS),
			     GFP_KERNEL);
		if (!io)
			goto err;

		w->private	= io;
		io->dc		= dc;

		dirty_init(w);
256
		io->bio.bi_iter.bi_sector = PTR_OFFSET(&w->key, 0);
257 258 259 260 261
		io->bio.bi_bdev		= PTR_CACHE(dc->disk.c,
						    &w->key, 0)->bdev;
		io->bio.bi_rw		= READ;
		io->bio.bi_end_io	= read_dirty_endio;

262
		if (bio_alloc_pages(&io->bio, GFP_KERNEL))
263 264
			goto err_free;

265
		trace_bcache_writeback(&w->key);
266

267
		down(&dc->in_flight);
268
		closure_call(&io->cl, read_dirty_submit, NULL, &cl);
269 270 271 272 273 274 275 276 277 278 279

		delay = writeback_delay(dc, KEY_SIZE(&w->key));
	}

	if (0) {
err_free:
		kfree(w->private);
err:
		bch_keybuf_del(&dc->writeback_keys, w);
	}

280 281 282 283
	/*
	 * Wait for outstanding writeback IOs to finish (and keybuf slots to be
	 * freed) before refilling again
	 */
284 285 286 287 288 289 290 291 292
	closure_sync(&cl);
}

/* Scan for dirty data */

void bcache_dev_sectors_dirty_add(struct cache_set *c, unsigned inode,
				  uint64_t offset, int nr_sectors)
{
	struct bcache_device *d = c->devices[inode];
293
	unsigned stripe_offset, stripe, sectors_dirty;
294 295 296 297

	if (!d)
		return;

298
	stripe = offset_to_stripe(d, offset);
299 300 301 302 303 304 305 306 307
	stripe_offset = offset & (d->stripe_size - 1);

	while (nr_sectors) {
		int s = min_t(unsigned, abs(nr_sectors),
			      d->stripe_size - stripe_offset);

		if (nr_sectors < 0)
			s = -s;

308 309 310 311 312 313 314 315 316 317
		if (stripe >= d->nr_stripes)
			return;

		sectors_dirty = atomic_add_return(s,
					d->stripe_sectors_dirty + stripe);
		if (sectors_dirty == d->stripe_size)
			set_bit(stripe, d->full_dirty_stripes);
		else
			clear_bit(stripe, d->full_dirty_stripes);

318 319 320 321 322 323 324 325
		nr_sectors -= s;
		stripe_offset = 0;
		stripe++;
	}
}

static bool dirty_pred(struct keybuf *buf, struct bkey *k)
{
326 327 328 329
	struct cached_dev *dc = container_of(buf, struct cached_dev, writeback_keys);

	BUG_ON(KEY_INODE(k) != dc->disk.id);

330 331 332
	return KEY_DIRTY(k);
}

333
static void refill_full_stripes(struct cached_dev *dc)
334
{
335 336 337 338 339
	struct keybuf *buf = &dc->writeback_keys;
	unsigned start_stripe, stripe, next_stripe;
	bool wrapped = false;

	stripe = offset_to_stripe(&dc->disk, KEY_OFFSET(&buf->last_scanned));
340

341 342
	if (stripe >= dc->disk.nr_stripes)
		stripe = 0;
343

344
	start_stripe = stripe;
345 346

	while (1) {
347 348
		stripe = find_next_bit(dc->disk.full_dirty_stripes,
				       dc->disk.nr_stripes, stripe);
349

350 351
		if (stripe == dc->disk.nr_stripes)
			goto next;
352

353 354 355 356 357 358 359 360 361 362 363 364 365 366 367 368 369 370 371 372 373 374 375
		next_stripe = find_next_zero_bit(dc->disk.full_dirty_stripes,
						 dc->disk.nr_stripes, stripe);

		buf->last_scanned = KEY(dc->disk.id,
					stripe * dc->disk.stripe_size, 0);

		bch_refill_keybuf(dc->disk.c, buf,
				  &KEY(dc->disk.id,
				       next_stripe * dc->disk.stripe_size, 0),
				  dirty_pred);

		if (array_freelist_empty(&buf->freelist))
			return;

		stripe = next_stripe;
next:
		if (wrapped && stripe > start_stripe)
			return;

		if (stripe == dc->disk.nr_stripes) {
			stripe = 0;
			wrapped = true;
		}
376 377 378
	}
}

379 380 381
/*
 * Returns true if we scanned the entire disk
 */
382 383 384
static bool refill_dirty(struct cached_dev *dc)
{
	struct keybuf *buf = &dc->writeback_keys;
385
	struct bkey start = KEY(dc->disk.id, 0, 0);
386
	struct bkey end = KEY(dc->disk.id, MAX_KEY_OFFSET, 0);
387 388 389 390 391 392 393 394 395 396
	struct bkey start_pos;

	/*
	 * make sure keybuf pos is inside the range for this disk - at bringup
	 * we might not be attached yet so this disk's inode nr isn't
	 * initialized then
	 */
	if (bkey_cmp(&buf->last_scanned, &start) < 0 ||
	    bkey_cmp(&buf->last_scanned, &end) > 0)
		buf->last_scanned = start;
397 398 399 400 401 402

	if (dc->partial_stripes_expensive) {
		refill_full_stripes(dc);
		if (array_freelist_empty(&buf->freelist))
			return false;
	}
403

404
	start_pos = buf->last_scanned;
405
	bch_refill_keybuf(dc->disk.c, buf, &end, dirty_pred);
406

407 408 409 410 411 412 413 414 415 416 417
	if (bkey_cmp(&buf->last_scanned, &end) < 0)
		return false;

	/*
	 * If we get to the end start scanning again from the beginning, and
	 * only scan up to where we initially started scanning from:
	 */
	buf->last_scanned = start;
	bch_refill_keybuf(dc->disk.c, buf, &start_pos, dirty_pred);

	return bkey_cmp(&buf->last_scanned, &start_pos) >= 0;
418 419 420 421 422 423 424 425 426 427
}

static int bch_writeback_thread(void *arg)
{
	struct cached_dev *dc = arg;
	bool searched_full_index;

	while (!kthread_should_stop()) {
		down_write(&dc->writeback_lock);
		if (!atomic_read(&dc->has_dirty) ||
428
		    (!test_bit(BCACHE_DEV_DETACHING, &dc->disk.flags) &&
429 430 431 432 433 434 435 436 437 438 439 440 441 442 443 444 445 446 447 448 449 450 451 452 453 454 455 456 457 458 459 460
		     !dc->writeback_running)) {
			up_write(&dc->writeback_lock);
			set_current_state(TASK_INTERRUPTIBLE);

			if (kthread_should_stop())
				return 0;

			try_to_freeze();
			schedule();
			continue;
		}

		searched_full_index = refill_dirty(dc);

		if (searched_full_index &&
		    RB_EMPTY_ROOT(&dc->writeback_keys.keys)) {
			atomic_set(&dc->has_dirty, 0);
			cached_dev_put(dc);
			SET_BDEV_STATE(&dc->sb, BDEV_STATE_CLEAN);
			bch_write_bdev_super(dc, NULL);
		}

		up_write(&dc->writeback_lock);

		bch_ratelimit_reset(&dc->writeback_rate);
		read_dirty(dc);

		if (searched_full_index) {
			unsigned delay = dc->writeback_delay * HZ;

			while (delay &&
			       !kthread_should_stop() &&
461
			       !test_bit(BCACHE_DEV_DETACHING, &dc->disk.flags))
462
				delay = schedule_timeout_interruptible(delay);
463 464 465 466
		}
	}

	return 0;
467 468
}

469 470
/* Init */

471 472 473 474 475 476
struct sectors_dirty_init {
	struct btree_op	op;
	unsigned	inode;
};

static int sectors_dirty_init_fn(struct btree_op *_op, struct btree *b,
477
				 struct bkey *k)
478
{
479 480
	struct sectors_dirty_init *op = container_of(_op,
						struct sectors_dirty_init, op);
481 482
	if (KEY_INODE(k) > op->inode)
		return MAP_DONE;
483

484 485 486 487 488
	if (KEY_DIRTY(k))
		bcache_dev_sectors_dirty_add(b->c, KEY_INODE(k),
					     KEY_START(k), KEY_SIZE(k));

	return MAP_CONTINUE;
489 490 491 492
}

void bch_sectors_dirty_init(struct cached_dev *dc)
{
493
	struct sectors_dirty_init op;
494

Kent Overstreet's avatar
Kent Overstreet committed
495
	bch_btree_op_init(&op.op, -1);
496 497
	op.inode = dc->disk.id;

498
	bch_btree_map_keys(&op.op, dc->disk.c, &KEY(op.inode, 0, 0),
499
			   sectors_dirty_init_fn, 0);
500 501

	dc->disk.sectors_dirty_last = bcache_dev_sectors_dirty(&dc->disk);
502 503
}

504
void bch_cached_dev_writeback_init(struct cached_dev *dc)
505
{
506
	sema_init(&dc->in_flight, 64);
507
	init_rwsem(&dc->writeback_lock);
508
	bch_keybuf_init(&dc->writeback_keys);
509 510 511 512 513 514 515

	dc->writeback_metadata		= true;
	dc->writeback_running		= true;
	dc->writeback_percent		= 10;
	dc->writeback_delay		= 30;
	dc->writeback_rate.rate		= 1024;

516 517 518
	dc->writeback_rate_update_seconds = 5;
	dc->writeback_rate_d_term	= 30;
	dc->writeback_rate_p_term_inverse = 6000;
519

520 521 522 523 524
	INIT_DELAYED_WORK(&dc->writeback_rate_update, update_writeback_rate);
}

int bch_cached_dev_writeback_start(struct cached_dev *dc)
{
525 526 527 528 529
	dc->writeback_thread = kthread_create(bch_writeback_thread, dc,
					      "bcache_writeback");
	if (IS_ERR(dc->writeback_thread))
		return PTR_ERR(dc->writeback_thread);

530 531 532
	schedule_delayed_work(&dc->writeback_rate_update,
			      dc->writeback_rate_update_seconds * HZ);

533 534
	bch_writeback_queue(dc);

535 536
	return 0;
}