xfs_log.c 111 KB
Newer Older
1
// SPDX-License-Identifier: GPL-2.0
Linus Torvalds's avatar
Linus Torvalds committed
2
/*
3 4
 * Copyright (c) 2000-2005 Silicon Graphics, Inc.
 * All Rights Reserved.
Linus Torvalds's avatar
Linus Torvalds committed
5 6
 */
#include "xfs.h"
7
#include "xfs_fs.h"
8
#include "xfs_shared.h"
9
#include "xfs_format.h"
10 11
#include "xfs_log_format.h"
#include "xfs_trans_resv.h"
Linus Torvalds's avatar
Linus Torvalds committed
12
#include "xfs_mount.h"
13
#include "xfs_errortag.h"
Linus Torvalds's avatar
Linus Torvalds committed
14
#include "xfs_error.h"
15 16 17
#include "xfs_trans.h"
#include "xfs_trans_priv.h"
#include "xfs_log.h"
Linus Torvalds's avatar
Linus Torvalds committed
18
#include "xfs_log_priv.h"
19
#include "xfs_trace.h"
20
#include "xfs_sysfs.h"
21
#include "xfs_sb.h"
22
#include "xfs_health.h"
Linus Torvalds's avatar
Linus Torvalds committed
23

24
kmem_zone_t	*xfs_log_ticket_zone;
Linus Torvalds's avatar
Linus Torvalds committed
25 26

/* Local miscellaneous function prototypes */
Mark Tinguely's avatar
Mark Tinguely committed
27 28 29 30 31 32
STATIC struct xlog *
xlog_alloc_log(
	struct xfs_mount	*mp,
	struct xfs_buftarg	*log_target,
	xfs_daddr_t		blk_offset,
	int			num_bblks);
33 34 35 36
STATIC int
xlog_space_left(
	struct xlog		*log,
	atomic64_t		*head);
Mark Tinguely's avatar
Mark Tinguely committed
37 38 39
STATIC void
xlog_dealloc_log(
	struct xlog		*log);
Linus Torvalds's avatar
Linus Torvalds committed
40 41

/* local state machine functions */
42
STATIC void xlog_state_done_syncing(
43
	struct xlog_in_core	*iclog);
44 45
STATIC void xlog_state_do_callback(
	struct xlog		*log);
Mark Tinguely's avatar
Mark Tinguely committed
46 47 48 49 50 51 52 53 54
STATIC int
xlog_state_get_iclog_space(
	struct xlog		*log,
	int			len,
	struct xlog_in_core	**iclog,
	struct xlog_ticket	*ticket,
	int			*continued_write,
	int			*logoffsetp);
STATIC void
55
xlog_grant_push_ail(
Mark Tinguely's avatar
Mark Tinguely committed
56 57 58
	struct xlog		*log,
	int			need_bytes);
STATIC void
59 60 61
xlog_sync(
	struct xlog		*log,
	struct xlog_in_core	*iclog);
62
#if defined(DEBUG)
Mark Tinguely's avatar
Mark Tinguely committed
63 64 65
STATIC void
xlog_verify_dest_ptr(
	struct xlog		*log,
66
	void			*ptr);
67 68
STATIC void
xlog_verify_grant_tail(
Mark Tinguely's avatar
Mark Tinguely committed
69 70 71 72 73
	struct xlog *log);
STATIC void
xlog_verify_iclog(
	struct xlog		*log,
	struct xlog_in_core	*iclog,
74
	int			count);
Mark Tinguely's avatar
Mark Tinguely committed
75 76 77
STATIC void
xlog_verify_tail_lsn(
	struct xlog		*log,
78
	struct xlog_in_core	*iclog);
Linus Torvalds's avatar
Linus Torvalds committed
79 80
#else
#define xlog_verify_dest_ptr(a,b)
81
#define xlog_verify_grant_tail(a)
82
#define xlog_verify_iclog(a,b,c)
83
#define xlog_verify_tail_lsn(a,b)
Linus Torvalds's avatar
Linus Torvalds committed
84 85
#endif

Mark Tinguely's avatar
Mark Tinguely committed
86 87 88
STATIC int
xlog_iclogs_empty(
	struct xlog		*log);
Linus Torvalds's avatar
Linus Torvalds committed
89

90 91 92
static int
xfs_log_cover(struct xfs_mount *);

93
static void
94
xlog_grant_sub_space(
95 96 97
	struct xlog		*log,
	atomic64_t		*head,
	int			bytes)
98
{
99 100
	int64_t	head_val = atomic64_read(head);
	int64_t new, old;
101

102 103
	do {
		int	cycle, space;
104

105
		xlog_crack_grant_head_val(head_val, &cycle, &space);
106

107 108 109 110 111 112 113 114 115 116
		space -= bytes;
		if (space < 0) {
			space += log->l_logsize;
			cycle--;
		}

		old = head_val;
		new = xlog_assign_grant_head_val(cycle, space);
		head_val = atomic64_cmpxchg(head, old, new);
	} while (head_val != old);
117 118 119
}

static void
120
xlog_grant_add_space(
121 122 123
	struct xlog		*log,
	atomic64_t		*head,
	int			bytes)
124
{
125 126
	int64_t	head_val = atomic64_read(head);
	int64_t new, old;
127

128 129 130
	do {
		int		tmp;
		int		cycle, space;
131

132
		xlog_crack_grant_head_val(head_val, &cycle, &space);
133

134 135 136 137 138 139 140 141 142 143 144 145
		tmp = log->l_logsize - space;
		if (tmp > bytes)
			space += bytes;
		else {
			space = bytes - tmp;
			cycle++;
		}

		old = head_val;
		new = xlog_assign_grant_head_val(cycle, space);
		head_val = atomic64_cmpxchg(head, old, new);
	} while (head_val != old);
146
}
147

148 149 150 151 152 153 154 155 156
STATIC void
xlog_grant_head_init(
	struct xlog_grant_head	*head)
{
	xlog_assign_grant_head(&head->grant, 1, 0);
	INIT_LIST_HEAD(&head->waiters);
	spin_lock_init(&head->lock);
}

157 158 159 160 161 162 163 164 165 166 167 168
STATIC void
xlog_grant_head_wake_all(
	struct xlog_grant_head	*head)
{
	struct xlog_ticket	*tic;

	spin_lock(&head->lock);
	list_for_each_entry(tic, &head->waiters, t_queue)
		wake_up_process(tic->t_task);
	spin_unlock(&head->lock);
}

169 170
static inline int
xlog_ticket_reservation(
171
	struct xlog		*log,
172 173
	struct xlog_grant_head	*head,
	struct xlog_ticket	*tic)
174
{
175 176 177 178
	if (head == &log->l_write_head) {
		ASSERT(tic->t_flags & XLOG_TIC_PERM_RESERV);
		return tic->t_unit_res;
	} else {
179
		if (tic->t_flags & XLOG_TIC_PERM_RESERV)
180
			return tic->t_unit_res * tic->t_cnt;
181
		else
182
			return tic->t_unit_res;
183 184 185 186
	}
}

STATIC bool
187
xlog_grant_head_wake(
188
	struct xlog		*log,
189
	struct xlog_grant_head	*head,
190 191 192 193
	int			*free_bytes)
{
	struct xlog_ticket	*tic;
	int			need_bytes;
194
	bool			woken_task = false;
195

196
	list_for_each_entry(tic, &head->waiters, t_queue) {
197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218

		/*
		 * There is a chance that the size of the CIL checkpoints in
		 * progress at the last AIL push target calculation resulted in
		 * limiting the target to the log head (l_last_sync_lsn) at the
		 * time. This may not reflect where the log head is now as the
		 * CIL checkpoints may have completed.
		 *
		 * Hence when we are woken here, it may be that the head of the
		 * log that has moved rather than the tail. As the tail didn't
		 * move, there still won't be space available for the
		 * reservation we require.  However, if the AIL has already
		 * pushed to the target defined by the old log head location, we
		 * will hang here waiting for something else to update the AIL
		 * push target.
		 *
		 * Therefore, if there isn't space to wake the first waiter on
		 * the grant head, we need to push the AIL again to ensure the
		 * target reflects both the current log tail and log head
		 * position before we wait for the tail to move again.
		 */

219
		need_bytes = xlog_ticket_reservation(log, head, tic);
220 221 222
		if (*free_bytes < need_bytes) {
			if (!woken_task)
				xlog_grant_push_ail(log, need_bytes);
223
			return false;
224
		}
225

226 227
		*free_bytes -= need_bytes;
		trace_xfs_log_grant_wake_up(log, tic);
228
		wake_up_process(tic->t_task);
229
		woken_task = true;
230 231 232 233 234 235
	}

	return true;
}

STATIC int
236
xlog_grant_head_wait(
237
	struct xlog		*log,
238
	struct xlog_grant_head	*head,
239
	struct xlog_ticket	*tic,
240 241
	int			need_bytes) __releases(&head->lock)
					    __acquires(&head->lock)
242
{
243
	list_add_tail(&tic->t_queue, &head->waiters);
244 245

	do {
246
		if (xlog_is_shutdown(log))
247 248 249
			goto shutdown;
		xlog_grant_push_ail(log, need_bytes);

250
		__set_current_state(TASK_UNINTERRUPTIBLE);
251
		spin_unlock(&head->lock);
252

253
		XFS_STATS_INC(log->l_mp, xs_sleep_logspace);
254

255 256
		trace_xfs_log_grant_sleep(log, tic);
		schedule();
257 258
		trace_xfs_log_grant_wake(log, tic);

259
		spin_lock(&head->lock);
260
		if (xlog_is_shutdown(log))
261
			goto shutdown;
262
	} while (xlog_space_left(log, &head->grant) < need_bytes);
263 264 265 266 267

	list_del_init(&tic->t_queue);
	return 0;
shutdown:
	list_del_init(&tic->t_queue);
268
	return -EIO;
269 270
}

271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289
/*
 * Atomically get the log space required for a log ticket.
 *
 * Once a ticket gets put onto head->waiters, it will only return after the
 * needed reservation is satisfied.
 *
 * This function is structured so that it has a lock free fast path. This is
 * necessary because every new transaction reservation will come through this
 * path. Hence any lock will be globally hot if we take it unconditionally on
 * every pass.
 *
 * As tickets are only ever moved on and off head->waiters under head->lock, we
 * only need to take that lock if we are going to add the ticket to the queue
 * and sleep. We can avoid taking the lock if the ticket was never added to
 * head->waiters because the t_queue list head will be empty and we hold the
 * only reference to it so it can safely be checked unlocked.
 */
STATIC int
xlog_grant_head_check(
290
	struct xlog		*log,
291 292 293 294 295 296 297
	struct xlog_grant_head	*head,
	struct xlog_ticket	*tic,
	int			*need_bytes)
{
	int			free_bytes;
	int			error = 0;

298
	ASSERT(!xlog_in_recovery(log));
299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 321 322 323 324

	/*
	 * If there are other waiters on the queue then give them a chance at
	 * logspace before us.  Wake up the first waiters, if we do not wake
	 * up all the waiters then go to sleep waiting for more free space,
	 * otherwise try to get some space for this transaction.
	 */
	*need_bytes = xlog_ticket_reservation(log, head, tic);
	free_bytes = xlog_space_left(log, &head->grant);
	if (!list_empty_careful(&head->waiters)) {
		spin_lock(&head->lock);
		if (!xlog_grant_head_wake(log, head, &free_bytes) ||
		    free_bytes < *need_bytes) {
			error = xlog_grant_head_wait(log, head, tic,
						     *need_bytes);
		}
		spin_unlock(&head->lock);
	} else if (free_bytes < *need_bytes) {
		spin_lock(&head->lock);
		error = xlog_grant_head_wait(log, head, tic, *need_bytes);
		spin_unlock(&head->lock);
	}

	return error;
}

325 326 327 328 329 330 331 332 333 334 335 336 337 338 339 340 341 342 343 344 345 346 347
static void
xlog_tic_reset_res(xlog_ticket_t *tic)
{
	tic->t_res_num = 0;
	tic->t_res_arr_sum = 0;
	tic->t_res_num_ophdrs = 0;
}

static void
xlog_tic_add_region(xlog_ticket_t *tic, uint len, uint type)
{
	if (tic->t_res_num == XLOG_TIC_LEN_MAX) {
		/* add to overflow and start again */
		tic->t_res_o_flow += tic->t_res_arr_sum;
		tic->t_res_num = 0;
		tic->t_res_arr_sum = 0;
	}

	tic->t_res_arr[tic->t_res_num].r_len = len;
	tic->t_res_arr[tic->t_res_num].r_type = type;
	tic->t_res_arr_sum += len;
	tic->t_res_num++;
}
348

349 350 351 352 353
bool
xfs_log_writable(
	struct xfs_mount	*mp)
{
	/*
354 355 356 357
	 * Do not write to the log on norecovery mounts, if the data or log
	 * devices are read-only, or if the filesystem is shutdown. Read-only
	 * mounts allow internal writes for log recovery and unmount purposes,
	 * so don't restrict that case.
358
	 */
359
	if (xfs_has_norecovery(mp))
360
		return false;
361 362
	if (xfs_readonly_buftarg(mp->m_ddev_targp))
		return false;
363 364
	if (xfs_readonly_buftarg(mp->m_log->l_targ))
		return false;
365
	if (xlog_is_shutdown(mp->m_log))
366 367 368 369
		return false;
	return true;
}

370 371 372 373 374 375 376 377
/*
 * Replenish the byte reservation required by moving the grant write head.
 */
int
xfs_log_regrant(
	struct xfs_mount	*mp,
	struct xlog_ticket	*tic)
{
378
	struct xlog		*log = mp->m_log;
379 380 381
	int			need_bytes;
	int			error = 0;

382
	if (xlog_is_shutdown(log))
383
		return -EIO;
384

385
	XFS_STATS_INC(mp, xs_try_logspace);
386 387 388 389 390 391 392 393 394 395 396 397 398 399 400 401 402 403 404 405 406 407 408 409 410 411 412 413 414 415 416 417 418 419 420 421 422 423 424 425 426

	/*
	 * This is a new transaction on the ticket, so we need to change the
	 * transaction ID so that the next transaction has a different TID in
	 * the log. Just add one to the existing tid so that we can see chains
	 * of rolling transactions in the log easily.
	 */
	tic->t_tid++;

	xlog_grant_push_ail(log, tic->t_unit_res);

	tic->t_curr_res = tic->t_unit_res;
	xlog_tic_reset_res(tic);

	if (tic->t_cnt > 0)
		return 0;

	trace_xfs_log_regrant(log, tic);

	error = xlog_grant_head_check(log, &log->l_write_head, tic,
				      &need_bytes);
	if (error)
		goto out_error;

	xlog_grant_add_space(log, &log->l_write_head.grant, need_bytes);
	trace_xfs_log_regrant_exit(log, tic);
	xlog_verify_grant_tail(log);
	return 0;

out_error:
	/*
	 * If we are failing, make sure the ticket doesn't have any current
	 * reservations.  We don't want to add this back when the ticket/
	 * transaction gets cancelled.
	 */
	tic->t_curr_res = 0;
	tic->t_cnt = 0;	/* ungrant will give back unit_res * t_cnt. */
	return error;
}

/*
427
 * Reserve log space and return a ticket corresponding to the reservation.
428 429 430 431 432 433 434 435 436 437 438 439
 *
 * Each reservation is going to reserve extra space for a log record header.
 * When writes happen to the on-disk log, we don't subtract the length of the
 * log record header from any reservation.  By wasting space in each
 * reservation, we prevent over allocation problems.
 */
int
xfs_log_reserve(
	struct xfs_mount	*mp,
	int		 	unit_bytes,
	int		 	cnt,
	struct xlog_ticket	**ticp,
440
	uint8_t		 	client,
441
	bool			permanent)
442
{
443
	struct xlog		*log = mp->m_log;
444 445 446 447 448 449
	struct xlog_ticket	*tic;
	int			need_bytes;
	int			error = 0;

	ASSERT(client == XFS_TRANSACTION || client == XFS_LOG);

450
	if (xlog_is_shutdown(log))
451
		return -EIO;
452

453
	XFS_STATS_INC(mp, xs_try_logspace);
454 455

	ASSERT(*ticp == NULL);
456
	tic = xlog_ticket_alloc(log, unit_bytes, cnt, client, permanent);
457 458
	*ticp = tic;

459 460
	xlog_grant_push_ail(log, tic->t_cnt ? tic->t_unit_res * tic->t_cnt
					    : tic->t_unit_res);
461 462 463 464 465 466 467 468 469 470 471 472 473 474 475 476 477 478 479 480 481 482 483 484 485

	trace_xfs_log_reserve(log, tic);

	error = xlog_grant_head_check(log, &log->l_reserve_head, tic,
				      &need_bytes);
	if (error)
		goto out_error;

	xlog_grant_add_space(log, &log->l_reserve_head.grant, need_bytes);
	xlog_grant_add_space(log, &log->l_write_head.grant, need_bytes);
	trace_xfs_log_reserve_exit(log, tic);
	xlog_verify_grant_tail(log);
	return 0;

out_error:
	/*
	 * If we are failing, make sure the ticket doesn't have any current
	 * reservations.  We don't want to add this back when the ticket/
	 * transaction gets cancelled.
	 */
	tic->t_curr_res = 0;
	tic->t_cnt = 0;	/* ungrant will give back unit_res * t_cnt. */
	return error;
}

486 487 488 489 490
/*
 * Run all the pending iclog callbacks and wake log force waiters and iclog
 * space waiters so they can process the newly set shutdown state. We really
 * don't care what order we process callbacks here because the log is shut down
 * and so state cannot change on disk anymore.
491 492 493 494 495
 *
 * We avoid processing actively referenced iclogs so that we don't run callbacks
 * while the iclog owner might still be preparing the iclog for IO submssion.
 * These will be caught by xlog_state_iclog_release() and call this function
 * again to process any callbacks that may have been added to that iclog.
496 497 498 499 500 501 502 503 504 505 506
 */
static void
xlog_state_shutdown_callbacks(
	struct xlog		*log)
{
	struct xlog_in_core	*iclog;
	LIST_HEAD(cb_list);

	spin_lock(&log->l_icloglock);
	iclog = log->l_iclog;
	do {
507 508 509 510
		if (atomic_read(&iclog->ic_refcnt)) {
			/* Reference holder will re-run iclog callbacks. */
			continue;
		}
511
		list_splice_init(&iclog->ic_callbacks, &cb_list);
512
		wake_up_all(&iclog->ic_write_wait);
513 514 515 516 517 518 519 520 521
		wake_up_all(&iclog->ic_force_wait);
	} while ((iclog = iclog->ic_next) != log->l_iclog);

	wake_up_all(&log->l_flush_wait);
	spin_unlock(&log->l_icloglock);

	xlog_cil_process_committed(&cb_list);
}

522 523
/*
 * Flush iclog to disk if this is the last reference to the given iclog and the
524 525 526 527 528 529 530 531 532 533 534 535 536 537 538 539 540 541 542 543 544 545 546
 * it is in the WANT_SYNC state.
 *
 * If the caller passes in a non-zero @old_tail_lsn and the current log tail
 * does not match, there may be metadata on disk that must be persisted before
 * this iclog is written.  To satisfy that requirement, set the
 * XLOG_ICL_NEED_FLUSH flag as a condition for writing this iclog with the new
 * log tail value.
 *
 * If XLOG_ICL_NEED_FUA is already set on the iclog, we need to ensure that the
 * log tail is updated correctly. NEED_FUA indicates that the iclog will be
 * written to stable storage, and implies that a commit record is contained
 * within the iclog. We need to ensure that the log tail does not move beyond
 * the tail that the first commit record in the iclog ordered against, otherwise
 * correct recovery of that checkpoint becomes dependent on future operations
 * performed on this iclog.
 *
 * Hence if NEED_FUA is set and the current iclog tail lsn is empty, write the
 * current tail into iclog. Once the iclog tail is set, future operations must
 * not modify it, otherwise they potentially violate ordering constraints for
 * the checkpoint commit that wrote the initial tail lsn value. The tail lsn in
 * the iclog will get zeroed on activation of the iclog after sync, so we
 * always capture the tail lsn on the iclog on the first NEED_FUA release
 * regardless of the number of active reference counts on this iclog.
547
 */
548

549
int
550 551
xlog_state_release_iclog(
	struct xlog		*log,
552 553
	struct xlog_in_core	*iclog,
	xfs_lsn_t		old_tail_lsn)
554
{
555
	xfs_lsn_t		tail_lsn;
556 557
	bool			last_ref;

558 559
	lockdep_assert_held(&log->l_icloglock);

560
	trace_xlog_iclog_release(iclog, _RET_IP_);
561 562 563 564 565 566 567 568 569 570 571
	/*
	 * Grabbing the current log tail needs to be atomic w.r.t. the writing
	 * of the tail LSN into the iclog so we guarantee that the log tail does
	 * not move between deciding if a cache flush is required and writing
	 * the LSN into the iclog below.
	 */
	if (old_tail_lsn || iclog->ic_state == XLOG_STATE_WANT_SYNC) {
		tail_lsn = xlog_assign_tail_lsn(log->l_mp);

		if (old_tail_lsn && tail_lsn != old_tail_lsn)
			iclog->ic_flags |= XLOG_ICL_NEED_FLUSH;
572 573 574 575

		if ((iclog->ic_flags & XLOG_ICL_NEED_FUA) &&
		    !iclog->ic_header.h_tail_lsn)
			iclog->ic_header.h_tail_lsn = cpu_to_be64(tail_lsn);
576 577
	}

578 579 580 581 582 583 584 585 586 587 588 589 590 591 592 593 594
	last_ref = atomic_dec_and_test(&iclog->ic_refcnt);

	if (xlog_is_shutdown(log)) {
		/*
		 * If there are no more references to this iclog, process the
		 * pending iclog callbacks that were waiting on the release of
		 * this iclog.
		 */
		if (last_ref) {
			spin_unlock(&log->l_icloglock);
			xlog_state_shutdown_callbacks(log);
			spin_lock(&log->l_icloglock);
		}
		return -EIO;
	}

	if (!last_ref)
595 596 597 598 599
		return 0;

	if (iclog->ic_state != XLOG_STATE_WANT_SYNC) {
		ASSERT(iclog->ic_state == XLOG_STATE_ACTIVE);
		return 0;
600 601
	}

602
	iclog->ic_state = XLOG_STATE_SYNCING;
603 604 605
	if (!iclog->ic_header.h_tail_lsn)
		iclog->ic_header.h_tail_lsn = cpu_to_be64(tail_lsn);
	xlog_verify_tail_lsn(log, iclog);
606 607 608 609 610
	trace_xlog_iclog_syncing(iclog, _RET_IP_);

	spin_unlock(&log->l_icloglock);
	xlog_sync(log, iclog);
	spin_lock(&log->l_icloglock);
611 612 613
	return 0;
}

Linus Torvalds's avatar
Linus Torvalds committed
614 615 616 617 618 619 620 621 622 623 624
/*
 * Mount a log filesystem
 *
 * mp		- ubiquitous xfs mount point structure
 * log_target	- buftarg of on-disk log device
 * blk_offset	- Start block # where block size is 512 bytes (BBSIZE)
 * num_bblocks	- Number of BBSIZE blocks in on-disk log
 *
 * Return error or zero.
 */
int
625 626 627 628 629
xfs_log_mount(
	xfs_mount_t	*mp,
	xfs_buftarg_t	*log_target,
	xfs_daddr_t	blk_offset,
	int		num_bblks)
Linus Torvalds's avatar
Linus Torvalds committed
630
{
631
	struct xlog	*log;
632
	bool		fatal = xfs_has_crc(mp);
633 634
	int		error = 0;
	int		min_logfsbs;
635

636
	if (!xfs_has_norecovery(mp)) {
637 638 639
		xfs_notice(mp, "Mounting V%d Filesystem",
			   XFS_SB_VERSION_NUM(&mp->m_sb));
	} else {
640
		xfs_notice(mp,
641 642
"Mounting V%d filesystem in no-recovery mode. Filesystem will be inconsistent.",
			   XFS_SB_VERSION_NUM(&mp->m_sb));
643
		ASSERT(xfs_is_readonly(mp));
Linus Torvalds's avatar
Linus Torvalds committed
644 645
	}

646 647 648
	log = xlog_alloc_log(mp, log_target, blk_offset, num_bblks);
	if (IS_ERR(log)) {
		error = PTR_ERR(log);
649 650
		goto out;
	}
651
	mp->m_log = log;
Linus Torvalds's avatar
Linus Torvalds committed
652

653 654 655 656 657 658 659 660 661 662 663 664 665 666 667 668 669 670 671 672
	/*
	 * Validate the given log space and drop a critical message via syslog
	 * if the log size is too small that would lead to some unexpected
	 * situations in transaction log space reservation stage.
	 *
	 * Note: we can't just reject the mount if the validation fails.  This
	 * would mean that people would have to downgrade their kernel just to
	 * remedy the situation as there is no way to grow the log (short of
	 * black magic surgery with xfs_db).
	 *
	 * We can, however, reject mounts for CRC format filesystems, as the
	 * mkfs binary being used to make the filesystem should never create a
	 * filesystem with a log that is too small.
	 */
	min_logfsbs = xfs_log_calc_minimum_size(mp);

	if (mp->m_sb.sb_logblocks < min_logfsbs) {
		xfs_warn(mp,
		"Log size %d blocks too small, minimum size is %d blocks",
			 mp->m_sb.sb_logblocks, min_logfsbs);
673
		error = -EINVAL;
674 675 676 677
	} else if (mp->m_sb.sb_logblocks > XFS_MAX_LOG_BLOCKS) {
		xfs_warn(mp,
		"Log size %d blocks too large, maximum size is %lld blocks",
			 mp->m_sb.sb_logblocks, XFS_MAX_LOG_BLOCKS);
678
		error = -EINVAL;
679 680 681 682 683
	} else if (XFS_FSB_TO_B(mp, mp->m_sb.sb_logblocks) > XFS_MAX_LOG_BYTES) {
		xfs_warn(mp,
		"log size %lld bytes too large, maximum size is %lld bytes",
			 XFS_FSB_TO_B(mp, mp->m_sb.sb_logblocks),
			 XFS_MAX_LOG_BYTES);
684
		error = -EINVAL;
685 686 687 688 689 690 691
	} else if (mp->m_sb.sb_logsunit > 1 &&
		   mp->m_sb.sb_logsunit % mp->m_sb.sb_blocksize) {
		xfs_warn(mp,
		"log stripe unit %u bytes must be a multiple of block size",
			 mp->m_sb.sb_logsunit);
		error = -EINVAL;
		fatal = true;
692 693
	}
	if (error) {
694 695 696 697 698
		/*
		 * Log check errors are always fatal on v5; or whenever bad
		 * metadata leads to a crash.
		 */
		if (fatal) {
699 700 701 702
			xfs_crit(mp, "AAIEEE! Log failed size checks. Abort!");
			ASSERT(0);
			goto out_free_log;
		}
703
		xfs_crit(mp, "Log size out of supported range.");
704
		xfs_crit(mp,
705
"Continuing onwards, but if log hangs are experienced then please report this message in the bug report.");
706 707
	}

708 709 710 711 712
	/*
	 * Initialize the AIL now we have a log.
	 */
	error = xfs_trans_ail_init(mp);
	if (error) {
713
		xfs_warn(mp, "AIL initialisation failed: error %d", error);
714
		goto out_free_log;
715
	}
716
	log->l_ailp = mp->m_ail;
717

Linus Torvalds's avatar
Linus Torvalds committed
718 719 720 721
	/*
	 * skip log recovery on a norecovery mount.  pretend it all
	 * just worked.
	 */
722
	if (!xfs_has_norecovery(mp)) {
723 724 725 726 727 728
		/*
		 * log recovery ignores readonly state and so we need to clear
		 * mount-based read only state so it can write to disk.
		 */
		bool	readonly = test_and_clear_bit(XFS_OPSTATE_READONLY,
						&mp->m_opstate);
729
		error = xlog_recover(log);
Linus Torvalds's avatar
Linus Torvalds committed
730
		if (readonly)
731
			set_bit(XFS_OPSTATE_READONLY, &mp->m_opstate);
Linus Torvalds's avatar
Linus Torvalds committed
732
		if (error) {
733 734
			xfs_warn(mp, "log mount/recovery failed: error %d",
				error);
735
			xlog_recover_cancel(log);
736
			goto out_destroy_ail;
Linus Torvalds's avatar
Linus Torvalds committed
737 738 739
		}
	}

740
	error = xfs_sysfs_init(&log->l_kobj, &xfs_log_ktype, &mp->m_kobj,
741 742 743 744
			       "log");
	if (error)
		goto out_destroy_ail;

Linus Torvalds's avatar
Linus Torvalds committed
745
	/* Normal transactions can now occur */
746
	clear_bit(XLOG_ACTIVE_RECOVERY, &log->l_opstate);
Linus Torvalds's avatar
Linus Torvalds committed
747

748 749 750 751 752
	/*
	 * Now the log has been fully initialised and we know were our
	 * space grant counters are, we can initialise the permanent ticket
	 * needed for delayed logging to work.
	 */
753
	xlog_cil_init_post_recovery(log);
754

Linus Torvalds's avatar
Linus Torvalds committed
755
	return 0;
756 757 758 759

out_destroy_ail:
	xfs_trans_ail_destroy(mp);
out_free_log:
760
	xlog_dealloc_log(log);
761
out:
762
	return error;
763
}
Linus Torvalds's avatar
Linus Torvalds committed
764 765

/*
766 767 768 769
 * Finish the recovery of the file system.  This is separate from the
 * xfs_log_mount() call, because it depends on the code in xfs_mountfs() to read
 * in the root and real-time bitmap inodes between calling xfs_log_mount() and
 * here.
Linus Torvalds's avatar
Linus Torvalds committed
770
 *
771 772 773
 * If we finish recovery successfully, start the background log work. If we are
 * not doing recovery, then we have a RO filesystem and we don't need to start
 * it.
Linus Torvalds's avatar
Linus Torvalds committed
774 775
 */
int
776 777
xfs_log_mount_finish(
	struct xfs_mount	*mp)
Linus Torvalds's avatar
Linus Torvalds committed
778
{
779
	struct xlog		*log = mp->m_log;
780
	bool			readonly;
781
	int			error = 0;
Linus Torvalds's avatar
Linus Torvalds committed
782

783
	if (xfs_has_norecovery(mp)) {
784
		ASSERT(xfs_is_readonly(mp));
785
		return 0;
Linus Torvalds's avatar
Linus Torvalds committed
786 787
	}

788 789 790 791 792 793
	/*
	 * log recovery ignores readonly state and so we need to clear
	 * mount-based read only state so it can write to disk.
	 */
	readonly = test_and_clear_bit(XFS_OPSTATE_READONLY, &mp->m_opstate);

794 795 796 797 798 799 800 801
	/*
	 * During the second phase of log recovery, we need iget and
	 * iput to behave like they do for an active filesystem.
	 * xfs_fs_drop_inode needs to be able to prevent the deletion
	 * of inodes before we're done replaying log items on those
	 * inodes.  Turn it off immediately after recovery finishes
	 * so that we don't leak the quota inodes if subsequent mount
	 * activities fail.
802 803 804 805 806 807
	 *
	 * We let all inodes involved in redo item processing end up on
	 * the LRU instead of being evicted immediately so that if we do
	 * something to an unlinked inode, the irele won't cause
	 * premature truncation and freeing of the inode, which results
	 * in log recovery failure.  We have to evict the unreferenced
808
	 * lru inodes after clearing SB_ACTIVE because we don't
809 810 811 812
	 * otherwise clean up the lru if there's a subsequent failure in
	 * xfs_mountfs, which leads to us leaking the inodes if nothing
	 * else (e.g. quotacheck) references the inodes before the
	 * mount failure occurs.
813
	 */
814
	mp->m_super->s_flags |= SB_ACTIVE;
815
	if (xlog_recovery_needed(log))
816
		error = xlog_recover_finish(log);
817 818
	if (!error)
		xfs_log_work_queue(mp);
819
	mp->m_super->s_flags &= ~SB_ACTIVE;
820
	evict_inodes(mp->m_super);
821

822 823 824 825 826 827 828 829 830
	/*
	 * Drain the buffer LRU after log recovery. This is required for v4
	 * filesystems to avoid leaving around buffers with NULL verifier ops,
	 * but we do it unconditionally to make sure we're always in a clean
	 * cache state after mount.
	 *
	 * Don't push in the error case because the AIL may have pending intents
	 * that aren't removed until recovery is cancelled.
	 */
831
	if (xlog_recovery_needed(log)) {
832 833 834 835 836 837 838 839
		if (!error) {
			xfs_log_force(mp, XFS_LOG_SYNC);
			xfs_ail_push_all_sync(mp->m_ail);
		}
		xfs_notice(mp, "Ending recovery (logdev: %s)",
				mp->m_logname ? mp->m_logname : "internal");
	} else {
		xfs_info(mp, "Ending clean mount");
840
	}
841
	xfs_buftarg_drain(mp->m_ddev_targp);
842

843
	clear_bit(XLOG_RECOVERY_NEEDED, &log->l_opstate);
844
	if (readonly)
845
		set_bit(XFS_OPSTATE_READONLY, &mp->m_opstate);
846

847
	/* Make sure the log is dead if we're returning failure. */
848
	ASSERT(!error || xlog_is_shutdown(log));
849

850 851 852 853 854 855 856
	return error;
}

/*
 * The mount has failed. Cancel the recovery if it hasn't completed and destroy
 * the log.
 */
857
void
858 859 860
xfs_log_mount_cancel(
	struct xfs_mount	*mp)
{
861
	xlog_recover_cancel(mp->m_log);
862
	xfs_log_unmount(mp);
Linus Torvalds's avatar
Linus Torvalds committed
863 864
}

865 866 867 868 869 870 871 872 873
/*
 * Flush out the iclog to disk ensuring that device caches are flushed and
 * the iclog hits stable storage before any completion waiters are woken.
 */
static inline int
xlog_force_iclog(
	struct xlog_in_core	*iclog)
{
	atomic_inc(&iclog->ic_refcnt);
874
	iclog->ic_flags |= XLOG_ICL_NEED_FLUSH | XLOG_ICL_NEED_FUA;
875 876 877 878 879
	if (iclog->ic_state == XLOG_STATE_ACTIVE)
		xlog_state_switch_iclogs(iclog->ic_log, iclog, 0);
	return xlog_state_release_iclog(iclog->ic_log, iclog, 0);
}

880
/*
881 882 883 884
 * Wait for the iclog and all prior iclogs to be written disk as required by the
 * log force state machine. Waiting on ic_force_wait ensures iclog completions
 * have been ordered and callbacks run before we are woken here, hence
 * guaranteeing that all the iclogs up to this one are on stable storage.
885
 */
886
int
887 888 889 890 891 892
xlog_wait_on_iclog(
	struct xlog_in_core	*iclog)
		__releases(iclog->ic_log->l_icloglock)
{
	struct xlog		*log = iclog->ic_log;

893
	trace_xlog_iclog_wait_on(iclog, _RET_IP_);
894
	if (!xlog_is_shutdown(log) &&
895 896 897 898 899 900 901 902
	    iclog->ic_state != XLOG_STATE_ACTIVE &&
	    iclog->ic_state != XLOG_STATE_DIRTY) {
		XFS_STATS_INC(log->l_mp, xs_log_force_sleep);
		xlog_wait(&iclog->ic_force_wait, &log->l_icloglock);
	} else {
		spin_unlock(&log->l_icloglock);
	}

903
	if (xlog_is_shutdown(log))
904 905 906 907
		return -EIO;
	return 0;
}

Linus Torvalds's avatar
Linus Torvalds committed
908
/*
909 910 911
 * Write out an unmount record using the ticket provided. We have to account for
 * the data space used in the unmount ticket as this write is not done from a
 * transaction context that has already done the accounting for us.
Linus Torvalds's avatar
Linus Torvalds committed
912
 */
913 914 915
static int
xlog_write_unmount_record(
	struct xlog		*log,
916
	struct xlog_ticket	*ticket)
917
{
918
	struct xfs_unmount_log_format ulf = {
919 920 921
		.magic = XLOG_UNMOUNT_TYPE,
	};
	struct xfs_log_iovec reg = {
922 923
		.i_addr = &ulf,
		.i_len = sizeof(ulf),
924 925 926 927 928 929
		.i_type = XLOG_REG_TYPE_UNMOUNT,
	};
	struct xfs_log_vec vec = {
		.lv_niovecs = 1,
		.lv_iovecp = &reg,
	};
930 931 932

	/* account for space used by record data */
	ticket->t_curr_res -= sizeof(ulf);
933

934
	return xlog_write(log, NULL, &vec, ticket, XLOG_UNMOUNT_TRANS);
935 936 937 938 939 940 941 942 943 944 945
}

/*
 * Mark the filesystem clean by writing an unmount record to the head of the
 * log.
 */
static void
xlog_unmount_write(
	struct xlog		*log)
{
	struct xfs_mount	*mp = log->l_mp;
946 947 948 949 950 951 952 953
	struct xlog_in_core	*iclog;
	struct xlog_ticket	*tic = NULL;
	int			error;

	error = xfs_log_reserve(mp, 600, 1, &tic, XFS_LOG, 0);
	if (error)
		goto out_err;

954
	error = xlog_write_unmount_record(log, tic);
955 956
	/*
	 * At this point, we're umounting anyway, so there's no point in
957
	 * transitioning log state to shutdown. Just continue...
958 959 960 961 962 963 964
	 */
out_err:
	if (error)
		xfs_alert(mp, "%s: unmount record failed", __func__);

	spin_lock(&log->l_icloglock);
	iclog = log->l_iclog;
965
	error = xlog_force_iclog(iclog);
966
	xlog_wait_on_iclog(iclog);
967 968 969

	if (tic) {
		trace_xfs_log_umount_write(log, tic);
970
		xfs_log_ticket_ungrant(log, tic);
971 972 973
	}
}

974 975 976 977 978 979 980 981 982 983 984 985
static void
xfs_log_unmount_verify_iclog(
	struct xlog		*log)
{
	struct xlog_in_core	*iclog = log->l_iclog;

	do {
		ASSERT(iclog->ic_state == XLOG_STATE_ACTIVE);
		ASSERT(iclog->ic_offset == 0);
	} while ((iclog = iclog->ic_next) != log->l_iclog);
}

Linus Torvalds's avatar
Linus Torvalds committed
986 987 988 989
/*
 * Unmount record used to have a string "Unmount filesystem--" in the
 * data section where the "Un" was really a magic number (XLOG_UNMOUNT_TYPE).
 * We just write the magic number now since that particular field isn't
990
 * currently architecture converted and "Unmount" is a bit foo.
Linus Torvalds's avatar
Linus Torvalds committed
991 992
 * As far as I know, there weren't any dependencies on the old behaviour.
 */
993
static void
994 995
xfs_log_unmount_write(
	struct xfs_mount	*mp)
Linus Torvalds's avatar
Linus Torvalds committed
996
{
997
	struct xlog		*log = mp->m_log;
Linus Torvalds's avatar
Linus Torvalds committed
998

999
	if (!xfs_log_writable(mp))
1000
		return;
Linus Torvalds's avatar
Linus Torvalds committed
1001

1002
	xfs_log_force(mp, XFS_LOG_SYNC);
Linus Torvalds's avatar
Linus Torvalds committed
1003

1004
	if (xlog_is_shutdown(log))
1005
		return;
1006 1007 1008 1009 1010 1011 1012 1013 1014 1015 1016 1017 1018 1019

	/*
	 * If we think the summary counters are bad, avoid writing the unmount
	 * record to force log recovery at next mount, after which the summary
	 * counters will be recalculated.  Refer to xlog_check_unmount_rec for
	 * more details.
	 */
	if (XFS_TEST_ERROR(xfs_fs_has_sickness(mp, XFS_SICK_FS_COUNTERS), mp,
			XFS_ERRTAG_FORCE_SUMMARY_RECALC)) {
		xfs_alert(mp, "%s: will fix summary counters at next mount",
				__func__);
		return;
	}

1020
	xfs_log_unmount_verify_iclog(log);
1021
	xlog_unmount_write(log);
1022
}
Linus Torvalds's avatar
Linus Torvalds committed
1023 1024

/*
1025
 * Empty the log for unmount/freeze.
1026 1027 1028 1029
 *
 * To do this, we first need to shut down the background log work so it is not
 * trying to cover the log as we clean up. We then need to unpin all objects in
 * the log so we can then flush them out. Once they have completed their IO and
1030
 * run the callbacks removing themselves from the AIL, we can cover the log.
Linus Torvalds's avatar
Linus Torvalds committed
1031
 */
1032
int
1033 1034
xfs_log_quiesce(
	struct xfs_mount	*mp)
Linus Torvalds's avatar
Linus Torvalds committed
1035
{
1036 1037 1038 1039 1040 1041 1042 1043 1044 1045 1046 1047 1048 1049
	/*
	 * Clear log incompat features since we're quiescing the log.  Report
	 * failures, though it's not fatal to have a higher log feature
	 * protection level than the log contents actually require.
	 */
	if (xfs_clear_incompat_log_features(mp)) {
		int error;

		error = xfs_sync_sb(mp, false);
		if (error)
			xfs_warn(mp,
	"Failed to clear log incompat features on quiesce");
	}

1050
	cancel_delayed_work_sync(&mp->m_log->l_work);
1051 1052 1053 1054
	xfs_log_force(mp, XFS_LOG_SYNC);

	/*
	 * The superblock buffer is uncached and while xfs_ail_push_all_sync()
1055
	 * will push it, xfs_buftarg_wait() will not wait for it. Further,
1056 1057 1058 1059 1060
	 * xfs_buf_iowait() cannot be used because it was pushed with the
	 * XBF_ASYNC flag set, so we need to use a lock/unlock pair to wait for
	 * the IO to complete.
	 */
	xfs_ail_push_all_sync(mp->m_ail);
1061
	xfs_buftarg_wait(mp->m_ddev_targp);
1062 1063
	xfs_buf_lock(mp->m_sb_bp);
	xfs_buf_unlock(mp->m_sb_bp);
1064 1065

	return xfs_log_cover(mp);
1066
}
1067

1068 1069 1070 1071 1072
void
xfs_log_clean(
	struct xfs_mount	*mp)
{
	xfs_log_quiesce(mp);
1073
	xfs_log_unmount_write(mp);
1074 1075 1076 1077 1078 1079 1080 1081 1082 1083 1084 1085 1086
}

/*
 * Shut down and release the AIL and Log.
 *
 * During unmount, we need to ensure we flush all the dirty metadata objects
 * from the AIL so that the log is empty before we write the unmount record to
 * the log. Once this is done, we can tear down the AIL and the log.
 */
void
xfs_log_unmount(
	struct xfs_mount	*mp)
{
1087
	xfs_log_clean(mp);
1088

1089 1090
	xfs_buftarg_drain(mp->m_ddev_targp);

1091
	xfs_trans_ail_destroy(mp);
1092 1093 1094

	xfs_sysfs_del(&mp->m_log->l_kobj);

1095
	xlog_dealloc_log(mp->m_log);
Linus Torvalds's avatar
Linus Torvalds committed
1096 1097
}

1098 1099 1100 1101 1102
void
xfs_log_item_init(
	struct xfs_mount	*mp,
	struct xfs_log_item	*item,
	int			type,
1103
	const struct xfs_item_ops *ops)
1104 1105 1106 1107 1108
{
	item->li_mountp = mp;
	item->li_ailp = mp->m_ail;
	item->li_type = type;
	item->li_ops = ops;
1109 1110 1111 1112
	item->li_lv = NULL;

	INIT_LIST_HEAD(&item->li_ail);
	INIT_LIST_HEAD(&item->li_cil);
1113
	INIT_LIST_HEAD(&item->li_bio_list);
1114
	INIT_LIST_HEAD(&item->li_trans);
1115 1116
}

1117 1118 1119
/*
 * Wake up processes waiting for log space after we have moved the log tail.
 */
Linus Torvalds's avatar
Linus Torvalds committed
1120
void
1121
xfs_log_space_wake(
1122
	struct xfs_mount	*mp)
Linus Torvalds's avatar
Linus Torvalds committed
1123
{
1124
	struct xlog		*log = mp->m_log;
1125
	int			free_bytes;
Linus Torvalds's avatar
Linus Torvalds committed
1126

1127
	if (xlog_is_shutdown(log))
Linus Torvalds's avatar
Linus Torvalds committed
1128 1129
		return;

1130
	if (!list_empty_careful(&log->l_write_head.waiters)) {
1131
		ASSERT(!xlog_in_recovery(log));
1132

1133 1134
		spin_lock(&log->l_write_head.lock);
		free_bytes = xlog_space_left(log, &log->l_write_head.grant);
1135
		xlog_grant_head_wake(log, &log->l_write_head, &free_bytes);
1136
		spin_unlock(&log->l_write_head.lock);
Linus Torvalds's avatar
Linus Torvalds committed
1137
	}
1138

1139
	if (!list_empty_careful(&log->l_reserve_head.waiters)) {
1140
		ASSERT(!xlog_in_recovery(log));
1141

1142 1143
		spin_lock(&log->l_reserve_head.lock);
		free_bytes = xlog_space_left(log, &log->l_reserve_head.grant);
1144
		xlog_grant_head_wake(log, &log->l_reserve_head, &free_bytes);
1145
		spin_unlock(&log->l_reserve_head.lock);
Linus Torvalds's avatar
Linus Torvalds committed
1146
	}
1147
}
Linus Torvalds's avatar
Linus Torvalds committed
1148 1149

/*
1150 1151 1152 1153
 * Determine if we have a transaction that has gone to disk that needs to be
 * covered. To begin the transition to the idle state firstly the log needs to
 * be idle. That means the CIL, the AIL and the iclogs needs to be empty before
 * we start attempting to cover the log.
1154
 *
1155 1156 1157 1158 1159 1160 1161 1162 1163 1164
 * Only if we are then in a state where covering is needed, the caller is
 * informed that dummy transactions are required to move the log into the idle
 * state.
 *
 * If there are any items in the AIl or CIL, then we do not want to attempt to
 * cover the log as we may be in a situation where there isn't log space
 * available to run a dummy transaction and this can lead to deadlocks when the
 * tail of the log is pinned by an item that is modified in the CIL.  Hence
 * there's no point in running a dummy transaction at this point because we
 * can't start trying to idle the log until both the CIL and AIL are empty.
Linus Torvalds's avatar
Linus Torvalds committed
1165
 */
1166 1167 1168
static bool
xfs_log_need_covered(
	struct xfs_mount	*mp)
Linus Torvalds's avatar
Linus Torvalds committed
1169
{
1170 1171
	struct xlog		*log = mp->m_log;
	bool			needed = false;
Linus Torvalds's avatar
Linus Torvalds committed
1172

1173
	if (!xlog_cil_empty(log))
1174
		return false;
1175

Eric Sandeen's avatar
Eric Sandeen committed
1176
	spin_lock(&log->l_icloglock);
1177 1178 1179 1180 1181 1182 1183
	switch (log->l_covered_state) {
	case XLOG_STATE_COVER_DONE:
	case XLOG_STATE_COVER_DONE2:
	case XLOG_STATE_COVER_IDLE:
		break;
	case XLOG_STATE_COVER_NEED:
	case XLOG_STATE_COVER_NEED2:
1184 1185 1186 1187 1188
		if (xfs_ail_min_lsn(log->l_ailp))
			break;
		if (!xlog_iclogs_empty(log))
			break;

1189
		needed = true;
1190 1191 1192 1193 1194
		if (log->l_covered_state == XLOG_STATE_COVER_NEED)
			log->l_covered_state = XLOG_STATE_COVER_DONE;
		else
			log->l_covered_state = XLOG_STATE_COVER_DONE2;
		break;
1195
	default:
1196
		needed = true;
1197
		break;
Linus Torvalds's avatar
Linus Torvalds committed
1198
	}
Eric Sandeen's avatar
Eric Sandeen committed
1199
	spin_unlock(&log->l_icloglock);
1200
	return needed;
Linus Torvalds's avatar
Linus Torvalds committed
1201 1202
}

1203 1204 1205 1206 1207 1208 1209 1210 1211 1212 1213
/*
 * Explicitly cover the log. This is similar to background log covering but
 * intended for usage in quiesce codepaths. The caller is responsible to ensure
 * the log is idle and suitable for covering. The CIL, iclog buffers and AIL
 * must all be empty.
 */
static int
xfs_log_cover(
	struct xfs_mount	*mp)
{
	int			error = 0;
1214
	bool			need_covered;
1215

1216 1217
	ASSERT((xlog_cil_empty(mp->m_log) && xlog_iclogs_empty(mp->m_log) &&
	        !xfs_ail_min_lsn(mp->m_log->l_ailp)) ||
1218
		xlog_is_shutdown(mp->m_log));
1219 1220 1221 1222

	if (!xfs_log_writable(mp))
		return 0;

1223 1224 1225 1226 1227 1228 1229 1230 1231 1232 1233 1234
	/*
	 * xfs_log_need_covered() is not idempotent because it progresses the
	 * state machine if the log requires covering. Therefore, we must call
	 * this function once and use the result until we've issued an sb sync.
	 * Do so first to make that abundantly clear.
	 *
	 * Fall into the covering sequence if the log needs covering or the
	 * mount has lazy superblock accounting to sync to disk. The sb sync
	 * used for covering accumulates the in-core counters, so covering
	 * handles this for us.
	 */
	need_covered = xfs_log_need_covered(mp);
1235
	if (!need_covered && !xfs_has_lazysbcount(mp))
1236 1237
		return 0;

1238 1239 1240 1241 1242 1243 1244 1245 1246
	/*
	 * To cover the log, commit the superblock twice (at most) in
	 * independent checkpoints. The first serves as a reference for the
	 * tail pointer. The sync transaction and AIL push empties the AIL and
	 * updates the in-core tail to the LSN of the first checkpoint. The
	 * second commit updates the on-disk tail with the in-core LSN,
	 * covering the log. Push the AIL one more time to leave it empty, as
	 * we found it.
	 */
1247
	do {
1248 1249 1250 1251
		error = xfs_sync_sb(mp, true);
		if (error)
			break;
		xfs_ail_push_all_sync(mp->m_ail);
1252
	} while (xfs_log_need_covered(mp));
1253 1254 1255 1256

	return error;
}

1257
/*
Linus Torvalds's avatar
Linus Torvalds committed
1258 1259 1260
 * We may be holding the log iclog lock upon entering this routine.
 */
xfs_lsn_t
1261
xlog_assign_tail_lsn_locked(
1262
	struct xfs_mount	*mp)
Linus Torvalds's avatar
Linus Torvalds committed
1263
{
1264
	struct xlog		*log = mp->m_log;
1265 1266 1267
	struct xfs_log_item	*lip;
	xfs_lsn_t		tail_lsn;

1268
	assert_spin_locked(&mp->m_ail->ail_lock);
Linus Torvalds's avatar
Linus Torvalds committed
1269

1270 1271 1272
	/*
	 * To make sure we always have a valid LSN for the log tail we keep
	 * track of the last LSN which was committed in log->l_last_sync_lsn,
1273
	 * and use that when the AIL was empty.
1274
	 */
1275 1276 1277 1278
	lip = xfs_ail_min(mp->m_ail);
	if (lip)
		tail_lsn = lip->li_lsn;
	else
1279
		tail_lsn = atomic64_read(&log->l_last_sync_lsn);
Dave Chinner's avatar
Dave Chinner committed
1280
	trace_xfs_log_assign_tail_lsn(log, tail_lsn);
1281
	atomic64_set(&log->l_tail_lsn, tail_lsn);
Linus Torvalds's avatar
Linus Torvalds committed
1282
	return tail_lsn;
1283
}
Linus Torvalds's avatar
Linus Torvalds committed
1284

1285 1286 1287 1288 1289 1290
xfs_lsn_t
xlog_assign_tail_lsn(
	struct xfs_mount	*mp)
{
	xfs_lsn_t		tail_lsn;

1291
	spin_lock(&mp->m_ail->ail_lock);
1292
	tail_lsn = xlog_assign_tail_lsn_locked(mp);
1293
	spin_unlock(&mp->m_ail->ail_lock);
1294 1295 1296 1297

	return tail_lsn;
}

Linus Torvalds's avatar
Linus Torvalds committed
1298 1299 1300 1301 1302 1303 1304 1305 1306 1307
/*
 * Return the space in the log between the tail and the head.  The head
 * is passed in the cycle/bytes formal parms.  In the special case where
 * the reserve head has wrapped passed the tail, this calculation is no
 * longer valid.  In this case, just return 0 which means there is no space
 * in the log.  This works for all places where this function is called
 * with the reserve head.  Of course, if the write head were to ever
 * wrap the tail, we should blow up.  Rather than catch this case here,
 * we depend on other ASSERTions in other parts of the code.   XXXmiken
 *
1308 1309 1310 1311 1312 1313
 * If reservation head is behind the tail, we have a problem. Warn about it,
 * but then treat it as if the log is empty.
 *
 * If the log is shut down, the head and tail may be invalid or out of whack, so
 * shortcut invalidity asserts in this case so that we don't trigger them
 * falsely.
Linus Torvalds's avatar
Linus Torvalds committed
1314
 */
1315
STATIC int
1316
xlog_space_left(
1317
	struct xlog	*log,
1318
	atomic64_t	*head)
Linus Torvalds's avatar
Linus Torvalds committed
1319
{
1320 1321 1322 1323
	int		tail_bytes;
	int		tail_cycle;
	int		head_cycle;
	int		head_bytes;
Linus Torvalds's avatar
Linus Torvalds committed
1324

1325
	xlog_crack_grant_head(head, &head_cycle, &head_bytes);
1326 1327
	xlog_crack_atomic_lsn(&log->l_tail_lsn, &tail_cycle, &tail_bytes);
	tail_bytes = BBTOB(tail_bytes);
1328
	if (tail_cycle == head_cycle && head_bytes >= tail_bytes)
1329 1330
		return log->l_logsize - (head_bytes - tail_bytes);
	if (tail_cycle + 1 < head_cycle)
Linus Torvalds's avatar
Linus Torvalds committed
1331
		return 0;
1332 1333 1334 1335 1336 1337

	/* Ignore potential inconsistency when shutdown. */
	if (xlog_is_shutdown(log))
		return log->l_logsize;

	if (tail_cycle < head_cycle) {
1338
		ASSERT(tail_cycle == (head_cycle - 1));
1339
		return tail_bytes - head_bytes;
Linus Torvalds's avatar
Linus Torvalds committed
1340
	}
1341 1342 1343 1344 1345 1346 1347 1348 1349 1350 1351 1352

	/*
	 * The reservation head is behind the tail. In this case we just want to
	 * return the size of the log as the amount of space left.
	 */
	xfs_alert(log->l_mp, "xlog_space_left: head behind tail");
	xfs_alert(log->l_mp, "  tail_cycle = %d, tail_bytes = %d",
		  tail_cycle, tail_bytes);
	xfs_alert(log->l_mp, "  GH   cycle = %d, GH   bytes = %d",
		  head_cycle, head_bytes);
	ASSERT(0);
	return log->l_logsize;
1353
}
Linus Torvalds's avatar
Linus Torvalds committed
1354 1355


1356
static void
1357 1358
xlog_ioend_work(
	struct work_struct	*work)
Linus Torvalds's avatar
Linus Torvalds committed
1359
{
1360 1361 1362 1363
	struct xlog_in_core     *iclog =
		container_of(work, struct xlog_in_core, ic_end_io_work);
	struct xlog		*log = iclog->ic_log;
	int			error;
Linus Torvalds's avatar
Linus Torvalds committed
1364

1365
	error = blk_status_to_errno(iclog->ic_bio.bi_status);
1366 1367 1368
#ifdef DEBUG
	/* treat writes with injected CRC errors as failed */
	if (iclog->ic_fail_crc)
1369
		error = -EIO;
1370 1371
#endif

Linus Torvalds's avatar
Linus Torvalds committed
1372
	/*
1373
	 * Race to shutdown the filesystem if we see an error.
Linus Torvalds's avatar
Linus Torvalds committed
1374
	 */
1375 1376 1377
	if (XFS_TEST_ERROR(error, log->l_mp, XFS_ERRTAG_IODONE_IOERR)) {
		xfs_alert(log->l_mp, "log I/O error %d", error);
		xfs_force_shutdown(log->l_mp, SHUTDOWN_LOG_IO_ERROR);
Linus Torvalds's avatar
Linus Torvalds committed
1378
	}
1379

1380
	xlog_state_done_syncing(iclog);
1381
	bio_uninit(&iclog->ic_bio);
1382

1383
	/*
1384 1385 1386 1387
	 * Drop the lock to signal that we are done. Nothing references the
	 * iclog after this, so an unmount waiting on this lock can now tear it
	 * down safely. As such, it is unsafe to reference the iclog after the
	 * unlock as we could race with it being freed.
1388
	 */
1389
	up(&iclog->ic_sema);
1390
}
Linus Torvalds's avatar
Linus Torvalds committed
1391 1392 1393 1394

/*
 * Return size of each in-core log record buffer.
 *
Malcolm Parsons's avatar
Malcolm Parsons committed
1395
 * All machines get 8 x 32kB buffers by default, unless tuned otherwise.
Linus Torvalds's avatar
Linus Torvalds committed
1396 1397 1398 1399 1400
 *
 * If the filesystem blocksize is too large, we may need to choose a
 * larger size since the directory code currently logs entire blocks.
 */
STATIC void
Mark Tinguely's avatar
Mark Tinguely committed
1401 1402 1403
xlog_get_iclog_buffer_size(
	struct xfs_mount	*mp,
	struct xlog		*log)
Linus Torvalds's avatar
Linus Torvalds committed
1404
{
1405
	if (mp->m_logbufs <= 0)
1406 1407 1408 1409 1410 1411
		mp->m_logbufs = XLOG_MAX_ICLOGS;
	if (mp->m_logbsize <= 0)
		mp->m_logbsize = XLOG_BIG_RECORD_BSIZE;

	log->l_iclog_bufs = mp->m_logbufs;
	log->l_iclog_size = mp->m_logbsize;
Linus Torvalds's avatar
Linus Torvalds committed
1412 1413

	/*
1414
	 * # headers = size / 32k - one header holds cycles from 32k of data.
Linus Torvalds's avatar
Linus Torvalds committed
1415
	 */
1416 1417 1418 1419
	log->l_iclog_heads =
		DIV_ROUND_UP(mp->m_logbsize, XLOG_HEADER_CYCLE_SIZE);
	log->l_iclog_hsize = log->l_iclog_heads << BBSHIFT;
}
Linus Torvalds's avatar
Linus Torvalds committed
1420

1421 1422 1423 1424
void
xfs_log_work_queue(
	struct xfs_mount        *mp)
{
1425
	queue_delayed_work(mp->m_sync_workqueue, &mp->m_log->l_work,
1426 1427 1428
				msecs_to_jiffies(xfs_syncd_centisecs * 10));
}

1429 1430 1431 1432 1433 1434 1435 1436 1437 1438 1439 1440 1441 1442 1443 1444 1445 1446 1447 1448 1449 1450 1451 1452 1453 1454
/*
 * Clear the log incompat flags if we have the opportunity.
 *
 * This only happens if we're about to log the second dummy transaction as part
 * of covering the log and we can get the log incompat feature usage lock.
 */
static inline void
xlog_clear_incompat(
	struct xlog		*log)
{
	struct xfs_mount	*mp = log->l_mp;

	if (!xfs_sb_has_incompat_log_feature(&mp->m_sb,
				XFS_SB_FEAT_INCOMPAT_LOG_ALL))
		return;

	if (log->l_covered_state != XLOG_STATE_COVER_DONE2)
		return;

	if (!down_write_trylock(&log->l_incompat_users))
		return;

	xfs_clear_incompat_log_features(mp);
	up_write(&log->l_incompat_users);
}

1455 1456 1457 1458 1459
/*
 * Every sync period we need to unpin all items in the AIL and push them to
 * disk. If there is nothing dirty, then we might need to cover the log to
 * indicate that the filesystem is idle.
 */
1460
static void
1461 1462 1463 1464 1465 1466 1467 1468
xfs_log_worker(
	struct work_struct	*work)
{
	struct xlog		*log = container_of(to_delayed_work(work),
						struct xlog, l_work);
	struct xfs_mount	*mp = log->l_mp;

	/* dgc: errors ignored - not fatal and nowhere to report them */
1469
	if (xfs_fs_writable(mp, SB_FREEZE_WRITE) && xfs_log_need_covered(mp)) {
1470 1471 1472 1473 1474 1475 1476 1477 1478 1479 1480
		/*
		 * Dump a transaction into the log that contains no real change.
		 * This is needed to stamp the current tail LSN into the log
		 * during the covering operation.
		 *
		 * We cannot use an inode here for this - that will push dirty
		 * state back up into the VFS and then periodic inode flushing
		 * will prevent log covering from making progress. Hence we
		 * synchronously log the superblock instead to ensure the
		 * superblock is immediately unpinned and can be written back.
		 */
1481
		xlog_clear_incompat(log);
1482 1483
		xfs_sync_sb(mp, true);
	} else
1484 1485 1486 1487 1488 1489 1490 1491 1492
		xfs_log_force(mp, 0);

	/* start pushing all the metadata that is currently dirty */
	xfs_ail_push_all(mp->m_ail);

	/* queue us up again */
	xfs_log_work_queue(mp);
}

Linus Torvalds's avatar
Linus Torvalds committed
1493 1494 1495 1496 1497
/*
 * This routine initializes some of the log structure for a given mount point.
 * Its primary purpose is to fill in enough, so recovery can occur.  However,
 * some other stuff may be filled in too.
 */
Mark Tinguely's avatar
Mark Tinguely committed
1498 1499 1500 1501 1502 1503
STATIC struct xlog *
xlog_alloc_log(
	struct xfs_mount	*mp,
	struct xfs_buftarg	*log_target,
	xfs_daddr_t		blk_offset,
	int			num_bblks)
Linus Torvalds's avatar
Linus Torvalds committed
1504
{
Mark Tinguely's avatar
Mark Tinguely committed
1505
	struct xlog		*log;
Linus Torvalds's avatar
Linus Torvalds committed
1506 1507 1508 1509
	xlog_rec_header_t	*head;
	xlog_in_core_t		**iclogp;
	xlog_in_core_t		*iclog, *prev_iclog=NULL;
	int			i;
1510
	int			error = -ENOMEM;
1511
	uint			log2_size = 0;
Linus Torvalds's avatar
Linus Torvalds committed
1512

Mark Tinguely's avatar
Mark Tinguely committed
1513
	log = kmem_zalloc(sizeof(struct xlog), KM_MAYFAIL);
1514
	if (!log) {
1515
		xfs_warn(mp, "Log allocation failed: No memory!");
1516 1517
		goto out;
	}
Linus Torvalds's avatar
Linus Torvalds committed
1518 1519 1520 1521 1522 1523 1524

	log->l_mp	   = mp;
	log->l_targ	   = log_target;
	log->l_logsize     = BBTOB(num_bblks);
	log->l_logBBstart  = blk_offset;
	log->l_logBBsize   = num_bblks;
	log->l_covered_state = XLOG_STATE_COVER_IDLE;
1525
	set_bit(XLOG_ACTIVE_RECOVERY, &log->l_opstate);
1526
	INIT_DELAYED_WORK(&log->l_work, xfs_log_worker);
Linus Torvalds's avatar
Linus Torvalds committed
1527 1528 1529

	log->l_prev_block  = -1;
	/* log->l_tail_lsn = 0x100000000LL; cycle = 1; current block = 0 */
1530 1531
	xlog_assign_atomic_lsn(&log->l_tail_lsn, 1, 0);
	xlog_assign_atomic_lsn(&log->l_last_sync_lsn, 1, 0);
Linus Torvalds's avatar
Linus Torvalds committed
1532
	log->l_curr_cycle  = 1;	    /* 0 is bad since this is initial value */
1533

1534
	if (xfs_has_logv2(mp) && mp->m_sb.sb_logsunit > 1)
1535 1536 1537 1538
		log->l_iclog_roundoff = mp->m_sb.sb_logsunit;
	else
		log->l_iclog_roundoff = BBSIZE;

1539 1540
	xlog_grant_head_init(&log->l_reserve_head);
	xlog_grant_head_init(&log->l_write_head);
Linus Torvalds's avatar
Linus Torvalds committed
1541

1542
	error = -EFSCORRUPTED;
1543
	if (xfs_has_sector(mp)) {
1544 1545
	        log2_size = mp->m_sb.sb_logsectlog;
		if (log2_size < BBSHIFT) {
1546 1547
			xfs_warn(mp, "Log sector size too small (0x%x < 0x%x)",
				log2_size, BBSHIFT);
1548 1549 1550
			goto out_free_log;
		}

1551 1552
	        log2_size -= BBSHIFT;
		if (log2_size > mp->m_sectbb_log) {
1553 1554
			xfs_warn(mp, "Log sector size too large (0x%x > 0x%x)",
				log2_size, mp->m_sectbb_log);
1555 1556
			goto out_free_log;
		}
1557 1558 1559

		/* for larger sector sizes, must have v2 or external log */
		if (log2_size && log->l_logBBstart > 0 &&
1560
			    !xfs_has_logv2(mp)) {
1561 1562 1563
			xfs_warn(mp,
		"log sector size (0x%x) invalid for configuration.",
				log2_size);
1564 1565
			goto out_free_log;
		}
Linus Torvalds's avatar
Linus Torvalds committed
1566
	}
1567
	log->l_sectBBsize = 1 << log2_size;
Linus Torvalds's avatar
Linus Torvalds committed
1568

1569 1570
	init_rwsem(&log->l_incompat_users);

Linus Torvalds's avatar
Linus Torvalds committed
1571 1572
	xlog_get_iclog_buffer_size(mp, log);

Eric Sandeen's avatar
Eric Sandeen committed
1573
	spin_lock_init(&log->l_icloglock);
1574
	init_waitqueue_head(&log->l_flush_wait);
Linus Torvalds's avatar
Linus Torvalds committed
1575 1576 1577 1578 1579 1580 1581 1582 1583 1584

	iclogp = &log->l_iclog;
	/*
	 * The amount of memory to allocate for the iclog structure is
	 * rather funky due to the way the structure is defined.  It is
	 * done this way so that we can use different sizes for machines
	 * with different amounts of memory.  See the definition of
	 * xlog_in_core_t in xfs_log_priv.h for details.
	 */
	ASSERT(log->l_iclog_size >= 4096);
1585
	for (i = 0; i < log->l_iclog_bufs; i++) {
1586 1587
		size_t bvec_size = howmany(log->l_iclog_size, PAGE_SIZE) *
				sizeof(struct bio_vec);
1588 1589 1590

		iclog = kmem_zalloc(sizeof(*iclog) + bvec_size, KM_MAYFAIL);
		if (!iclog)
1591 1592
			goto out_free_iclog;

1593
		*iclogp = iclog;
Linus Torvalds's avatar
Linus Torvalds committed
1594 1595
		iclog->ic_prev = prev_iclog;
		prev_iclog = iclog;
1596

1597 1598
		iclog->ic_data = kvzalloc(log->l_iclog_size,
				GFP_KERNEL | __GFP_RETRY_MAYFAIL);
1599
		if (!iclog->ic_data)
1600
			goto out_free_iclog;
1601
#ifdef DEBUG
1602
		log->l_iclog_bak[i] = &iclog->ic_header;
1603
#endif
Linus Torvalds's avatar
Linus Torvalds committed
1604 1605
		head = &iclog->ic_header;
		memset(head, 0, sizeof(xlog_rec_header_t));
1606 1607
		head->h_magicno = cpu_to_be32(XLOG_HEADER_MAGIC_NUM);
		head->h_version = cpu_to_be32(
1608
			xfs_has_logv2(log->l_mp) ? 2 : 1);
1609
		head->h_size = cpu_to_be32(log->l_iclog_size);
Linus Torvalds's avatar
Linus Torvalds committed
1610
		/* new fields */
1611
		head->h_fmt = cpu_to_be32(XLOG_FMT);
Linus Torvalds's avatar
Linus Torvalds committed
1612 1613
		memcpy(&head->h_fs_uuid, &mp->m_sb.sb_uuid, sizeof(uuid_t));

1614
		iclog->ic_size = log->l_iclog_size - log->l_iclog_hsize;
Linus Torvalds's avatar
Linus Torvalds committed
1615 1616
		iclog->ic_state = XLOG_STATE_ACTIVE;
		iclog->ic_log = log;
1617
		atomic_set(&iclog->ic_refcnt, 0);
1618
		INIT_LIST_HEAD(&iclog->ic_callbacks);
1619
		iclog->ic_datap = (char *)iclog->ic_data + log->l_iclog_hsize;
Linus Torvalds's avatar
Linus Torvalds committed
1620

1621 1622
		init_waitqueue_head(&iclog->ic_force_wait);
		init_waitqueue_head(&iclog->ic_write_wait);
1623 1624
		INIT_WORK(&iclog->ic_end_io_work, xlog_ioend_work);
		sema_init(&iclog->ic_sema, 1);
Linus Torvalds's avatar
Linus Torvalds committed
1625 1626 1627 1628 1629 1630

		iclogp = &iclog->ic_next;
	}
	*iclogp = log->l_iclog;			/* complete ring */
	log->l_iclog->ic_prev = prev_iclog;	/* re-write 1st prev ptr */

1631
	log->l_ioend_workqueue = alloc_workqueue("xfs-log/%s",
1632 1633 1634
			XFS_WQFLAGS(WQ_FREEZABLE | WQ_MEM_RECLAIM |
				    WQ_HIGHPRI),
			0, mp->m_super->s_id);
1635 1636 1637
	if (!log->l_ioend_workqueue)
		goto out_free_iclog;

1638 1639
	error = xlog_cil_init(log);
	if (error)
1640
		goto out_destroy_workqueue;
Linus Torvalds's avatar
Linus Torvalds committed
1641
	return log;
1642

1643 1644
out_destroy_workqueue:
	destroy_workqueue(log->l_ioend_workqueue);
1645 1646 1647
out_free_iclog:
	for (iclog = log->l_iclog; iclog; iclog = prev_iclog) {
		prev_iclog = iclog->ic_next;
1648
		kmem_free(iclog->ic_data);
1649
		kmem_free(iclog);
1650 1651
		if (prev_iclog == log->l_iclog)
			break;
1652 1653 1654
	}
out_free_log:
	kmem_free(log);
1655
out:
1656
	return ERR_PTR(error);
Linus Torvalds's avatar
Linus Torvalds committed
1657 1658 1659
}	/* xlog_alloc_log */

/*
1660 1661 1662 1663 1664
 * Compute the LSN that we'd need to push the log tail towards in order to have
 * (a) enough on-disk log space to log the number of bytes specified, (b) at
 * least 25% of the log space free, and (c) at least 256 blocks free.  If the
 * log free space already meets all three thresholds, this function returns
 * NULLCOMMITLSN.
Linus Torvalds's avatar
Linus Torvalds committed
1665
 */
1666 1667
xfs_lsn_t
xlog_grant_push_threshold(
1668
	struct xlog	*log,
1669
	int		need_bytes)
Linus Torvalds's avatar
Linus Torvalds committed
1670
{
1671
	xfs_lsn_t	threshold_lsn = 0;
1672
	xfs_lsn_t	last_sync_lsn;
1673 1674 1675 1676 1677 1678 1679 1680
	int		free_blocks;
	int		free_bytes;
	int		threshold_block;
	int		threshold_cycle;
	int		free_threshold;

	ASSERT(BTOBB(need_bytes) < log->l_logBBsize);

1681
	free_bytes = xlog_space_left(log, &log->l_reserve_head.grant);
1682 1683 1684 1685 1686 1687 1688 1689
	free_blocks = BTOBBT(free_bytes);

	/*
	 * Set the threshold for the minimum number of free blocks in the
	 * log to the maximum of what the caller needs, one quarter of the
	 * log, and 256 blocks.
	 */
	free_threshold = BTOBB(need_bytes);
Dave Chinner's avatar
Dave Chinner committed
1690 1691
	free_threshold = max(free_threshold, (log->l_logBBsize >> 2));
	free_threshold = max(free_threshold, 256);
1692
	if (free_blocks >= free_threshold)
1693
		return NULLCOMMITLSN;
1694

1695 1696 1697
	xlog_crack_atomic_lsn(&log->l_tail_lsn, &threshold_cycle,
						&threshold_block);
	threshold_block += free_threshold;
Linus Torvalds's avatar
Linus Torvalds committed
1698
	if (threshold_block >= log->l_logBBsize) {
1699 1700
		threshold_block -= log->l_logBBsize;
		threshold_cycle += 1;
Linus Torvalds's avatar
Linus Torvalds committed
1701
	}
1702 1703 1704 1705
	threshold_lsn = xlog_assign_lsn(threshold_cycle,
					threshold_block);
	/*
	 * Don't pass in an lsn greater than the lsn of the last
1706 1707
	 * log record known to be on disk. Use a snapshot of the last sync lsn
	 * so that it doesn't change between the compare and the set.
Linus Torvalds's avatar
Linus Torvalds committed
1708
	 */
1709 1710 1711
	last_sync_lsn = atomic64_read(&log->l_last_sync_lsn);
	if (XFS_LSN_CMP(threshold_lsn, last_sync_lsn) > 0)
		threshold_lsn = last_sync_lsn;
1712

1713 1714 1715 1716 1717 1718 1719 1720 1721 1722 1723 1724 1725 1726 1727 1728 1729 1730
	return threshold_lsn;
}

/*
 * Push the tail of the log if we need to do so to maintain the free log space
 * thresholds set out by xlog_grant_push_threshold.  We may need to adopt a
 * policy which pushes on an lsn which is further along in the log once we
 * reach the high water mark.  In this manner, we would be creating a low water
 * mark.
 */
STATIC void
xlog_grant_push_ail(
	struct xlog	*log,
	int		need_bytes)
{
	xfs_lsn_t	threshold_lsn;

	threshold_lsn = xlog_grant_push_threshold(log, need_bytes);
1731
	if (threshold_lsn == NULLCOMMITLSN || xlog_is_shutdown(log))
1732 1733
		return;

1734 1735 1736 1737 1738
	/*
	 * Get the transaction layer to kick the dirty buffers out to
	 * disk asynchronously. No point in trying to do this if
	 * the filesystem is shutting down.
	 */
1739
	xfs_ail_push(log->l_ailp, threshold_lsn);
1740
}
Linus Torvalds's avatar
Linus Torvalds committed
1741

1742 1743 1744 1745 1746 1747 1748 1749 1750 1751 1752 1753
/*
 * Stamp cycle number in every block
 */
STATIC void
xlog_pack_data(
	struct xlog		*log,
	struct xlog_in_core	*iclog,
	int			roundoff)
{
	int			i, j, k;
	int			size = iclog->ic_offset + roundoff;
	__be32			cycle_lsn;
1754
	char			*dp;
1755 1756 1757 1758 1759 1760 1761 1762 1763 1764 1765 1766

	cycle_lsn = CYCLE_LSN_DISK(iclog->ic_header.h_lsn);

	dp = iclog->ic_datap;
	for (i = 0; i < BTOBB(size); i++) {
		if (i >= (XLOG_HEADER_CYCLE_SIZE / BBSIZE))
			break;
		iclog->ic_header.h_cycle_data[i] = *(__be32 *)dp;
		*(__be32 *)dp = cycle_lsn;
		dp += BBSIZE;
	}

1767
	if (xfs_has_logv2(log->l_mp)) {
1768 1769 1770 1771 1772 1773 1774 1775 1776 1777 1778 1779 1780 1781 1782 1783 1784 1785 1786 1787 1788
		xlog_in_core_2_t *xhdr = iclog->ic_data;

		for ( ; i < BTOBB(size); i++) {
			j = i / (XLOG_HEADER_CYCLE_SIZE / BBSIZE);
			k = i % (XLOG_HEADER_CYCLE_SIZE / BBSIZE);
			xhdr[j].hic_xheader.xh_cycle_data[k] = *(__be32 *)dp;
			*(__be32 *)dp = cycle_lsn;
			dp += BBSIZE;
		}

		for (i = 1; i < log->l_iclog_heads; i++)
			xhdr[i].hic_xheader.xh_cycle = cycle_lsn;
	}
}

/*
 * Calculate the checksum for a log buffer.
 *
 * This is a little more complicated than it should be because the various
 * headers and the actual data are non-contiguous.
 */
1789
__le32
1790 1791 1792 1793 1794 1795
xlog_cksum(
	struct xlog		*log,
	struct xlog_rec_header	*rhead,
	char			*dp,
	int			size)
{
1796
	uint32_t		crc;
1797 1798

	/* first generate the crc for the record header ... */
Dave Chinner's avatar
Dave Chinner committed
1799
	crc = xfs_start_cksum_update((char *)rhead,
1800 1801 1802 1803
			      sizeof(struct xlog_rec_header),
			      offsetof(struct xlog_rec_header, h_crc));

	/* ... then for additional cycle data for v2 logs ... */
1804
	if (xfs_has_logv2(log->l_mp)) {
1805 1806
		union xlog_in_core2 *xhdr = (union xlog_in_core2 *)rhead;
		int		i;
1807
		int		xheads;
1808

1809
		xheads = DIV_ROUND_UP(size, XLOG_HEADER_CYCLE_SIZE);
1810

1811
		for (i = 1; i < xheads; i++) {
1812 1813 1814 1815 1816 1817 1818 1819 1820 1821 1822
			crc = crc32c(crc, &xhdr[i].hic_xheader,
				     sizeof(struct xlog_rec_ext_header));
		}
	}

	/* ... and finally for the payload */
	crc = crc32c(crc, dp, size);

	return xfs_end_cksum(crc);
}

1823 1824 1825 1826 1827 1828
static void
xlog_bio_end_io(
	struct bio		*bio)
{
	struct xlog_in_core	*iclog = bio->bi_private;

1829
	queue_work(iclog->ic_log->l_ioend_workqueue,
1830 1831 1832
		   &iclog->ic_end_io_work);
}

1833
static int
1834 1835 1836 1837 1838 1839 1840 1841 1842 1843
xlog_map_iclog_data(
	struct bio		*bio,
	void			*data,
	size_t			count)
{
	do {
		struct page	*page = kmem_to_page(data);
		unsigned int	off = offset_in_page(data);
		size_t		len = min_t(size_t, count, PAGE_SIZE - off);

1844 1845
		if (bio_add_page(bio, page, len, off) != len)
			return -EIO;
1846 1847 1848 1849

		data += len;
		count -= len;
	} while (count);
1850 1851

	return 0;
1852 1853
}

1854 1855 1856 1857 1858
STATIC void
xlog_write_iclog(
	struct xlog		*log,
	struct xlog_in_core	*iclog,
	uint64_t		bno,
1859
	unsigned int		count)
1860
{
1861
	ASSERT(bno < log->l_logBBsize);
1862
	trace_xlog_iclog_write(iclog, _RET_IP_);
1863 1864 1865 1866 1867 1868 1869 1870 1871

	/*
	 * We lock the iclogbufs here so that we can serialise against I/O
	 * completion during unmount.  We might be processing a shutdown
	 * triggered during unmount, and that can occur asynchronously to the
	 * unmount thread, and hence we need to ensure that completes before
	 * tearing down the iclogbufs.  Hence we need to hold the buffer lock
	 * across the log IO to archieve that.
	 */
1872
	down(&iclog->ic_sema);
1873
	if (xlog_is_shutdown(log)) {
1874 1875 1876
		/*
		 * It would seem logical to return EIO here, but we rely on
		 * the log state machine to propagate I/O errors instead of
1877 1878 1879
		 * doing it here.  We kick of the state machine and unlock
		 * the buffer manually, the code needs to be kept in sync
		 * with the I/O completion path.
1880
		 */
1881
		xlog_state_done_syncing(iclog);
1882
		up(&iclog->ic_sema);
1883
		return;
1884 1885
	}

1886 1887 1888 1889 1890
	bio_init(&iclog->ic_bio, iclog->ic_bvec, howmany(count, PAGE_SIZE));
	bio_set_dev(&iclog->ic_bio, log->l_targ->bt_bdev);
	iclog->ic_bio.bi_iter.bi_sector = log->l_logBBstart + bno;
	iclog->ic_bio.bi_end_io = xlog_bio_end_io;
	iclog->ic_bio.bi_private = iclog;
1891 1892 1893 1894 1895 1896 1897

	/*
	 * We use REQ_SYNC | REQ_IDLE here to tell the block layer the are more
	 * IOs coming immediately after this one. This prevents the block layer
	 * writeback throttle from throttling log writes behind background
	 * metadata writeback and causing priority inversions.
	 */
1898
	iclog->ic_bio.bi_opf = REQ_OP_WRITE | REQ_META | REQ_SYNC | REQ_IDLE;
1899
	if (iclog->ic_flags & XLOG_ICL_NEED_FLUSH) {
1900
		iclog->ic_bio.bi_opf |= REQ_PREFLUSH;
1901 1902 1903 1904 1905 1906 1907 1908 1909
		/*
		 * For external log devices, we also need to flush the data
		 * device cache first to ensure all metadata writeback covered
		 * by the LSN in this iclog is on stable storage. This is slow,
		 * but it *must* complete before we issue the external log IO.
		 */
		if (log->l_targ != log->l_mp->m_ddev_targp)
			blkdev_issue_flush(log->l_mp->m_ddev_targp->bt_bdev);
	}
1910 1911
	if (iclog->ic_flags & XLOG_ICL_NEED_FUA)
		iclog->ic_bio.bi_opf |= REQ_FUA;
1912

1913
	iclog->ic_flags &= ~(XLOG_ICL_NEED_FLUSH | XLOG_ICL_NEED_FUA);
1914

1915 1916 1917 1918
	if (xlog_map_iclog_data(&iclog->ic_bio, iclog->ic_data, count)) {
		xfs_force_shutdown(log->l_mp, SHUTDOWN_LOG_IO_ERROR);
		return;
	}
1919
	if (is_vmalloc_addr(iclog->ic_data))
1920
		flush_kernel_vmap_range(iclog->ic_data, count);
1921 1922 1923 1924 1925 1926 1927 1928 1929 1930 1931 1932 1933 1934 1935 1936 1937 1938

	/*
	 * If this log buffer would straddle the end of the log we will have
	 * to split it up into two bios, so that we can continue at the start.
	 */
	if (bno + BTOBB(count) > log->l_logBBsize) {
		struct bio *split;

		split = bio_split(&iclog->ic_bio, log->l_logBBsize - bno,
				  GFP_NOIO, &fs_bio_set);
		bio_chain(split, &iclog->ic_bio);
		submit_bio(split);

		/* restart at logical offset zero for the remainder */
		iclog->ic_bio.bi_iter.bi_sector = log->l_logBBstart;
	}

	submit_bio(&iclog->ic_bio);
1939
}
Linus Torvalds's avatar
Linus Torvalds committed
1940

1941 1942 1943 1944 1945
/*
 * We need to bump cycle number for the part of the iclog that is
 * written to the start of the log. Watch out for the header magic
 * number case, though.
 */
1946
static void
1947 1948 1949 1950 1951 1952 1953 1954 1955 1956 1957 1958 1959 1960 1961 1962 1963 1964
xlog_split_iclog(
	struct xlog		*log,
	void			*data,
	uint64_t		bno,
	unsigned int		count)
{
	unsigned int		split_offset = BBTOB(log->l_logBBsize - bno);
	unsigned int		i;

	for (i = split_offset; i < count; i += BBSIZE) {
		uint32_t cycle = get_unaligned_be32(data + i);

		if (++cycle == XLOG_HEADER_MAGIC_NUM)
			cycle++;
		put_unaligned_be32(cycle, data + i);
	}
}

1965 1966 1967 1968 1969 1970 1971 1972 1973 1974
static int
xlog_calc_iclog_size(
	struct xlog		*log,
	struct xlog_in_core	*iclog,
	uint32_t		*roundoff)
{
	uint32_t		count_init, count;

	/* Add for LR header */
	count_init = log->l_iclog_hsize + iclog->ic_offset;
1975
	count = roundup(count_init, log->l_iclog_roundoff);
1976 1977 1978

	*roundoff = count - count_init;

1979 1980
	ASSERT(count >= count_init);
	ASSERT(*roundoff < log->l_iclog_roundoff);
1981 1982 1983
	return count;
}

Linus Torvalds's avatar
Linus Torvalds committed
1984 1985 1986 1987 1988 1989 1990 1991 1992 1993 1994 1995 1996 1997 1998 1999 2000 2001 2002 2003
/*
 * Flush out the in-core log (iclog) to the on-disk log in an asynchronous 
 * fashion.  Previously, we should have moved the current iclog
 * ptr in the log to point to the next available iclog.  This allows further
 * write to continue while this code syncs out an iclog ready to go.
 * Before an in-core log can be written out, the data section must be scanned
 * to save away the 1st word of each BBSIZE block into the header.  We replace
 * it with the current cycle count.  Each BBSIZE block is tagged with the
 * cycle count because there in an implicit assumption that drives will
 * guarantee that entire 512 byte blocks get written at once.  In other words,
 * we can't have part of a 512 byte block written and part not written.  By
 * tagging each block, we will know which blocks are valid when recovering
 * after an unclean shutdown.
 *
 * This routine is single threaded on the iclog.  No other thread can be in
 * this routine with the same iclog.  Changing contents of iclog can there-
 * fore be done without grabbing the state machine lock.  Updating the global
 * log will require grabbing the lock though.
 *
 * The entire log manager uses a logical block numbering scheme.  Only
2004 2005
 * xlog_write_iclog knows about the fact that the log may not start with
 * block zero on a given device.
Linus Torvalds's avatar
Linus Torvalds committed
2006
 */
2007
STATIC void
Mark Tinguely's avatar
Mark Tinguely committed
2008 2009 2010
xlog_sync(
	struct xlog		*log,
	struct xlog_in_core	*iclog)
Linus Torvalds's avatar
Linus Torvalds committed
2011
{
2012 2013 2014 2015
	unsigned int		count;		/* byte count of bwrite */
	unsigned int		roundoff;       /* roundoff to BB or stripe */
	uint64_t		bno;
	unsigned int		size;
Linus Torvalds's avatar
Linus Torvalds committed
2016

2017
	ASSERT(atomic_read(&iclog->ic_refcnt) == 0);
2018
	trace_xlog_iclog_sync(iclog, _RET_IP_);
Linus Torvalds's avatar
Linus Torvalds committed
2019

2020
	count = xlog_calc_iclog_size(log, iclog, &roundoff);
Linus Torvalds's avatar
Linus Torvalds committed
2021 2022

	/* move grant heads by roundoff in sync */
2023 2024
	xlog_grant_add_space(log, &log->l_reserve_head.grant, roundoff);
	xlog_grant_add_space(log, &log->l_write_head.grant, roundoff);
Linus Torvalds's avatar
Linus Torvalds committed
2025 2026 2027 2028 2029

	/* put cycle number in every block */
	xlog_pack_data(log, iclog, roundoff); 

	/* real byte length */
2030
	size = iclog->ic_offset;
2031
	if (xfs_has_logv2(log->l_mp))
2032 2033
		size += roundoff;
	iclog->ic_header.h_len = cpu_to_be32(size);
Linus Torvalds's avatar
Linus Torvalds committed
2034

2035
	XFS_STATS_INC(log->l_mp, xs_log_writes);
2036
	XFS_STATS_ADD(log->l_mp, xs_log_blocks, BTOBB(count));
Linus Torvalds's avatar
Linus Torvalds committed
2037

2038 2039
	bno = BLOCK_LSN(be64_to_cpu(iclog->ic_header.h_lsn));

Linus Torvalds's avatar
Linus Torvalds committed
2040
	/* Do we need to split this write into 2 parts? */
2041
	if (bno + BTOBB(count) > log->l_logBBsize)
2042
		xlog_split_iclog(log, &iclog->ic_header, bno, count);
2043 2044 2045 2046

	/* calculcate the checksum */
	iclog->ic_header.h_crc = xlog_cksum(log, &iclog->ic_header,
					    iclog->ic_datap, size);
2047 2048 2049 2050 2051 2052 2053
	/*
	 * Intentionally corrupt the log record CRC based on the error injection
	 * frequency, if defined. This facilitates testing log recovery in the
	 * event of torn writes. Hence, set the IOABORT state to abort the log
	 * write on I/O completion and shutdown the fs. The subsequent mount
	 * detects the bad CRC and attempts to recover.
	 */
2054
#ifdef DEBUG
2055
	if (XFS_TEST_ERROR(false, log->l_mp, XFS_ERRTAG_LOG_BAD_CRC)) {
2056
		iclog->ic_header.h_crc &= cpu_to_le32(0xAAAAAAAA);
2057
		iclog->ic_fail_crc = true;
2058 2059 2060 2061
		xfs_warn(log->l_mp,
	"Intentionally corrupted log record at LSN 0x%llx. Shutdown imminent.",
			 be64_to_cpu(iclog->ic_header.h_lsn));
	}
2062
#endif
2063
	xlog_verify_iclog(log, iclog, count);
2064
	xlog_write_iclog(log, iclog, bno, count);
2065
}
Linus Torvalds's avatar
Linus Torvalds committed
2066 2067

/*
2068
 * Deallocate a log structure
Linus Torvalds's avatar
Linus Torvalds committed
2069
 */
2070
STATIC void
Mark Tinguely's avatar
Mark Tinguely committed
2071 2072
xlog_dealloc_log(
	struct xlog	*log)
Linus Torvalds's avatar
Linus Torvalds committed
2073 2074 2075 2076
{
	xlog_in_core_t	*iclog, *next_iclog;
	int		i;

2077 2078
	xlog_cil_destroy(log);

2079
	/*
2080 2081 2082 2083 2084
	 * Cycle all the iclogbuf locks to make sure all log IO completion
	 * is done before we tear down these buffers.
	 */
	iclog = log->l_iclog;
	for (i = 0; i < log->l_iclog_bufs; i++) {
2085 2086
		down(&iclog->ic_sema);
		up(&iclog->ic_sema);
2087 2088 2089
		iclog = iclog->ic_next;
	}

Linus Torvalds's avatar
Linus Torvalds committed
2090
	iclog = log->l_iclog;
2091
	for (i = 0; i < log->l_iclog_bufs; i++) {
Linus Torvalds's avatar
Linus Torvalds committed
2092
		next_iclog = iclog->ic_next;
2093
		kmem_free(iclog->ic_data);
2094
		kmem_free(iclog);
Linus Torvalds's avatar
Linus Torvalds committed
2095 2096 2097 2098
		iclog = next_iclog;
	}

	log->l_mp->m_log = NULL;
2099
	destroy_workqueue(log->l_ioend_workqueue);
2100
	kmem_free(log);
2101
}
Linus Torvalds's avatar
Linus Torvalds committed
2102 2103 2104 2105 2106

/*
 * Update counters atomically now that memcpy is done.
 */
static inline void
Mark Tinguely's avatar
Mark Tinguely committed
2107 2108 2109 2110 2111
xlog_state_finish_copy(
	struct xlog		*log,
	struct xlog_in_core	*iclog,
	int			record_cnt,
	int			copy_bytes)
Linus Torvalds's avatar
Linus Torvalds committed
2112
{
2113
	lockdep_assert_held(&log->l_icloglock);
Linus Torvalds's avatar
Linus Torvalds committed
2114

2115
	be32_add_cpu(&iclog->ic_header.h_num_logops, record_cnt);
Linus Torvalds's avatar
Linus Torvalds committed
2116
	iclog->ic_offset += copy_bytes;
2117
}
Linus Torvalds's avatar
Linus Torvalds committed
2118

2119 2120 2121 2122
/*
 * print out info relating to regions written which consume
 * the reservation
 */
2123 2124 2125 2126
void
xlog_print_tic_res(
	struct xfs_mount	*mp,
	struct xlog_ticket	*ticket)
2127 2128 2129 2130 2131
{
	uint i;
	uint ophdr_spc = ticket->t_res_num_ophdrs * (uint)sizeof(xlog_op_header_t);

	/* match with XLOG_REG_TYPE_* in xfs_log.h */
2132
#define REG_TYPE_STR(type, str)	[XLOG_REG_TYPE_##type] = str
2133
	static char *res_type_str[] = {
2134 2135 2136 2137 2138 2139 2140 2141 2142 2143 2144 2145 2146 2147 2148 2149 2150 2151 2152
	    REG_TYPE_STR(BFORMAT, "bformat"),
	    REG_TYPE_STR(BCHUNK, "bchunk"),
	    REG_TYPE_STR(EFI_FORMAT, "efi_format"),
	    REG_TYPE_STR(EFD_FORMAT, "efd_format"),
	    REG_TYPE_STR(IFORMAT, "iformat"),
	    REG_TYPE_STR(ICORE, "icore"),
	    REG_TYPE_STR(IEXT, "iext"),
	    REG_TYPE_STR(IBROOT, "ibroot"),
	    REG_TYPE_STR(ILOCAL, "ilocal"),
	    REG_TYPE_STR(IATTR_EXT, "iattr_ext"),
	    REG_TYPE_STR(IATTR_BROOT, "iattr_broot"),
	    REG_TYPE_STR(IATTR_LOCAL, "iattr_local"),
	    REG_TYPE_STR(QFORMAT, "qformat"),
	    REG_TYPE_STR(DQUOT, "dquot"),
	    REG_TYPE_STR(QUOTAOFF, "quotaoff"),
	    REG_TYPE_STR(LRHEADER, "LR header"),
	    REG_TYPE_STR(UNMOUNT, "unmount"),
	    REG_TYPE_STR(COMMIT, "commit"),
	    REG_TYPE_STR(TRANSHDR, "trans header"),
2153 2154 2155 2156 2157 2158 2159
	    REG_TYPE_STR(ICREATE, "inode create"),
	    REG_TYPE_STR(RUI_FORMAT, "rui_format"),
	    REG_TYPE_STR(RUD_FORMAT, "rud_format"),
	    REG_TYPE_STR(CUI_FORMAT, "cui_format"),
	    REG_TYPE_STR(CUD_FORMAT, "cud_format"),
	    REG_TYPE_STR(BUI_FORMAT, "bui_format"),
	    REG_TYPE_STR(BUD_FORMAT, "bud_format"),
2160
	};
2161
	BUILD_BUG_ON(ARRAY_SIZE(res_type_str) != XLOG_REG_TYPE_MAX + 1);
2162
#undef REG_TYPE_STR
2163

2164
	xfs_warn(mp, "ticket reservation summary:");
2165 2166 2167 2168 2169 2170 2171 2172 2173 2174 2175 2176
	xfs_warn(mp, "  unit res    = %d bytes",
		 ticket->t_unit_res);
	xfs_warn(mp, "  current res = %d bytes",
		 ticket->t_curr_res);
	xfs_warn(mp, "  total reg   = %u bytes (o/flow = %u bytes)",
		 ticket->t_res_arr_sum, ticket->t_res_o_flow);
	xfs_warn(mp, "  ophdrs      = %u (ophdr space = %u bytes)",
		 ticket->t_res_num_ophdrs, ophdr_spc);
	xfs_warn(mp, "  ophdr + reg = %u bytes",
		 ticket->t_res_arr_sum + ticket->t_res_o_flow + ophdr_spc);
	xfs_warn(mp, "  num regions = %u",
		 ticket->t_res_num);
2177 2178

	for (i = 0; i < ticket->t_res_num; i++) {
2179
		uint r_type = ticket->t_res_arr[i].r_type;
2180
		xfs_warn(mp, "region[%u]: %s - %u bytes", i,
2181
			    ((r_type <= 0 || r_type > XLOG_REG_TYPE_MAX) ?
2182
			    "bad-rtype" : res_type_str[r_type]),
2183 2184 2185 2186
			    ticket->t_res_arr[i].r_len);
	}
}

2187 2188 2189 2190 2191
/*
 * Print a summary of the transaction.
 */
void
xlog_print_trans(
2192
	struct xfs_trans	*tp)
2193
{
2194 2195
	struct xfs_mount	*mp = tp->t_mountp;
	struct xfs_log_item	*lip;
2196 2197 2198

	/* dump core transaction and ticket info */
	xfs_warn(mp, "transaction summary:");
2199 2200 2201
	xfs_warn(mp, "  log res   = %d", tp->t_log_res);
	xfs_warn(mp, "  log count = %d", tp->t_log_count);
	xfs_warn(mp, "  flags     = 0x%x", tp->t_flags);
2202 2203 2204 2205

	xlog_print_tic_res(mp, tp->t_ticket);

	/* dump each log item */
2206
	list_for_each_entry(lip, &tp->t_items, li_trans) {
2207 2208 2209 2210 2211 2212
		struct xfs_log_vec	*lv = lip->li_lv;
		struct xfs_log_iovec	*vec;
		int			i;

		xfs_warn(mp, "log item: ");
		xfs_warn(mp, "  type	= 0x%x", lip->li_type);
2213
		xfs_warn(mp, "  flags	= 0x%lx", lip->li_flags);
2214 2215 2216 2217 2218 2219 2220 2221 2222 2223 2224 2225 2226 2227 2228 2229
		if (!lv)
			continue;
		xfs_warn(mp, "  niovecs	= %d", lv->lv_niovecs);
		xfs_warn(mp, "  size	= %d", lv->lv_size);
		xfs_warn(mp, "  bytes	= %d", lv->lv_bytes);
		xfs_warn(mp, "  buf len	= %d", lv->lv_buf_len);

		/* dump each iovec for the log item */
		vec = lv->lv_iovecp;
		for (i = 0; i < lv->lv_niovecs; i++) {
			int dumplen = min(vec->i_len, 32);

			xfs_warn(mp, "  iovec[%d]", i);
			xfs_warn(mp, "    type	= 0x%x", vec->i_type);
			xfs_warn(mp, "    len	= %d", vec->i_len);
			xfs_warn(mp, "    first %d bytes of iovec[%d]:", dumplen, i);
2230
			xfs_hex_dump(vec->i_addr, dumplen);
2231 2232 2233 2234 2235 2236

			vec++;
		}
	}
}

Dave Chinner's avatar
Dave Chinner committed
2237
/*
2238 2239 2240
 * Calculate the potential space needed by the log vector.  We may need a start
 * record, and each region gets its own struct xlog_op_header and may need to be
 * double word aligned.
Dave Chinner's avatar
Dave Chinner committed
2241 2242 2243 2244
 */
static int
xlog_write_calc_vec_length(
	struct xlog_ticket	*ticket,
2245
	struct xfs_log_vec	*log_vector,
2246
	uint			optype)
Dave Chinner's avatar
Dave Chinner committed
2247
{
2248
	struct xfs_log_vec	*lv;
2249
	int			headers = 0;
Dave Chinner's avatar
Dave Chinner committed
2250 2251 2252
	int			len = 0;
	int			i;

2253 2254 2255
	if (optype & XLOG_START_TRANS)
		headers++;

2256
	for (lv = log_vector; lv; lv = lv->lv_next) {
2257 2258 2259 2260
		/* we don't write ordered log vectors */
		if (lv->lv_buf_len == XFS_LOG_VEC_ORDERED)
			continue;

2261 2262 2263 2264
		headers += lv->lv_niovecs;

		for (i = 0; i < lv->lv_niovecs; i++) {
			struct xfs_log_iovec	*vecp = &lv->lv_iovecp[i];
Dave Chinner's avatar
Dave Chinner committed
2265

2266 2267 2268
			len += vecp->i_len;
			xlog_tic_add_region(ticket, vecp->i_len, vecp->i_type);
		}
Dave Chinner's avatar
Dave Chinner committed
2269 2270 2271 2272 2273 2274 2275 2276
	}

	ticket->t_res_num_ophdrs += headers;
	len += headers * sizeof(struct xlog_op_header);

	return len;
}

2277
static void
Dave Chinner's avatar
Dave Chinner committed
2278
xlog_write_start_rec(
2279
	struct xlog_op_header	*ophdr,
Dave Chinner's avatar
Dave Chinner committed
2280 2281 2282 2283 2284 2285 2286 2287 2288 2289 2290
	struct xlog_ticket	*ticket)
{
	ophdr->oh_tid	= cpu_to_be32(ticket->t_tid);
	ophdr->oh_clientid = ticket->t_clientid;
	ophdr->oh_len = 0;
	ophdr->oh_flags = XLOG_START_TRANS;
	ophdr->oh_res2 = 0;
}

static xlog_op_header_t *
xlog_write_setup_ophdr(
2291
	struct xlog		*log,
2292
	struct xlog_op_header	*ophdr,
Dave Chinner's avatar
Dave Chinner committed
2293 2294 2295 2296 2297 2298 2299 2300 2301 2302 2303 2304 2305 2306 2307 2308 2309 2310 2311 2312 2313
	struct xlog_ticket	*ticket,
	uint			flags)
{
	ophdr->oh_tid = cpu_to_be32(ticket->t_tid);
	ophdr->oh_clientid = ticket->t_clientid;
	ophdr->oh_res2 = 0;

	/* are we copying a commit or unmount record? */
	ophdr->oh_flags = flags;

	/*
	 * We've seen logs corrupted with bad transaction client ids.  This
	 * makes sure that XFS doesn't generate them on.  Turn this into an EIO
	 * and shut down the filesystem.
	 */
	switch (ophdr->oh_clientid)  {
	case XFS_TRANSACTION:
	case XFS_VOLUME:
	case XFS_LOG:
		break;
	default:
2314
		xfs_warn(log->l_mp,
2315
			"Bad XFS transaction clientid 0x%x in ticket "PTR_FMT,
Dave Chinner's avatar
Dave Chinner committed
2316 2317 2318 2319 2320 2321 2322 2323 2324 2325 2326
			ophdr->oh_clientid, ticket);
		return NULL;
	}

	return ophdr;
}

/*
 * Set up the parameters of the region copy into the log. This has
 * to handle region write split across multiple log buffers - this
 * state is kept external to this function so that this code can
2327
 * be written in an obvious, self documenting manner.
Dave Chinner's avatar
Dave Chinner committed
2328 2329 2330 2331 2332 2333 2334 2335 2336 2337 2338 2339 2340 2341 2342 2343 2344 2345 2346 2347 2348 2349 2350 2351 2352 2353 2354 2355 2356 2357 2358 2359 2360 2361 2362 2363 2364 2365 2366 2367 2368 2369 2370 2371 2372 2373
 */
static int
xlog_write_setup_copy(
	struct xlog_ticket	*ticket,
	struct xlog_op_header	*ophdr,
	int			space_available,
	int			space_required,
	int			*copy_off,
	int			*copy_len,
	int			*last_was_partial_copy,
	int			*bytes_consumed)
{
	int			still_to_copy;

	still_to_copy = space_required - *bytes_consumed;
	*copy_off = *bytes_consumed;

	if (still_to_copy <= space_available) {
		/* write of region completes here */
		*copy_len = still_to_copy;
		ophdr->oh_len = cpu_to_be32(*copy_len);
		if (*last_was_partial_copy)
			ophdr->oh_flags |= (XLOG_END_TRANS|XLOG_WAS_CONT_TRANS);
		*last_was_partial_copy = 0;
		*bytes_consumed = 0;
		return 0;
	}

	/* partial write of region, needs extra log op header reservation */
	*copy_len = space_available;
	ophdr->oh_len = cpu_to_be32(*copy_len);
	ophdr->oh_flags |= XLOG_CONTINUE_TRANS;
	if (*last_was_partial_copy)
		ophdr->oh_flags |= XLOG_WAS_CONT_TRANS;
	*bytes_consumed += *copy_len;
	(*last_was_partial_copy)++;

	/* account for new log op header */
	ticket->t_curr_res -= sizeof(struct xlog_op_header);
	ticket->t_res_num_ophdrs++;

	return sizeof(struct xlog_op_header);
}

static int
xlog_write_copy_finish(
2374
	struct xlog		*log,
Dave Chinner's avatar
Dave Chinner committed
2375 2376 2377 2378 2379 2380
	struct xlog_in_core	*iclog,
	uint			flags,
	int			*record_cnt,
	int			*data_cnt,
	int			*partial_copy,
	int			*partial_copy_len,
2381
	int			log_offset)
Dave Chinner's avatar
Dave Chinner committed
2382
{
2383 2384
	int			error;

Dave Chinner's avatar
Dave Chinner committed
2385 2386 2387 2388 2389
	if (*partial_copy) {
		/*
		 * This iclog has already been marked WANT_SYNC by
		 * xlog_state_get_iclog_space.
		 */
2390
		spin_lock(&log->l_icloglock);
Dave Chinner's avatar
Dave Chinner committed
2391 2392 2393
		xlog_state_finish_copy(log, iclog, *record_cnt, *data_cnt);
		*record_cnt = 0;
		*data_cnt = 0;
2394
		goto release_iclog;
Dave Chinner's avatar
Dave Chinner committed
2395 2396 2397 2398 2399
	}

	*partial_copy = 0;
	*partial_copy_len = 0;

2400 2401
	if (iclog->ic_size - log_offset > sizeof(xlog_op_header_t))
		return 0;
Dave Chinner's avatar
Dave Chinner committed
2402

2403 2404 2405 2406 2407
	/* no more space in this iclog - push it. */
	spin_lock(&log->l_icloglock);
	xlog_state_finish_copy(log, iclog, *record_cnt, *data_cnt);
	*record_cnt = 0;
	*data_cnt = 0;
2408

2409 2410 2411 2412 2413
	if (iclog->ic_state == XLOG_STATE_ACTIVE)
		xlog_state_switch_iclogs(log, iclog, 0);
	else
		ASSERT(iclog->ic_state == XLOG_STATE_WANT_SYNC ||
			xlog_is_shutdown(log));
2414
release_iclog:
2415
	error = xlog_state_release_iclog(log, iclog, 0);
2416 2417
	spin_unlock(&log->l_icloglock);
	return error;
Dave Chinner's avatar
Dave Chinner committed
2418 2419
}

Linus Torvalds's avatar
Linus Torvalds committed
2420 2421 2422 2423 2424 2425 2426 2427 2428 2429 2430 2431 2432 2433 2434 2435 2436 2437 2438 2439 2440 2441 2442 2443 2444 2445 2446 2447 2448 2449 2450 2451 2452 2453 2454 2455 2456 2457 2458 2459
/*
 * Write some region out to in-core log
 *
 * This will be called when writing externally provided regions or when
 * writing out a commit record for a given transaction.
 *
 * General algorithm:
 *	1. Find total length of this write.  This may include adding to the
 *		lengths passed in.
 *	2. Check whether we violate the tickets reservation.
 *	3. While writing to this iclog
 *	    A. Reserve as much space in this iclog as can get
 *	    B. If this is first write, save away start lsn
 *	    C. While writing this region:
 *		1. If first write of transaction, write start record
 *		2. Write log operation header (header per region)
 *		3. Find out if we can fit entire region into this iclog
 *		4. Potentially, verify destination memcpy ptr
 *		5. Memcpy (partial) region
 *		6. If partial copy, release iclog; otherwise, continue
 *			copying more regions into current iclog
 *	4. Mark want sync bit (in simulation mode)
 *	5. Release iclog for potential flush to on-disk log.
 *
 * ERRORS:
 * 1.	Panic if reservation is overrun.  This should never happen since
 *	reservation amounts are generated internal to the filesystem.
 * NOTES:
 * 1. Tickets are single threaded data structures.
 * 2. The XLOG_END_TRANS & XLOG_CONTINUE_TRANS flags are passed down to the
 *	syncing routine.  When a single log_write region needs to span
 *	multiple in-core logs, the XLOG_CONTINUE_TRANS bit should be set
 *	on all log operation writes which don't contain the end of the
 *	region.  The XLOG_END_TRANS bit is used for the in-core log
 *	operation which contains the end of the continued log_write region.
 * 3. When xlog_state_get_iclog_space() grabs the rest of the current iclog,
 *	we don't really know exactly how much space will be used.  As a result,
 *	we don't update ic_offset until the end when we know exactly how many
 *	bytes have been written out.
 */
2460
int
2461
xlog_write(
2462
	struct xlog		*log,
2463
	struct xfs_cil_ctx	*ctx,
2464
	struct xfs_log_vec	*log_vector,
2465
	struct xlog_ticket	*ticket,
2466
	uint			optype)
Linus Torvalds's avatar
Linus Torvalds committed
2467
{
Christoph Hellwig's avatar
Christoph Hellwig committed
2468
	struct xlog_in_core	*iclog = NULL;
2469 2470 2471
	struct xfs_log_vec	*lv = log_vector;
	struct xfs_log_iovec	*vecp = lv->lv_iovecp;
	int			index = 0;
Christoph Hellwig's avatar
Christoph Hellwig committed
2472 2473 2474 2475 2476 2477
	int			len;
	int			partial_copy = 0;
	int			partial_copy_len = 0;
	int			contwr = 0;
	int			record_cnt = 0;
	int			data_cnt = 0;
2478
	int			error = 0;
Christoph Hellwig's avatar
Christoph Hellwig committed
2479

2480
	/*
2481 2482 2483 2484
	 * If this is a commit or unmount transaction, we don't need a start
	 * record to be written.  We do, however, have to account for the
	 * commit or unmount header that gets written. Hence we always have
	 * to account for an extra xlog_op_header here.
2485
	 */
2486
	ticket->t_curr_res -= sizeof(struct xlog_op_header);
2487 2488 2489
	if (ticket->t_curr_res < 0) {
		xfs_alert_tag(log->l_mp, XFS_PTAG_LOGRES,
		     "ctx ticket reservation ran out. Need to up reservation");
2490
		xlog_print_tic_res(log->l_mp, ticket);
2491 2492
		xfs_force_shutdown(log->l_mp, SHUTDOWN_LOG_IO_ERROR);
	}
Linus Torvalds's avatar
Linus Torvalds committed
2493

2494
	len = xlog_write_calc_vec_length(ticket, log_vector, optype);
2495
	while (lv && (!lv->lv_niovecs || index < lv->lv_niovecs)) {
2496
		void		*ptr;
Christoph Hellwig's avatar
Christoph Hellwig committed
2497
		int		log_offset;
Linus Torvalds's avatar
Linus Torvalds committed
2498

Christoph Hellwig's avatar
Christoph Hellwig committed
2499 2500 2501 2502
		error = xlog_state_get_iclog_space(log, len, &iclog, ticket,
						   &contwr, &log_offset);
		if (error)
			return error;
Linus Torvalds's avatar
Linus Torvalds committed
2503

Christoph Hellwig's avatar
Christoph Hellwig committed
2504
		ASSERT(log_offset <= iclog->ic_size - 1);
2505
		ptr = iclog->ic_datap + log_offset;
Linus Torvalds's avatar
Linus Torvalds committed
2506

2507 2508 2509 2510 2511 2512 2513 2514 2515
		/*
		 * If we have a context pointer, pass it the first iclog we are
		 * writing to so it can record state needed for iclog write
		 * ordering.
		 */
		if (ctx) {
			xlog_cil_set_ctx_write_state(ctx, iclog);
			ctx = NULL;
		}
Dave Chinner's avatar
Dave Chinner committed
2516

Christoph Hellwig's avatar
Christoph Hellwig committed
2517 2518 2519 2520
		/*
		 * This loop writes out as many regions as can fit in the amount
		 * of space which was allocated by xlog_state_get_iclog_space().
		 */
2521 2522
		while (lv && (!lv->lv_niovecs || index < lv->lv_niovecs)) {
			struct xfs_log_iovec	*reg;
Christoph Hellwig's avatar
Christoph Hellwig committed
2523 2524 2525
			struct xlog_op_header	*ophdr;
			int			copy_len;
			int			copy_off;
2526
			bool			ordered = false;
2527
			bool			wrote_start_rec = false;
2528 2529 2530 2531 2532 2533 2534

			/* ordered log vectors have no regions to write */
			if (lv->lv_buf_len == XFS_LOG_VEC_ORDERED) {
				ASSERT(lv->lv_niovecs == 0);
				ordered = true;
				goto next_lv;
			}
Christoph Hellwig's avatar
Christoph Hellwig committed
2535

2536
			reg = &vecp[index];
2537 2538
			ASSERT(reg->i_len % sizeof(int32_t) == 0);
			ASSERT((unsigned long)ptr % sizeof(int32_t) == 0);
Christoph Hellwig's avatar
Christoph Hellwig committed
2539

2540 2541 2542 2543 2544
			/*
			 * Before we start formatting log vectors, we need to
			 * write a start record. Only do this for the first
			 * iclog we write to.
			 */
2545
			if (optype & XLOG_START_TRANS) {
2546
				xlog_write_start_rec(ptr, ticket);
2547
				xlog_write_adv_cnt(&ptr, &len, &log_offset,
2548
						sizeof(struct xlog_op_header));
2549 2550
				optype &= ~XLOG_START_TRANS;
				wrote_start_rec = true;
Christoph Hellwig's avatar
Christoph Hellwig committed
2551
			}
Dave Chinner's avatar
Dave Chinner committed
2552

2553
			ophdr = xlog_write_setup_ophdr(log, ptr, ticket, optype);
Christoph Hellwig's avatar
Christoph Hellwig committed
2554
			if (!ophdr)
2555
				return -EIO;
Christoph Hellwig's avatar
Christoph Hellwig committed
2556

2557
			xlog_write_adv_cnt(&ptr, &len, &log_offset,
Christoph Hellwig's avatar
Christoph Hellwig committed
2558 2559 2560 2561
					   sizeof(struct xlog_op_header));

			len += xlog_write_setup_copy(ticket, ophdr,
						     iclog->ic_size-log_offset,
2562
						     reg->i_len,
Christoph Hellwig's avatar
Christoph Hellwig committed
2563 2564 2565 2566 2567
						     &copy_off, &copy_len,
						     &partial_copy,
						     &partial_copy_len);
			xlog_verify_dest_ptr(log, ptr);

2568 2569 2570 2571 2572 2573 2574 2575
			/*
			 * Copy region.
			 *
			 * Unmount records just log an opheader, so can have
			 * empty payloads with no data region to copy. Hence we
			 * only copy the payload if the vector says it has data
			 * to copy.
			 */
Christoph Hellwig's avatar
Christoph Hellwig committed
2576
			ASSERT(copy_len >= 0);
2577 2578 2579 2580 2581
			if (copy_len > 0) {
				memcpy(ptr, reg->i_addr + copy_off, copy_len);
				xlog_write_adv_cnt(&ptr, &len, &log_offset,
						   copy_len);
			}
2582
			copy_len += sizeof(struct xlog_op_header);
Christoph Hellwig's avatar
Christoph Hellwig committed
2583
			record_cnt++;
2584
			if (wrote_start_rec) {
2585 2586 2587
				copy_len += sizeof(struct xlog_op_header);
				record_cnt++;
			}
Christoph Hellwig's avatar
Christoph Hellwig committed
2588 2589
			data_cnt += contwr ? copy_len : 0;

2590
			error = xlog_write_copy_finish(log, iclog, optype,
Christoph Hellwig's avatar
Christoph Hellwig committed
2591 2592 2593
						       &record_cnt, &data_cnt,
						       &partial_copy,
						       &partial_copy_len,
2594
						       log_offset);
Christoph Hellwig's avatar
Christoph Hellwig committed
2595 2596 2597 2598 2599 2600 2601 2602 2603 2604 2605 2606 2607 2608 2609 2610 2611 2612
			if (error)
				return error;

			/*
			 * if we had a partial copy, we need to get more iclog
			 * space but we don't want to increment the region
			 * index because there is still more is this region to
			 * write.
			 *
			 * If we completed writing this region, and we flushed
			 * the iclog (indicated by resetting of the record
			 * count), then we also need to get more log space. If
			 * this was the last record, though, we are done and
			 * can just return.
			 */
			if (partial_copy)
				break;

2613
			if (++index == lv->lv_niovecs) {
2614
next_lv:
2615 2616 2617 2618 2619
				lv = lv->lv_next;
				index = 0;
				if (lv)
					vecp = lv->lv_iovecp;
			}
2620
			if (record_cnt == 0 && !ordered) {
2621
				if (!lv)
Christoph Hellwig's avatar
Christoph Hellwig committed
2622 2623 2624 2625 2626 2627 2628 2629
					return 0;
				break;
			}
		}
	}

	ASSERT(len == 0);

2630
	spin_lock(&log->l_icloglock);
Christoph Hellwig's avatar
Christoph Hellwig committed
2631
	xlog_state_finish_copy(log, iclog, record_cnt, data_cnt);
2632
	error = xlog_state_release_iclog(log, iclog, 0);
2633
	spin_unlock(&log->l_icloglock);
Linus Torvalds's avatar
Linus Torvalds committed
2634

2635
	return error;
Christoph Hellwig's avatar
Christoph Hellwig committed
2636
}
Linus Torvalds's avatar
Linus Torvalds committed
2637

2638 2639 2640 2641 2642 2643
static void
xlog_state_activate_iclog(
	struct xlog_in_core	*iclog,
	int			*iclogs_changed)
{
	ASSERT(list_empty_careful(&iclog->ic_callbacks));
2644
	trace_xlog_iclog_activate(iclog, _RET_IP_);
2645 2646 2647 2648 2649 2650 2651 2652 2653 2654 2655 2656 2657 2658 2659 2660 2661 2662 2663 2664 2665 2666 2667 2668

	/*
	 * If the number of ops in this iclog indicate it just contains the
	 * dummy transaction, we can change state into IDLE (the second time
	 * around). Otherwise we should change the state into NEED a dummy.
	 * We don't need to cover the dummy.
	 */
	if (*iclogs_changed == 0 &&
	    iclog->ic_header.h_num_logops == cpu_to_be32(XLOG_COVER_OPS)) {
		*iclogs_changed = 1;
	} else {
		/*
		 * We have two dirty iclogs so start over.  This could also be
		 * num of ops indicating this is not the dummy going out.
		 */
		*iclogs_changed = 2;
	}

	iclog->ic_state	= XLOG_STATE_ACTIVE;
	iclog->ic_offset = 0;
	iclog->ic_header.h_num_logops = 0;
	memset(iclog->ic_header.h_cycle_data, 0,
		sizeof(iclog->ic_header.h_cycle_data));
	iclog->ic_header.h_lsn = 0;
2669
	iclog->ic_header.h_tail_lsn = 0;
2670 2671
}

2672
/*
2673 2674
 * Loop through all iclogs and mark all iclogs currently marked DIRTY as
 * ACTIVE after iclog I/O has completed.
Linus Torvalds's avatar
Linus Torvalds committed
2675
 */
2676 2677
static void
xlog_state_activate_iclogs(
2678
	struct xlog		*log,
2679
	int			*iclogs_changed)
Linus Torvalds's avatar
Linus Torvalds committed
2680
{
2681
	struct xlog_in_core	*iclog = log->l_iclog;
Linus Torvalds's avatar
Linus Torvalds committed
2682 2683

	do {
2684 2685 2686 2687 2688 2689 2690 2691 2692 2693
		if (iclog->ic_state == XLOG_STATE_DIRTY)
			xlog_state_activate_iclog(iclog, iclogs_changed);
		/*
		 * The ordering of marking iclogs ACTIVE must be maintained, so
		 * an iclog doesn't become ACTIVE beyond one that is SYNCING.
		 */
		else if (iclog->ic_state != XLOG_STATE_ACTIVE)
			break;
	} while ((iclog = iclog->ic_next) != log->l_iclog);
}
2694

2695 2696 2697 2698 2699
static int
xlog_covered_state(
	int			prev_state,
	int			iclogs_changed)
{
2700
	/*
2701 2702 2703 2704
	 * We go to NEED for any non-covering writes. We go to NEED2 if we just
	 * wrote the first covering record (DONE). We go to IDLE if we just
	 * wrote the second covering record (DONE2) and remain in IDLE until a
	 * non-covering write occurs.
2705
	 */
2706 2707
	switch (prev_state) {
	case XLOG_STATE_COVER_IDLE:
2708 2709
		if (iclogs_changed == 1)
			return XLOG_STATE_COVER_IDLE;
2710
		fallthrough;
2711 2712 2713 2714 2715 2716 2717 2718 2719 2720 2721 2722 2723 2724
	case XLOG_STATE_COVER_NEED:
	case XLOG_STATE_COVER_NEED2:
		break;
	case XLOG_STATE_COVER_DONE:
		if (iclogs_changed == 1)
			return XLOG_STATE_COVER_NEED2;
		break;
	case XLOG_STATE_COVER_DONE2:
		if (iclogs_changed == 1)
			return XLOG_STATE_COVER_IDLE;
		break;
	default:
		ASSERT(0);
	}
2725

2726 2727
	return XLOG_STATE_COVER_NEED;
}
Linus Torvalds's avatar
Linus Torvalds committed
2728

2729 2730 2731 2732 2733 2734
STATIC void
xlog_state_clean_iclog(
	struct xlog		*log,
	struct xlog_in_core	*dirty_iclog)
{
	int			iclogs_changed = 0;
Linus Torvalds's avatar
Linus Torvalds committed
2735

2736 2737
	trace_xlog_iclog_clean(dirty_iclog, _RET_IP_);

2738
	dirty_iclog->ic_state = XLOG_STATE_DIRTY;
Linus Torvalds's avatar
Linus Torvalds committed
2739

2740 2741 2742 2743 2744 2745
	xlog_state_activate_iclogs(log, &iclogs_changed);
	wake_up_all(&dirty_iclog->ic_force_wait);

	if (iclogs_changed) {
		log->l_covered_state = xlog_covered_state(log->l_covered_state,
				iclogs_changed);
Linus Torvalds's avatar
Linus Torvalds committed
2746
	}
2747
}
Linus Torvalds's avatar
Linus Torvalds committed
2748 2749 2750

STATIC xfs_lsn_t
xlog_get_lowest_lsn(
2751
	struct xlog		*log)
Linus Torvalds's avatar
Linus Torvalds committed
2752
{
2753 2754
	struct xlog_in_core	*iclog = log->l_iclog;
	xfs_lsn_t		lowest_lsn = 0, lsn;
Linus Torvalds's avatar
Linus Torvalds committed
2755 2756

	do {
2757 2758
		if (iclog->ic_state == XLOG_STATE_ACTIVE ||
		    iclog->ic_state == XLOG_STATE_DIRTY)
2759 2760 2761 2762
			continue;

		lsn = be64_to_cpu(iclog->ic_header.h_lsn);
		if ((lsn && !lowest_lsn) || XFS_LSN_CMP(lsn, lowest_lsn) < 0)
Linus Torvalds's avatar
Linus Torvalds committed
2763
			lowest_lsn = lsn;
2764 2765
	} while ((iclog = iclog->ic_next) != log->l_iclog);

2766
	return lowest_lsn;
Linus Torvalds's avatar
Linus Torvalds committed
2767 2768
}

2769 2770 2771 2772 2773 2774 2775 2776 2777 2778 2779 2780 2781 2782 2783 2784 2785 2786 2787 2788 2789 2790 2791 2792 2793 2794 2795 2796
/*
 * Completion of a iclog IO does not imply that a transaction has completed, as
 * transactions can be large enough to span many iclogs. We cannot change the
 * tail of the log half way through a transaction as this may be the only
 * transaction in the log and moving the tail to point to the middle of it
 * will prevent recovery from finding the start of the transaction. Hence we
 * should only update the last_sync_lsn if this iclog contains transaction
 * completion callbacks on it.
 *
 * We have to do this before we drop the icloglock to ensure we are the only one
 * that can update it.
 *
 * If we are moving the last_sync_lsn forwards, we also need to ensure we kick
 * the reservation grant head pushing. This is due to the fact that the push
 * target is bound by the current last_sync_lsn value. Hence if we have a large
 * amount of log space bound up in this committing transaction then the
 * last_sync_lsn value may be the limiting factor preventing tail pushing from
 * freeing space in the log. Hence once we've updated the last_sync_lsn we
 * should push the AIL to ensure the push target (and hence the grant head) is
 * no longer bound by the old log head location and can move forwards and make
 * progress again.
 */
static void
xlog_state_set_callback(
	struct xlog		*log,
	struct xlog_in_core	*iclog,
	xfs_lsn_t		header_lsn)
{
2797
	trace_xlog_iclog_callback(iclog, _RET_IP_);
2798 2799 2800 2801 2802 2803 2804 2805 2806 2807 2808 2809
	iclog->ic_state = XLOG_STATE_CALLBACK;

	ASSERT(XFS_LSN_CMP(atomic64_read(&log->l_last_sync_lsn),
			   header_lsn) <= 0);

	if (list_empty_careful(&iclog->ic_callbacks))
		return;

	atomic64_set(&log->l_last_sync_lsn, header_lsn);
	xlog_grant_push_ail(log, 0);
}

2810 2811 2812 2813 2814 2815 2816 2817
/*
 * Return true if we need to stop processing, false to continue to the next
 * iclog. The caller will need to run callbacks if the iclog is returned in the
 * XLOG_STATE_CALLBACK state.
 */
static bool
xlog_state_iodone_process_iclog(
	struct xlog		*log,
2818
	struct xlog_in_core	*iclog)
2819 2820
{
	xfs_lsn_t		lowest_lsn;
2821
	xfs_lsn_t		header_lsn;
2822

2823 2824 2825 2826 2827 2828
	switch (iclog->ic_state) {
	case XLOG_STATE_ACTIVE:
	case XLOG_STATE_DIRTY:
		/*
		 * Skip all iclogs in the ACTIVE & DIRTY states:
		 */
2829
		return false;
2830 2831
	case XLOG_STATE_DONE_SYNC:
		/*
2832 2833 2834 2835
		 * Now that we have an iclog that is in the DONE_SYNC state, do
		 * one more check here to see if we have chased our tail around.
		 * If this is not the lowest lsn iclog, then we will leave it
		 * for another completion to process.
2836 2837 2838 2839 2840 2841 2842 2843 2844 2845
		 */
		header_lsn = be64_to_cpu(iclog->ic_header.h_lsn);
		lowest_lsn = xlog_get_lowest_lsn(log);
		if (lowest_lsn && XFS_LSN_CMP(lowest_lsn, header_lsn) < 0)
			return false;
		xlog_state_set_callback(log, iclog, header_lsn);
		return false;
	default:
		/*
		 * Can only perform callbacks in order.  Since this iclog is not
2846 2847
		 * in the DONE_SYNC state, we skip the rest and just try to
		 * clean up.
2848
		 */
2849 2850 2851 2852
		return true;
	}
}

2853 2854
/*
 * Loop over all the iclogs, running attached callbacks on them. Return true if
2855 2856 2857 2858
 * we ran any callbacks, indicating that we dropped the icloglock. We don't need
 * to handle transient shutdown state here at all because
 * xlog_state_shutdown_callbacks() will be run to do the necessary shutdown
 * cleanup of the callbacks.
2859 2860 2861 2862 2863 2864 2865 2866 2867 2868 2869 2870 2871 2872
 */
static bool
xlog_state_do_iclog_callbacks(
	struct xlog		*log)
		__releases(&log->l_icloglock)
		__acquires(&log->l_icloglock)
{
	struct xlog_in_core	*first_iclog = log->l_iclog;
	struct xlog_in_core	*iclog = first_iclog;
	bool			ran_callback = false;

	do {
		LIST_HEAD(cb_list);

2873 2874 2875 2876 2877
		if (xlog_state_iodone_process_iclog(log, iclog))
			break;
		if (iclog->ic_state != XLOG_STATE_CALLBACK) {
			iclog = iclog->ic_next;
			continue;
2878 2879 2880 2881 2882 2883 2884 2885 2886 2887
		}
		list_splice_init(&iclog->ic_callbacks, &cb_list);
		spin_unlock(&log->l_icloglock);

		trace_xlog_iclog_callbacks_start(iclog, _RET_IP_);
		xlog_cil_process_committed(&cb_list);
		trace_xlog_iclog_callbacks_done(iclog, _RET_IP_);
		ran_callback = true;

		spin_lock(&log->l_icloglock);
2888
		xlog_state_clean_iclog(log, iclog);
2889 2890 2891 2892 2893 2894 2895 2896 2897 2898 2899
		iclog = iclog->ic_next;
	} while (iclog != first_iclog);

	return ran_callback;
}


/*
 * Loop running iclog completion callbacks until there are no more iclogs in a
 * state that can run callbacks.
 */
Linus Torvalds's avatar
Linus Torvalds committed
2900 2901
STATIC void
xlog_state_do_callback(
2902
	struct xlog		*log)
Linus Torvalds's avatar
Linus Torvalds committed
2903
{
2904 2905
	int			flushcnt = 0;
	int			repeats = 0;
Linus Torvalds's avatar
Linus Torvalds committed
2906

Eric Sandeen's avatar
Eric Sandeen committed
2907
	spin_lock(&log->l_icloglock);
2908 2909 2910
	while (xlog_state_do_iclog_callbacks(log)) {
		if (xlog_is_shutdown(log))
			break;
2911

2912
		if (++repeats > 5000) {
2913 2914
			flushcnt += repeats;
			repeats = 0;
2915
			xfs_warn(log->l_mp,
2916
				"%s: possible infinite loop (%d iterations)",
2917
				__func__, flushcnt);
Linus Torvalds's avatar
Linus Torvalds committed
2918
		}
2919
	}
Linus Torvalds's avatar
Linus Torvalds committed
2920

2921
	if (log->l_iclog->ic_state == XLOG_STATE_ACTIVE)
2922
		wake_up_all(&log->l_flush_wait);
2923 2924

	spin_unlock(&log->l_icloglock);
2925
}
Linus Torvalds's avatar
Linus Torvalds committed
2926 2927 2928 2929 2930 2931


/*
 * Finish transitioning this iclog to the dirty state.
 *
 * Callbacks could take time, so they are done outside the scope of the
2932
 * global state machine log lock.
Linus Torvalds's avatar
Linus Torvalds committed
2933
 */
2934
STATIC void
Linus Torvalds's avatar
Linus Torvalds committed
2935
xlog_state_done_syncing(
2936
	struct xlog_in_core	*iclog)
Linus Torvalds's avatar
Linus Torvalds committed
2937
{
2938
	struct xlog		*log = iclog->ic_log;
Linus Torvalds's avatar
Linus Torvalds committed
2939

Eric Sandeen's avatar
Eric Sandeen committed
2940
	spin_lock(&log->l_icloglock);
2941
	ASSERT(atomic_read(&iclog->ic_refcnt) == 0);
2942
	trace_xlog_iclog_sync_done(iclog, _RET_IP_);
Linus Torvalds's avatar
Linus Torvalds committed
2943 2944 2945

	/*
	 * If we got an error, either on the first buffer, or in the case of
2946 2947
	 * split log writes, on the second, we shut down the file system and
	 * no iclogs should ever be attempted to be written to disk again.
Linus Torvalds's avatar
Linus Torvalds committed
2948
	 */
2949
	if (!xlog_is_shutdown(log)) {
2950
		ASSERT(iclog->ic_state == XLOG_STATE_SYNCING);
Linus Torvalds's avatar
Linus Torvalds committed
2951
		iclog->ic_state = XLOG_STATE_DONE_SYNC;
2952
	}
Linus Torvalds's avatar
Linus Torvalds committed
2953 2954 2955 2956 2957 2958

	/*
	 * Someone could be sleeping prior to writing out the next
	 * iclog buffer, we wake them all, one will get to do the
	 * I/O, the others get to wait for the result.
	 */
2959
	wake_up_all(&iclog->ic_write_wait);
Eric Sandeen's avatar
Eric Sandeen committed
2960
	spin_unlock(&log->l_icloglock);
2961
	xlog_state_do_callback(log);
2962
}
Linus Torvalds's avatar
Linus Torvalds committed
2963 2964 2965

/*
 * If the head of the in-core log ring is not (ACTIVE or DIRTY), then we must
2966 2967 2968
 * sleep.  We wait on the flush queue on the head iclog as that should be
 * the first iclog to complete flushing. Hence if all iclogs are syncing,
 * we will wait here and all new writes will sleep until a sync completes.
Linus Torvalds's avatar
Linus Torvalds committed
2969 2970 2971 2972 2973 2974 2975 2976 2977 2978 2979 2980 2981
 *
 * The in-core logs are used in a circular fashion. They are not used
 * out-of-order even when an iclog past the head is free.
 *
 * return:
 *	* log_offset where xlog_write() can start writing into the in-core
 *		log's data space.
 *	* in-core log pointer to which xlog_write() should write.
 *	* boolean indicating this is a continued write to an in-core log.
 *		If this is the last write, then the in-core log's offset field
 *		needs to be incremented, depending on the amount of data which
 *		is copied.
 */
2982
STATIC int
Mark Tinguely's avatar
Mark Tinguely committed
2983 2984 2985 2986 2987 2988 2989
xlog_state_get_iclog_space(
	struct xlog		*log,
	int			len,
	struct xlog_in_core	**iclogp,
	struct xlog_ticket	*ticket,
	int			*continued_write,
	int			*logoffsetp)
Linus Torvalds's avatar
Linus Torvalds committed
2990 2991 2992 2993 2994 2995
{
	int		  log_offset;
	xlog_rec_header_t *head;
	xlog_in_core_t	  *iclog;

restart:
Eric Sandeen's avatar
Eric Sandeen committed
2996
	spin_lock(&log->l_icloglock);
2997
	if (xlog_is_shutdown(log)) {
Eric Sandeen's avatar
Eric Sandeen committed
2998
		spin_unlock(&log->l_icloglock);
2999
		return -EIO;
Linus Torvalds's avatar
Linus Torvalds committed
3000 3001 3002
	}

	iclog = log->l_iclog;
3003
	if (iclog->ic_state != XLOG_STATE_ACTIVE) {
3004
		XFS_STATS_INC(log->l_mp, xs_log_noiclogs);
3005 3006

		/* Wait for log writes to have flushed */
3007
		xlog_wait(&log->l_flush_wait, &log->l_icloglock);
Linus Torvalds's avatar
Linus Torvalds committed
3008 3009
		goto restart;
	}
3010

Linus Torvalds's avatar
Linus Torvalds committed
3011 3012
	head = &iclog->ic_header;

3013
	atomic_inc(&iclog->ic_refcnt);	/* prevents sync */
Linus Torvalds's avatar
Linus Torvalds committed
3014 3015
	log_offset = iclog->ic_offset;

3016 3017
	trace_xlog_iclog_get_space(iclog, _RET_IP_);

Linus Torvalds's avatar
Linus Torvalds committed
3018 3019 3020 3021 3022 3023 3024
	/* On the 1st write to an iclog, figure out lsn.  This works
	 * if iclogs marked XLOG_STATE_WANT_SYNC always write out what they are
	 * committing to.  If the offset is set, that's how many blocks
	 * must be written.
	 */
	if (log_offset == 0) {
		ticket->t_curr_res -= log->l_iclog_hsize;
3025
		xlog_tic_add_region(ticket,
3026 3027
				    log->l_iclog_hsize,
				    XLOG_REG_TYPE_LRHEADER);
3028 3029
		head->h_cycle = cpu_to_be32(log->l_curr_cycle);
		head->h_lsn = cpu_to_be64(
3030
			xlog_assign_lsn(log->l_curr_cycle, log->l_curr_block));
Linus Torvalds's avatar
Linus Torvalds committed
3031 3032 3033 3034 3035 3036 3037 3038 3039 3040 3041 3042 3043
		ASSERT(log->l_curr_block >= 0);
	}

	/* If there is enough room to write everything, then do it.  Otherwise,
	 * claim the rest of the region and make sure the XLOG_STATE_WANT_SYNC
	 * bit is on, so this will get flushed out.  Don't update ic_offset
	 * until you know exactly how many bytes get copied.  Therefore, wait
	 * until later to update ic_offset.
	 *
	 * xlog_write() algorithm assumes that at least 2 xlog_op_header_t's
	 * can fit into remaining data section.
	 */
	if (iclog->ic_size - iclog->ic_offset < 2*sizeof(xlog_op_header_t)) {
3044 3045
		int		error = 0;

Linus Torvalds's avatar
Linus Torvalds committed
3046 3047
		xlog_state_switch_iclogs(log, iclog, iclog->ic_size);

3048
		/*
3049 3050 3051
		 * If we are the only one writing to this iclog, sync it to
		 * disk.  We need to do an atomic compare and decrement here to
		 * avoid racing with concurrent atomic_dec_and_lock() calls in
3052 3053 3054
		 * xlog_state_release_iclog() when there is more than one
		 * reference to the iclog.
		 */
3055
		if (!atomic_add_unless(&iclog->ic_refcnt, -1, 1))
3056
			error = xlog_state_release_iclog(log, iclog, 0);
3057 3058 3059
		spin_unlock(&log->l_icloglock);
		if (error)
			return error;
Linus Torvalds's avatar
Linus Torvalds committed
3060 3061 3062 3063 3064 3065 3066 3067 3068 3069 3070 3071 3072 3073 3074 3075 3076 3077 3078
		goto restart;
	}

	/* Do we have enough room to write the full amount in the remainder
	 * of this iclog?  Or must we continue a write on the next iclog and
	 * mark this iclog as completely taken?  In the case where we switch
	 * iclogs (to mark it taken), this particular iclog will release/sync
	 * to disk in xlog_write().
	 */
	if (len <= iclog->ic_size - iclog->ic_offset) {
		*continued_write = 0;
		iclog->ic_offset += len;
	} else {
		*continued_write = 1;
		xlog_state_switch_iclogs(log, iclog, iclog->ic_size);
	}
	*iclogp = iclog;

	ASSERT(iclog->ic_offset <= iclog->ic_size);
Eric Sandeen's avatar
Eric Sandeen committed
3079
	spin_unlock(&log->l_icloglock);
Linus Torvalds's avatar
Linus Torvalds committed
3080 3081 3082

	*logoffsetp = log_offset;
	return 0;
3083
}
Linus Torvalds's avatar
Linus Torvalds committed
3084

3085
/*
3086 3087 3088 3089 3090
 * The first cnt-1 times a ticket goes through here we don't need to move the
 * grant write head because the permanent reservation has reserved cnt times the
 * unit amount.  Release part of current permanent unit reservation and reset
 * current reservation to be one units worth.  Also move grant reservation head
 * forward.
Linus Torvalds's avatar
Linus Torvalds committed
3091
 */
3092 3093
void
xfs_log_ticket_regrant(
Mark Tinguely's avatar
Mark Tinguely committed
3094 3095
	struct xlog		*log,
	struct xlog_ticket	*ticket)
Linus Torvalds's avatar
Linus Torvalds committed
3096
{
3097
	trace_xfs_log_ticket_regrant(log, ticket);
3098

Linus Torvalds's avatar
Linus Torvalds committed
3099 3100 3101
	if (ticket->t_cnt > 0)
		ticket->t_cnt--;

3102
	xlog_grant_sub_space(log, &log->l_reserve_head.grant,
3103
					ticket->t_curr_res);
3104
	xlog_grant_sub_space(log, &log->l_write_head.grant,
3105
					ticket->t_curr_res);
Linus Torvalds's avatar
Linus Torvalds committed
3106
	ticket->t_curr_res = ticket->t_unit_res;
3107
	xlog_tic_reset_res(ticket);
3108

3109
	trace_xfs_log_ticket_regrant_sub(log, ticket);
3110

Linus Torvalds's avatar
Linus Torvalds committed
3111
	/* just return if we still have some of the pre-reserved space */
3112 3113 3114 3115
	if (!ticket->t_cnt) {
		xlog_grant_add_space(log, &log->l_reserve_head.grant,
				     ticket->t_unit_res);
		trace_xfs_log_ticket_regrant_exit(log, ticket);
Linus Torvalds's avatar
Linus Torvalds committed
3116

3117 3118 3119
		ticket->t_curr_res = ticket->t_unit_res;
		xlog_tic_reset_res(ticket);
	}
Linus Torvalds's avatar
Linus Torvalds committed
3120

3121 3122
	xfs_log_ticket_put(ticket);
}
Linus Torvalds's avatar
Linus Torvalds committed
3123 3124 3125 3126 3127 3128 3129 3130 3131 3132 3133 3134 3135 3136 3137

/*
 * Give back the space left from a reservation.
 *
 * All the information we need to make a correct determination of space left
 * is present.  For non-permanent reservations, things are quite easy.  The
 * count should have been decremented to zero.  We only need to deal with the
 * space remaining in the current reservation part of the ticket.  If the
 * ticket contains a permanent reservation, there may be left over space which
 * needs to be released.  A count of N means that N-1 refills of the current
 * reservation can be done before we need to ask for more space.  The first
 * one goes to fill up the first current reservation.  Once we run out of
 * space, the count will stay at zero and the only space remaining will be
 * in the current reservation field.
 */
3138 3139
void
xfs_log_ticket_ungrant(
Mark Tinguely's avatar
Mark Tinguely committed
3140 3141
	struct xlog		*log,
	struct xlog_ticket	*ticket)
Linus Torvalds's avatar
Linus Torvalds committed
3142
{
3143 3144 3145
	int			bytes;

	trace_xfs_log_ticket_ungrant(log, ticket);
3146

Linus Torvalds's avatar
Linus Torvalds committed
3147 3148 3149
	if (ticket->t_cnt > 0)
		ticket->t_cnt--;

3150
	trace_xfs_log_ticket_ungrant_sub(log, ticket);
Linus Torvalds's avatar
Linus Torvalds committed
3151

3152 3153
	/*
	 * If this is a permanent reservation ticket, we may be able to free
Linus Torvalds's avatar
Linus Torvalds committed
3154 3155
	 * up more space based on the remaining count.
	 */
3156
	bytes = ticket->t_curr_res;
Linus Torvalds's avatar
Linus Torvalds committed
3157 3158
	if (ticket->t_cnt > 0) {
		ASSERT(ticket->t_flags & XLOG_TIC_PERM_RESERV);
3159
		bytes += ticket->t_unit_res*ticket->t_cnt;
Linus Torvalds's avatar
Linus Torvalds committed
3160 3161
	}

3162 3163
	xlog_grant_sub_space(log, &log->l_reserve_head.grant, bytes);
	xlog_grant_sub_space(log, &log->l_write_head.grant, bytes);
3164

3165
	trace_xfs_log_ticket_ungrant_exit(log, ticket);
3166

3167
	xfs_log_space_wake(log->l_mp);
3168
	xfs_log_ticket_put(ticket);
3169
}
Linus Torvalds's avatar
Linus Torvalds committed
3170 3171

/*
3172 3173
 * This routine will mark the current iclog in the ring as WANT_SYNC and move
 * the current iclog pointer to the next iclog in the ring.
Linus Torvalds's avatar
Linus Torvalds committed
3174
 */
3175
void
Mark Tinguely's avatar
Mark Tinguely committed
3176 3177 3178 3179
xlog_state_switch_iclogs(
	struct xlog		*log,
	struct xlog_in_core	*iclog,
	int			eventual_size)
Linus Torvalds's avatar
Linus Torvalds committed
3180 3181
{
	ASSERT(iclog->ic_state == XLOG_STATE_ACTIVE);
3182
	assert_spin_locked(&log->l_icloglock);
3183
	trace_xlog_iclog_switch(iclog, _RET_IP_);
3184

Linus Torvalds's avatar
Linus Torvalds committed
3185 3186 3187
	if (!eventual_size)
		eventual_size = iclog->ic_offset;
	iclog->ic_state = XLOG_STATE_WANT_SYNC;
3188
	iclog->ic_header.h_prev_block = cpu_to_be32(log->l_prev_block);
Linus Torvalds's avatar
Linus Torvalds committed
3189 3190 3191 3192 3193 3194 3195
	log->l_prev_block = log->l_curr_block;
	log->l_prev_cycle = log->l_curr_cycle;

	/* roll log?: ic_offset changed later */
	log->l_curr_block += BTOBB(eventual_size)+BTOBB(log->l_iclog_hsize);

	/* Round up to next log-sunit */
3196
	if (log->l_iclog_roundoff > BBSIZE) {
3197
		uint32_t sunit_bb = BTOBB(log->l_iclog_roundoff);
Linus Torvalds's avatar
Linus Torvalds committed
3198 3199 3200 3201
		log->l_curr_block = roundup(log->l_curr_block, sunit_bb);
	}

	if (log->l_curr_block >= log->l_logBBsize) {
3202 3203 3204 3205 3206 3207 3208 3209 3210 3211
		/*
		 * Rewind the current block before the cycle is bumped to make
		 * sure that the combined LSN never transiently moves forward
		 * when the log wraps to the next cycle. This is to support the
		 * unlocked sample of these fields from xlog_valid_lsn(). Most
		 * other cases should acquire l_icloglock.
		 */
		log->l_curr_block -= log->l_logBBsize;
		ASSERT(log->l_curr_block >= 0);
		smp_wmb();
Linus Torvalds's avatar
Linus Torvalds committed
3212 3213 3214 3215 3216 3217
		log->l_curr_cycle++;
		if (log->l_curr_cycle == XLOG_HEADER_MAGIC_NUM)
			log->l_curr_cycle++;
	}
	ASSERT(iclog == log->l_iclog);
	log->l_iclog = iclog->ic_next;
3218
}
Linus Torvalds's avatar
Linus Torvalds committed
3219

3220 3221 3222 3223 3224 3225 3226 3227 3228 3229 3230 3231 3232 3233 3234 3235 3236 3237 3238 3239 3240 3241 3242 3243 3244 3245 3246 3247 3248
/*
 * Force the iclog to disk and check if the iclog has been completed before
 * xlog_force_iclog() returns. This can happen on synchronous (e.g.
 * pmem) or fast async storage because we drop the icloglock to issue the IO.
 * If completion has already occurred, tell the caller so that it can avoid an
 * unnecessary wait on the iclog.
 */
static int
xlog_force_and_check_iclog(
	struct xlog_in_core	*iclog,
	bool			*completed)
{
	xfs_lsn_t		lsn = be64_to_cpu(iclog->ic_header.h_lsn);
	int			error;

	*completed = false;
	error = xlog_force_iclog(iclog);
	if (error)
		return error;

	/*
	 * If the iclog has already been completed and reused the header LSN
	 * will have been rewritten by completion
	 */
	if (be64_to_cpu(iclog->ic_header.h_lsn) != lsn)
		*completed = true;
	return 0;
}

Linus Torvalds's avatar
Linus Torvalds committed
3249 3250 3251 3252 3253 3254 3255 3256 3257 3258 3259 3260 3261 3262 3263 3264
/*
 * Write out all data in the in-core log as of this exact moment in time.
 *
 * Data may be written to the in-core log during this call.  However,
 * we don't guarantee this data will be written out.  A change from past
 * implementation means this routine will *not* write out zero length LRs.
 *
 * Basically, we try and perform an intelligent scan of the in-core logs.
 * If we determine there is no flushable data, we just return.  There is no
 * flushable data if:
 *
 *	1. the current iclog is active and has no data; the previous iclog
 *		is in the active or dirty state.
 *	2. the current iclog is drity, and the previous iclog is in the
 *		active or dirty state.
 *
3265
 * We may sleep if:
Linus Torvalds's avatar
Linus Torvalds committed
3266 3267 3268 3269 3270 3271 3272 3273 3274 3275
 *
 *	1. the current iclog is not in the active nor dirty state.
 *	2. the current iclog dirty, and the previous iclog is not in the
 *		active nor dirty state.
 *	3. the current iclog is active, and there is another thread writing
 *		to this particular iclog.
 *	4. a) the current iclog is active and has no other writers
 *	   b) when we return from flushing out this iclog, it is still
 *		not in the active nor dirty state.
 */
3276
int
3277
xfs_log_force(
3278
	struct xfs_mount	*mp,
3279
	uint			flags)
Linus Torvalds's avatar
Linus Torvalds committed
3280
{
3281
	struct xlog		*log = mp->m_log;
3282 3283
	struct xlog_in_core	*iclog;

3284
	XFS_STATS_INC(mp, xs_log_force);
3285
	trace_xfs_log_force(mp, 0, _RET_IP_);
Linus Torvalds's avatar
Linus Torvalds committed
3286

3287
	xlog_cil_force(log);
3288

Eric Sandeen's avatar
Eric Sandeen committed
3289
	spin_lock(&log->l_icloglock);
3290
	if (xlog_is_shutdown(log))
3291
		goto out_error;
Linus Torvalds's avatar
Linus Torvalds committed
3292

3293
	iclog = log->l_iclog;
3294 3295
	trace_xlog_iclog_force(iclog, _RET_IP_);

3296 3297 3298
	if (iclog->ic_state == XLOG_STATE_DIRTY ||
	    (iclog->ic_state == XLOG_STATE_ACTIVE &&
	     atomic_read(&iclog->ic_refcnt) == 0 && iclog->ic_offset == 0)) {
Linus Torvalds's avatar
Linus Torvalds committed
3299
		/*
3300 3301 3302 3303 3304
		 * If the head is dirty or (active and empty), then we need to
		 * look at the previous iclog.
		 *
		 * If the previous iclog is active or dirty we are done.  There
		 * is nothing to sync out. Otherwise, we attach ourselves to the
Linus Torvalds's avatar
Linus Torvalds committed
3305 3306
		 * previous iclog and go to sleep.
		 */
3307 3308 3309
		iclog = iclog->ic_prev;
	} else if (iclog->ic_state == XLOG_STATE_ACTIVE) {
		if (atomic_read(&iclog->ic_refcnt) == 0) {
3310
			/* We have exclusive access to this iclog. */
3311 3312 3313
			bool	completed;

			if (xlog_force_and_check_iclog(iclog, &completed))
3314
				goto out_error;
Linus Torvalds's avatar
Linus Torvalds committed
3315

3316
			if (completed)
3317 3318 3319
				goto out_unlock;
		} else {
			/*
3320 3321 3322
			 * Someone else is still writing to this iclog, so we
			 * need to ensure that when they release the iclog it
			 * gets synced immediately as we may be waiting on it.
3323 3324
			 */
			xlog_state_switch_iclogs(log, iclog, 0);
Linus Torvalds's avatar
Linus Torvalds committed
3325 3326
		}
	}
3327

3328 3329 3330 3331 3332 3333 3334 3335 3336
	/*
	 * The iclog we are about to wait on may contain the checkpoint pushed
	 * by the above xlog_cil_force() call, but it may not have been pushed
	 * to disk yet. Like the ACTIVE case above, we need to make sure caches
	 * are flushed when this iclog is written.
	 */
	if (iclog->ic_state == XLOG_STATE_WANT_SYNC)
		iclog->ic_flags |= XLOG_ICL_NEED_FLUSH | XLOG_ICL_NEED_FUA;

3337 3338
	if (flags & XFS_LOG_SYNC)
		return xlog_wait_on_iclog(iclog);
3339 3340 3341 3342 3343 3344
out_unlock:
	spin_unlock(&log->l_icloglock);
	return 0;
out_error:
	spin_unlock(&log->l_icloglock);
	return -EIO;
3345
}
Linus Torvalds's avatar
Linus Torvalds committed
3346

3347 3348 3349 3350 3351 3352 3353 3354 3355 3356 3357 3358 3359 3360
/*
 * Force the log to a specific LSN.
 *
 * If an iclog with that lsn can be found:
 *	If it is in the DIRTY state, just return.
 *	If it is in the ACTIVE state, move the in-core log into the WANT_SYNC
 *		state and go to sleep or return.
 *	If it is in any other state, go to sleep or return.
 *
 * Synchronous forces are implemented with a wait queue.  All callers trying
 * to force a given lsn to disk must wait on the queue attached to the
 * specific in-core log.  When given in-core log finally completes its write
 * to disk, that thread will wake up all threads waiting on the queue.
 */
3361
static int
3362 3363
xlog_force_lsn(
	struct xlog		*log,
3364 3365
	xfs_lsn_t		lsn,
	uint			flags,
3366 3367
	int			*log_flushed,
	bool			already_slept)
Linus Torvalds's avatar
Linus Torvalds committed
3368
{
3369
	struct xlog_in_core	*iclog;
3370
	bool			completed;
3371

3372
	spin_lock(&log->l_icloglock);
3373
	if (xlog_is_shutdown(log))
3374
		goto out_error;
Linus Torvalds's avatar
Linus Torvalds committed
3375

3376
	iclog = log->l_iclog;
3377
	while (be64_to_cpu(iclog->ic_header.h_lsn) != lsn) {
3378
		trace_xlog_iclog_force_lsn(iclog, _RET_IP_);
3379 3380 3381 3382
		iclog = iclog->ic_next;
		if (iclog == log->l_iclog)
			goto out_unlock;
	}
3383

3384 3385
	switch (iclog->ic_state) {
	case XLOG_STATE_ACTIVE:
3386 3387 3388 3389 3390 3391 3392 3393 3394 3395 3396 3397 3398 3399 3400 3401
		/*
		 * We sleep here if we haven't already slept (e.g. this is the
		 * first time we've looked at the correct iclog buf) and the
		 * buffer before us is going to be sync'ed.  The reason for this
		 * is that if we are doing sync transactions here, by waiting
		 * for the previous I/O to complete, we can allow a few more
		 * transactions into this iclog before we close it down.
		 *
		 * Otherwise, we mark the buffer WANT_SYNC, and bump up the
		 * refcnt so we can release the log (which drops the ref count).
		 * The state switch keeps new transaction commits from using
		 * this buffer.  When the current commits finish writing into
		 * the buffer, the refcount will drop to zero and the buffer
		 * will go out then.
		 */
		if (!already_slept &&
3402 3403
		    (iclog->ic_prev->ic_state == XLOG_STATE_WANT_SYNC ||
		     iclog->ic_prev->ic_state == XLOG_STATE_SYNCING)) {
3404 3405
			xlog_wait(&iclog->ic_prev->ic_write_wait,
					&log->l_icloglock);
3406
			return -EAGAIN;
Linus Torvalds's avatar
Linus Torvalds committed
3407
		}
3408
		if (xlog_force_and_check_iclog(iclog, &completed))
3409
			goto out_error;
3410 3411
		if (log_flushed)
			*log_flushed = 1;
3412 3413
		if (completed)
			goto out_unlock;
3414 3415 3416 3417 3418 3419 3420 3421 3422 3423 3424 3425 3426 3427 3428 3429 3430 3431 3432
		break;
	case XLOG_STATE_WANT_SYNC:
		/*
		 * This iclog may contain the checkpoint pushed by the
		 * xlog_cil_force_seq() call, but there are other writers still
		 * accessing it so it hasn't been pushed to disk yet. Like the
		 * ACTIVE case above, we need to make sure caches are flushed
		 * when this iclog is written.
		 */
		iclog->ic_flags |= XLOG_ICL_NEED_FLUSH | XLOG_ICL_NEED_FUA;
		break;
	default:
		/*
		 * The entire checkpoint was written by the CIL force and is on
		 * its way to disk already. It will be stable when it
		 * completes, so we don't need to manipulate caches here at all.
		 * We just need to wait for completion if necessary.
		 */
		break;
3433
	}
Linus Torvalds's avatar
Linus Torvalds committed
3434

3435 3436
	if (flags & XFS_LOG_SYNC)
		return xlog_wait_on_iclog(iclog);
3437
out_unlock:
3438 3439
	spin_unlock(&log->l_icloglock);
	return 0;
3440 3441 3442
out_error:
	spin_unlock(&log->l_icloglock);
	return -EIO;
3443 3444
}

3445
/*
3446
 * Force the log to a specific checkpoint sequence.
3447
 *
3448 3449 3450 3451 3452
 * First force the CIL so that all the required changes have been flushed to the
 * iclogs. If the CIL force completed it will return a commit LSN that indicates
 * the iclog that needs to be flushed to stable storage. If the caller needs
 * a synchronous log force, we will wait on the iclog with the LSN returned by
 * xlog_cil_force_seq() to be completed.
3453 3454
 */
int
3455
xfs_log_force_seq(
3456
	struct xfs_mount	*mp,
3457
	xfs_csn_t		seq,
3458 3459 3460
	uint			flags,
	int			*log_flushed)
{
3461 3462
	struct xlog		*log = mp->m_log;
	xfs_lsn_t		lsn;
3463
	int			ret;
3464
	ASSERT(seq != 0);
3465 3466

	XFS_STATS_INC(mp, xs_log_force);
3467
	trace_xfs_log_force(mp, seq, _RET_IP_);
3468

3469
	lsn = xlog_cil_force_seq(log, seq);
3470 3471 3472
	if (lsn == NULLCOMMITLSN)
		return 0;

3473 3474 3475 3476 3477
	ret = xlog_force_lsn(log, lsn, flags, log_flushed, false);
	if (ret == -EAGAIN) {
		XFS_STATS_INC(mp, xs_log_force_sleep);
		ret = xlog_force_lsn(log, lsn, flags, log_flushed, true);
	}
3478 3479 3480
	return ret;
}

Linus Torvalds's avatar
Linus Torvalds committed
3481
/*
Malcolm Parsons's avatar
Malcolm Parsons committed
3482
 * Free a used ticket when its refcount falls to zero.
Linus Torvalds's avatar
Linus Torvalds committed
3483
 */
3484 3485 3486
void
xfs_log_ticket_put(
	xlog_ticket_t	*ticket)
Linus Torvalds's avatar
Linus Torvalds committed
3487
{
3488
	ASSERT(atomic_read(&ticket->t_ref) > 0);
3489
	if (atomic_dec_and_test(&ticket->t_ref))
3490
		kmem_cache_free(xfs_log_ticket_zone, ticket);
3491
}
Linus Torvalds's avatar
Linus Torvalds committed
3492

3493 3494 3495 3496 3497 3498 3499 3500
xlog_ticket_t *
xfs_log_ticket_get(
	xlog_ticket_t	*ticket)
{
	ASSERT(atomic_read(&ticket->t_ref) > 0);
	atomic_inc(&ticket->t_ref);
	return ticket;
}
Linus Torvalds's avatar
Linus Torvalds committed
3501 3502

/*
3503 3504
 * Figure out the total log space unit (in bytes) that would be
 * required for a log ticket.
Linus Torvalds's avatar
Linus Torvalds committed
3505
 */
3506 3507 3508
static int
xlog_calc_unit_res(
	struct xlog		*log,
3509
	int			unit_bytes)
Linus Torvalds's avatar
Linus Torvalds committed
3510
{
3511 3512
	int			iclog_space;
	uint			num_headers;
Linus Torvalds's avatar
Linus Torvalds committed
3513 3514 3515 3516 3517 3518 3519 3520

	/*
	 * Permanent reservations have up to 'cnt'-1 active log operations
	 * in the log.  A unit in this case is the amount of space for one
	 * of these log operations.  Normal reservations have a cnt of 1
	 * and their unit amount is the total amount of space required.
	 *
	 * The following lines of code account for non-transaction data
3521 3522 3523 3524 3525 3526 3527 3528 3529 3530 3531 3532 3533 3534 3535 3536 3537 3538 3539 3540 3541 3542
	 * which occupy space in the on-disk log.
	 *
	 * Normal form of a transaction is:
	 * <oph><trans-hdr><start-oph><reg1-oph><reg1><reg2-oph>...<commit-oph>
	 * and then there are LR hdrs, split-recs and roundoff at end of syncs.
	 *
	 * We need to account for all the leadup data and trailer data
	 * around the transaction data.
	 * And then we need to account for the worst case in terms of using
	 * more space.
	 * The worst case will happen if:
	 * - the placement of the transaction happens to be such that the
	 *   roundoff is at its maximum
	 * - the transaction data is synced before the commit record is synced
	 *   i.e. <transaction-data><roundoff> | <commit-rec><roundoff>
	 *   Therefore the commit record is in its own Log Record.
	 *   This can happen as the commit record is called with its
	 *   own region to xlog_write().
	 *   This then means that in the worst case, roundoff can happen for
	 *   the commit-rec as well.
	 *   The commit-rec is smaller than padding in this scenario and so it is
	 *   not added separately.
Linus Torvalds's avatar
Linus Torvalds committed
3543 3544
	 */

3545 3546 3547 3548
	/* for trans header */
	unit_bytes += sizeof(xlog_op_header_t);
	unit_bytes += sizeof(xfs_trans_header_t);

Linus Torvalds's avatar
Linus Torvalds committed
3549
	/* for start-rec */
3550 3551
	unit_bytes += sizeof(xlog_op_header_t);

3552 3553 3554 3555 3556 3557 3558 3559 3560 3561 3562 3563 3564 3565 3566 3567 3568 3569 3570 3571 3572 3573 3574 3575 3576 3577 3578 3579 3580
	/*
	 * for LR headers - the space for data in an iclog is the size minus
	 * the space used for the headers. If we use the iclog size, then we
	 * undercalculate the number of headers required.
	 *
	 * Furthermore - the addition of op headers for split-recs might
	 * increase the space required enough to require more log and op
	 * headers, so take that into account too.
	 *
	 * IMPORTANT: This reservation makes the assumption that if this
	 * transaction is the first in an iclog and hence has the LR headers
	 * accounted to it, then the remaining space in the iclog is
	 * exclusively for this transaction.  i.e. if the transaction is larger
	 * than the iclog, it will be the only thing in that iclog.
	 * Fundamentally, this means we must pass the entire log vector to
	 * xlog_write to guarantee this.
	 */
	iclog_space = log->l_iclog_size - log->l_iclog_hsize;
	num_headers = howmany(unit_bytes, iclog_space);

	/* for split-recs - ophdrs added when data split over LRs */
	unit_bytes += sizeof(xlog_op_header_t) * num_headers;

	/* add extra header reservations if we overrun */
	while (!num_headers ||
	       howmany(unit_bytes, iclog_space) > num_headers) {
		unit_bytes += sizeof(xlog_op_header_t);
		num_headers++;
	}
3581
	unit_bytes += log->l_iclog_hsize * num_headers;
Linus Torvalds's avatar
Linus Torvalds committed
3582

3583 3584 3585
	/* for commit-rec LR header - note: padding will subsume the ophdr */
	unit_bytes += log->l_iclog_hsize;

3586 3587
	/* roundoff padding for transaction data and one for commit record */
	unit_bytes += 2 * log->l_iclog_roundoff;
Linus Torvalds's avatar
Linus Torvalds committed
3588

3589 3590 3591
	return unit_bytes;
}

3592 3593 3594 3595 3596 3597 3598 3599
int
xfs_log_calc_unit_res(
	struct xfs_mount	*mp,
	int			unit_bytes)
{
	return xlog_calc_unit_res(mp->m_log, unit_bytes);
}

3600 3601 3602 3603 3604 3605 3606 3607 3608
/*
 * Allocate and initialise a new log ticket.
 */
struct xlog_ticket *
xlog_ticket_alloc(
	struct xlog		*log,
	int			unit_bytes,
	int			cnt,
	char			client,
3609
	bool			permanent)
3610 3611 3612 3613
{
	struct xlog_ticket	*tic;
	int			unit_res;

3614
	tic = kmem_cache_zalloc(xfs_log_ticket_zone, GFP_NOFS | __GFP_NOFAIL);
3615

3616
	unit_res = xlog_calc_unit_res(log, unit_bytes);
3617

3618
	atomic_set(&tic->t_ref, 1);
3619
	tic->t_task		= current;
3620
	INIT_LIST_HEAD(&tic->t_queue);
3621 3622
	tic->t_unit_res		= unit_res;
	tic->t_curr_res		= unit_res;
Linus Torvalds's avatar
Linus Torvalds committed
3623 3624
	tic->t_cnt		= cnt;
	tic->t_ocnt		= cnt;
3625
	tic->t_tid		= prandom_u32();
Linus Torvalds's avatar
Linus Torvalds committed
3626
	tic->t_clientid		= client;
3627
	if (permanent)
Linus Torvalds's avatar
Linus Torvalds committed
3628 3629
		tic->t_flags |= XLOG_TIC_PERM_RESERV;

3630
	xlog_tic_reset_res(tic);
3631

Linus Torvalds's avatar
Linus Torvalds committed
3632
	return tic;
3633
}
Linus Torvalds's avatar
Linus Torvalds committed
3634

3635
#if defined(DEBUG)
Linus Torvalds's avatar
Linus Torvalds committed
3636 3637 3638 3639 3640
/*
 * Make sure that the destination ptr is within the valid data region of
 * one of the iclogs.  This uses backup pointers stored in a different
 * part of the log in case we trash the log structure.
 */
3641
STATIC void
3642
xlog_verify_dest_ptr(
3643
	struct xlog	*log,
3644
	void		*ptr)
Linus Torvalds's avatar
Linus Torvalds committed
3645 3646 3647 3648
{
	int i;
	int good_ptr = 0;

3649 3650 3651
	for (i = 0; i < log->l_iclog_bufs; i++) {
		if (ptr >= log->l_iclog_bak[i] &&
		    ptr <= log->l_iclog_bak[i] + log->l_iclog_size)
Linus Torvalds's avatar
Linus Torvalds committed
3652 3653
			good_ptr++;
	}
3654 3655

	if (!good_ptr)
3656
		xfs_emerg(log->l_mp, "%s: invalid ptr", __func__);
3657
}
Linus Torvalds's avatar
Linus Torvalds committed
3658

3659 3660 3661 3662 3663 3664 3665 3666 3667 3668 3669
/*
 * Check to make sure the grant write head didn't just over lap the tail.  If
 * the cycles are the same, we can't be overlapping.  Otherwise, make sure that
 * the cycles differ by exactly one and check the byte count.
 *
 * This check is run unlocked, so can give false positives. Rather than assert
 * on failures, use a warn-once flag and a panic tag to allow the admin to
 * determine if they want to panic the machine when such an error occurs. For
 * debug kernels this will have the same effect as using an assert but, unlinke
 * an assert, it can be turned off at runtime.
 */
3670 3671
STATIC void
xlog_verify_grant_tail(
3672
	struct xlog	*log)
3673
{
3674
	int		tail_cycle, tail_blocks;
3675
	int		cycle, space;
3676

3677
	xlog_crack_grant_head(&log->l_write_head.grant, &cycle, &space);
3678 3679
	xlog_crack_atomic_lsn(&log->l_tail_lsn, &tail_cycle, &tail_blocks);
	if (tail_cycle != cycle) {
3680
		if (cycle - 1 != tail_cycle &&
3681
		    !test_and_set_bit(XLOG_TAIL_WARN, &log->l_opstate)) {
3682 3683 3684 3685 3686
			xfs_alert_tag(log->l_mp, XFS_PTAG_LOGRES,
				"%s: cycle - 1 != tail_cycle", __func__);
		}

		if (space > BBTOB(tail_blocks) &&
3687
		    !test_and_set_bit(XLOG_TAIL_WARN, &log->l_opstate)) {
3688 3689 3690
			xfs_alert_tag(log->l_mp, XFS_PTAG_LOGRES,
				"%s: space > BBTOB(tail_blocks)", __func__);
		}
3691 3692 3693
	}
}

Linus Torvalds's avatar
Linus Torvalds committed
3694 3695
/* check if it will fit */
STATIC void
Mark Tinguely's avatar
Mark Tinguely committed
3696 3697
xlog_verify_tail_lsn(
	struct xlog		*log,
3698
	struct xlog_in_core	*iclog)
Linus Torvalds's avatar
Linus Torvalds committed
3699
{
3700 3701
	xfs_lsn_t	tail_lsn = be64_to_cpu(iclog->ic_header.h_tail_lsn);
	int		blocks;
Linus Torvalds's avatar
Linus Torvalds committed
3702 3703 3704 3705 3706

    if (CYCLE_LSN(tail_lsn) == log->l_prev_cycle) {
	blocks =
	    log->l_logBBsize - (log->l_prev_block - BLOCK_LSN(tail_lsn));
	if (blocks < BTOBB(iclog->ic_offset)+BTOBB(log->l_iclog_hsize))
3707
		xfs_emerg(log->l_mp, "%s: ran out of log space", __func__);
Linus Torvalds's avatar
Linus Torvalds committed
3708 3709 3710 3711
    } else {
	ASSERT(CYCLE_LSN(tail_lsn)+1 == log->l_prev_cycle);

	if (BLOCK_LSN(tail_lsn) == log->l_prev_block)
3712
		xfs_emerg(log->l_mp, "%s: tail wrapped", __func__);
Linus Torvalds's avatar
Linus Torvalds committed
3713 3714 3715

	blocks = BLOCK_LSN(tail_lsn) - log->l_prev_block;
	if (blocks < BTOBB(iclog->ic_offset) + 1)
3716
		xfs_emerg(log->l_mp, "%s: ran out of log space", __func__);
Linus Torvalds's avatar
Linus Torvalds committed
3717
    }
3718
}
Linus Torvalds's avatar
Linus Torvalds committed
3719 3720 3721 3722 3723 3724 3725 3726 3727 3728 3729 3730 3731 3732 3733 3734 3735

/*
 * Perform a number of checks on the iclog before writing to disk.
 *
 * 1. Make sure the iclogs are still circular
 * 2. Make sure we have a good magic number
 * 3. Make sure we don't have magic numbers in the data
 * 4. Check fields of each log operation header for:
 *	A. Valid client identifier
 *	B. tid ptr value falls in valid ptr space (user space code)
 *	C. Length in log record header is correct according to the
 *		individual operation headers within record.
 * 5. When a bwrite will occur within 5 blocks of the front of the physical
 *	log, check the preceding blocks of the physical log to make sure all
 *	the cycle numbers agree with the current cycle number.
 */
STATIC void
Mark Tinguely's avatar
Mark Tinguely committed
3736 3737 3738
xlog_verify_iclog(
	struct xlog		*log,
	struct xlog_in_core	*iclog,
3739
	int			count)
Linus Torvalds's avatar
Linus Torvalds committed
3740 3741 3742 3743
{
	xlog_op_header_t	*ophead;
	xlog_in_core_t		*icptr;
	xlog_in_core_2_t	*xhdr;
3744
	void			*base_ptr, *ptr, *p;
3745
	ptrdiff_t		field_offset;
3746
	uint8_t			clientid;
Linus Torvalds's avatar
Linus Torvalds committed
3747 3748 3749 3750
	int			len, i, j, k, op_len;
	int			idx;

	/* check validity of iclog pointers */
Eric Sandeen's avatar
Eric Sandeen committed
3751
	spin_lock(&log->l_icloglock);
Linus Torvalds's avatar
Linus Torvalds committed
3752
	icptr = log->l_iclog;
3753 3754 3755
	for (i = 0; i < log->l_iclog_bufs; i++, icptr = icptr->ic_next)
		ASSERT(icptr);

Linus Torvalds's avatar
Linus Torvalds committed
3756
	if (icptr != log->l_iclog)
3757
		xfs_emerg(log->l_mp, "%s: corrupt iclog ring", __func__);
Eric Sandeen's avatar
Eric Sandeen committed
3758
	spin_unlock(&log->l_icloglock);
Linus Torvalds's avatar
Linus Torvalds committed
3759 3760

	/* check log magic numbers */
3761
	if (iclog->ic_header.h_magicno != cpu_to_be32(XLOG_HEADER_MAGIC_NUM))
3762
		xfs_emerg(log->l_mp, "%s: invalid magic num", __func__);
Linus Torvalds's avatar
Linus Torvalds committed
3763

3764 3765 3766
	base_ptr = ptr = &iclog->ic_header;
	p = &iclog->ic_header;
	for (ptr += BBSIZE; ptr < base_ptr + count; ptr += BBSIZE) {
3767
		if (*(__be32 *)ptr == cpu_to_be32(XLOG_HEADER_MAGIC_NUM))
3768 3769
			xfs_emerg(log->l_mp, "%s: unexpected magic num",
				__func__);
Linus Torvalds's avatar
Linus Torvalds committed
3770 3771 3772
	}

	/* check fields */
3773
	len = be32_to_cpu(iclog->ic_header.h_num_logops);
3774 3775
	base_ptr = ptr = iclog->ic_datap;
	ophead = ptr;
3776
	xhdr = iclog->ic_data;
Linus Torvalds's avatar
Linus Torvalds committed
3777
	for (i = 0; i < len; i++) {
3778
		ophead = ptr;
Linus Torvalds's avatar
Linus Torvalds committed
3779 3780

		/* clientid is only 1 byte */
3781 3782
		p = &ophead->oh_clientid;
		field_offset = p - base_ptr;
3783
		if (field_offset & 0x1ff) {
Linus Torvalds's avatar
Linus Torvalds committed
3784 3785
			clientid = ophead->oh_clientid;
		} else {
3786
			idx = BTOBBT((char *)&ophead->oh_clientid - iclog->ic_datap);
Linus Torvalds's avatar
Linus Torvalds committed
3787 3788 3789
			if (idx >= (XLOG_HEADER_CYCLE_SIZE / BBSIZE)) {
				j = idx / (XLOG_HEADER_CYCLE_SIZE / BBSIZE);
				k = idx % (XLOG_HEADER_CYCLE_SIZE / BBSIZE);
3790 3791
				clientid = xlog_get_client_id(
					xhdr[j].hic_xheader.xh_cycle_data[k]);
Linus Torvalds's avatar
Linus Torvalds committed
3792
			} else {
3793 3794
				clientid = xlog_get_client_id(
					iclog->ic_header.h_cycle_data[idx]);
Linus Torvalds's avatar
Linus Torvalds committed
3795 3796 3797
			}
		}
		if (clientid != XFS_TRANSACTION && clientid != XFS_LOG)
3798
			xfs_warn(log->l_mp,
3799
				"%s: invalid clientid %d op "PTR_FMT" offset 0x%lx",
3800 3801
				__func__, clientid, ophead,
				(unsigned long)field_offset);
Linus Torvalds's avatar
Linus Torvalds committed
3802 3803

		/* check length */
3804 3805
		p = &ophead->oh_len;
		field_offset = p - base_ptr;
3806
		if (field_offset & 0x1ff) {
3807
			op_len = be32_to_cpu(ophead->oh_len);
Linus Torvalds's avatar
Linus Torvalds committed
3808
		} else {
3809 3810
			idx = BTOBBT((uintptr_t)&ophead->oh_len -
				    (uintptr_t)iclog->ic_datap);
Linus Torvalds's avatar
Linus Torvalds committed
3811 3812 3813
			if (idx >= (XLOG_HEADER_CYCLE_SIZE / BBSIZE)) {
				j = idx / (XLOG_HEADER_CYCLE_SIZE / BBSIZE);
				k = idx % (XLOG_HEADER_CYCLE_SIZE / BBSIZE);
3814
				op_len = be32_to_cpu(xhdr[j].hic_xheader.xh_cycle_data[k]);
Linus Torvalds's avatar
Linus Torvalds committed
3815
			} else {
3816
				op_len = be32_to_cpu(iclog->ic_header.h_cycle_data[idx]);
Linus Torvalds's avatar
Linus Torvalds committed
3817 3818 3819 3820
			}
		}
		ptr += sizeof(xlog_op_header_t) + op_len;
	}
3821
}
3822
#endif
Linus Torvalds's avatar
Linus Torvalds committed
3823 3824

/*
3825 3826 3827
 * Perform a forced shutdown on the log. This should be called once and once
 * only by the high level filesystem shutdown code to shut the log subsystem
 * down cleanly.
3828
 *
3829 3830 3831 3832 3833 3834 3835
 * Our main objectives here are to make sure that:
 *	a. if the shutdown was not due to a log IO error, flush the logs to
 *	   disk. Anything modified after this is ignored.
 *	b. the log gets atomically marked 'XLOG_IO_ERROR' for all interested
 *	   parties to find out. Nothing new gets queued after this is done.
 *	c. Tasks sleeping on log reservations, pinned objects and
 *	   other resources get woken up.
3836
 *
3837 3838
 * Return true if the shutdown cause was a log IO error and we actually shut the
 * log down.
Linus Torvalds's avatar
Linus Torvalds committed
3839
 */
3840 3841 3842 3843
bool
xlog_force_shutdown(
	struct xlog	*log,
	int		shutdown_flags)
Linus Torvalds's avatar
Linus Torvalds committed
3844
{
3845
	bool		log_error = (shutdown_flags & SHUTDOWN_LOG_IO_ERROR);
Linus Torvalds's avatar
Linus Torvalds committed
3846 3847

	/*
3848 3849
	 * If this happens during log recovery then we aren't using the runtime
	 * log mechanisms yet so there's nothing to shut down.
Linus Torvalds's avatar
Linus Torvalds committed
3850
	 */
3851 3852
	if (!log || xlog_in_recovery(log))
		return false;
Linus Torvalds's avatar
Linus Torvalds committed
3853

3854
	ASSERT(!xlog_is_shutdown(log));
3855 3856

	/*
3857
	 * Flush all the completed transactions to disk before marking the log
3858 3859 3860 3861 3862 3863
	 * being shut down. We need to do this first as shutting down the log
	 * before the force will prevent the log force from flushing the iclogs
	 * to disk.
	 *
	 * Re-entry due to a log IO error shutdown during the log force is
	 * prevented by the atomicity of higher level shutdown code.
3864
	 */
3865 3866
	if (!log_error)
		xfs_log_force(log->l_mp, XFS_LOG_SYNC);
3867

Linus Torvalds's avatar
Linus Torvalds committed
3868
	/*
3869 3870 3871 3872 3873 3874 3875 3876 3877
	 * Atomically set the shutdown state. If the shutdown state is already
	 * set, there someone else is performing the shutdown and so we are done
	 * here. This should never happen because we should only ever get called
	 * once by the first shutdown caller.
	 *
	 * Much of the log state machine transitions assume that shutdown state
	 * cannot change once they hold the log->l_icloglock. Hence we need to
	 * hold that lock here, even though we use the atomic test_and_set_bit()
	 * operation to set the shutdown state.
Linus Torvalds's avatar
Linus Torvalds committed
3878
	 */
Eric Sandeen's avatar
Eric Sandeen committed
3879
	spin_lock(&log->l_icloglock);
3880 3881 3882 3883 3884
	if (test_and_set_bit(XLOG_IO_ERROR, &log->l_opstate)) {
		spin_unlock(&log->l_icloglock);
		ASSERT(0);
		return false;
	}
Eric Sandeen's avatar
Eric Sandeen committed
3885
	spin_unlock(&log->l_icloglock);
Linus Torvalds's avatar
Linus Torvalds committed
3886 3887

	/*
3888 3889 3890 3891
	 * We don't want anybody waiting for log reservations after this. That
	 * means we have to wake up everybody queued up on reserveq as well as
	 * writeq.  In addition, we make sure in xlog_{re}grant_log_space that
	 * we don't enqueue anything once the SHUTDOWN flag is set, and this
3892
	 * action is protected by the grant locks.
Linus Torvalds's avatar
Linus Torvalds committed
3893
	 */
3894 3895
	xlog_grant_head_wake_all(&log->l_reserve_head);
	xlog_grant_head_wake_all(&log->l_write_head);
Linus Torvalds's avatar
Linus Torvalds committed
3896 3897

	/*
3898 3899 3900 3901
	 * Wake up everybody waiting on xfs_log_force. Wake the CIL push first
	 * as if the log writes were completed. The abort handling in the log
	 * item committed callback functions will do this again under lock to
	 * avoid races.
Linus Torvalds's avatar
Linus Torvalds committed
3902
	 */
3903
	spin_lock(&log->l_cilp->xc_push_lock);
3904
	wake_up_all(&log->l_cilp->xc_start_wait);
3905
	wake_up_all(&log->l_cilp->xc_commit_wait);
3906
	spin_unlock(&log->l_cilp->xc_push_lock);
3907
	xlog_state_shutdown_callbacks(log);
Linus Torvalds's avatar
Linus Torvalds committed
3908

3909
	return log_error;
Linus Torvalds's avatar
Linus Torvalds committed
3910 3911
}

3912
STATIC int
Mark Tinguely's avatar
Mark Tinguely committed
3913 3914
xlog_iclogs_empty(
	struct xlog	*log)
Linus Torvalds's avatar
Linus Torvalds committed
3915 3916 3917 3918 3919 3920 3921 3922 3923
{
	xlog_in_core_t	*iclog;

	iclog = log->l_iclog;
	do {
		/* endianness does not matter here, zero is zero in
		 * any language.
		 */
		if (iclog->ic_header.h_num_logops)
3924
			return 0;
Linus Torvalds's avatar
Linus Torvalds committed
3925 3926
		iclog = iclog->ic_next;
	} while (iclog != log->l_iclog);
3927
	return 1;
Linus Torvalds's avatar
Linus Torvalds committed
3928
}
3929

3930 3931 3932 3933 3934 3935 3936 3937 3938 3939 3940 3941 3942 3943 3944 3945 3946
/*
 * Verify that an LSN stamped into a piece of metadata is valid. This is
 * intended for use in read verifiers on v5 superblocks.
 */
bool
xfs_log_check_lsn(
	struct xfs_mount	*mp,
	xfs_lsn_t		lsn)
{
	struct xlog		*log = mp->m_log;
	bool			valid;

	/*
	 * norecovery mode skips mount-time log processing and unconditionally
	 * resets the in-core LSN. We can't validate in this mode, but
	 * modifications are not allowed anyways so just return true.
	 */
3947
	if (xfs_has_norecovery(mp))
3948 3949 3950 3951 3952 3953 3954 3955 3956 3957 3958 3959 3960 3961 3962 3963 3964 3965 3966 3967 3968 3969 3970 3971
		return true;

	/*
	 * Some metadata LSNs are initialized to NULL (e.g., the agfl). This is
	 * handled by recovery and thus safe to ignore here.
	 */
	if (lsn == NULLCOMMITLSN)
		return true;

	valid = xlog_valid_lsn(mp->m_log, lsn);

	/* warn the user about what's gone wrong before verifier failure */
	if (!valid) {
		spin_lock(&log->l_icloglock);
		xfs_warn(mp,
"Corruption warning: Metadata has LSN (%d:%d) ahead of current LSN (%d:%d). "
"Please unmount and run xfs_repair (>= v4.3) to resolve.",
			 CYCLE_LSN(lsn), BLOCK_LSN(lsn),
			 log->l_curr_cycle, log->l_curr_block);
		spin_unlock(&log->l_icloglock);
	}

	return valid;
}
3972

3973 3974 3975 3976 3977 3978 3979 3980 3981 3982 3983 3984 3985 3986 3987 3988 3989 3990 3991
/*
 * Notify the log that we're about to start using a feature that is protected
 * by a log incompat feature flag.  This will prevent log covering from
 * clearing those flags.
 */
void
xlog_use_incompat_feat(
	struct xlog		*log)
{
	down_read(&log->l_incompat_users);
}

/* Notify the log that we've finished using log incompat features. */
void
xlog_drop_incompat_feat(
	struct xlog		*log)
{
	up_read(&log->l_incompat_users);
}