inode.c 136 KB
Newer Older
1
/*
2
 *  linux/fs/ext4/inode.c
3 4 5 6 7 8 9 10 11 12 13 14 15 16 17
 *
 * Copyright (C) 1992, 1993, 1994, 1995
 * Remy Card (card@masi.ibp.fr)
 * Laboratoire MASI - Institut Blaise Pascal
 * Universite Pierre et Marie Curie (Paris VI)
 *
 *  from
 *
 *  linux/fs/minix/inode.c
 *
 *  Copyright (C) 1991, 1992  Linus Torvalds
 *
 *  64-bit file support on 64-bit platforms by Jakub Jelinek
 *	(jj@sunsite.ms.mff.cuni.cz)
 *
18
 *  Assorted race fixes, rewrite of ext4_get_block() by Al Viro, 2000
19 20 21 22
 */

#include <linux/fs.h>
#include <linux/time.h>
23
#include <linux/jbd2.h>
24 25 26 27 28 29
#include <linux/highuid.h>
#include <linux/pagemap.h>
#include <linux/quotaops.h>
#include <linux/string.h>
#include <linux/buffer_head.h>
#include <linux/writeback.h>
30
#include <linux/pagevec.h>
31
#include <linux/mpage.h>
32
#include <linux/namei.h>
33 34
#include <linux/uio.h>
#include <linux/bio.h>
35
#include <linux/workqueue.h>
36
#include <linux/kernel.h>
37
#include <linux/printk.h>
38
#include <linux/slab.h>
39
#include <linux/ratelimit.h>
40

41
#include "ext4_jbd2.h"
42 43
#include "xattr.h"
#include "acl.h"
44
#include "truncate.h"
45

46 47
#include <trace/events/ext4.h>

48 49
#define MPAGE_DA_EXTENT_TAIL 0x01

50 51 52
static inline int ext4_begin_ordered_truncate(struct inode *inode,
					      loff_t new_size)
{
53
	trace_ext4_begin_ordered_truncate(inode, new_size);
54 55 56 57 58 59 60 61 62 63 64
	/*
	 * If jinode is zero, then we never opened the file for
	 * writing, so there's no need to call
	 * jbd2_journal_begin_ordered_truncate() since there's no
	 * outstanding writes we need to flush.
	 */
	if (!EXT4_I(inode)->jinode)
		return 0;
	return jbd2_journal_begin_ordered_truncate(EXT4_JOURNAL(inode),
						   EXT4_I(inode)->jinode,
						   new_size);
65 66
}

67
static void ext4_invalidatepage(struct page *page, unsigned long offset);
68 69 70 71 72 73
static int noalloc_get_block_write(struct inode *inode, sector_t iblock,
				   struct buffer_head *bh_result, int create);
static int ext4_set_bh_endio(struct buffer_head *bh, struct inode *inode);
static void ext4_end_io_buffer_write(struct buffer_head *bh, int uptodate);
static int __ext4_journalled_writepage(struct page *page, unsigned int len);
static int ext4_bh_delay_or_unwritten(handle_t *handle, struct buffer_head *bh);
74 75 76
static int ext4_discard_partial_page_buffers_no_lock(handle_t *handle,
		struct inode *inode, struct page *page, loff_t from,
		loff_t length, int flags);
77

78 79 80
/*
 * Test whether an inode is a fast symlink.
 */
81
static int ext4_inode_is_fast_symlink(struct inode *inode)
82
{
83
	int ea_blocks = EXT4_I(inode)->i_file_acl ?
84 85 86 87 88 89 90 91 92 93
		(inode->i_sb->s_blocksize >> 9) : 0;

	return (S_ISLNK(inode->i_mode) && inode->i_blocks - ea_blocks == 0);
}

/*
 * Restart the transaction associated with *handle.  This does a commit,
 * so before we call here everything must be consistently dirtied against
 * this transaction.
 */
94
int ext4_truncate_restart_trans(handle_t *handle, struct inode *inode,
95
				 int nblocks)
96
{
97 98 99
	int ret;

	/*
100
	 * Drop i_data_sem to avoid deadlock with ext4_map_blocks.  At this
101 102 103 104
	 * moment, get_block can be called only for blocks inside i_size since
	 * page cache has been already dropped and writes are blocked by
	 * i_mutex. So we can safely drop the i_data_sem here.
	 */
105
	BUG_ON(EXT4_JOURNAL(inode) == NULL);
106
	jbd_debug(2, "restarting handle %p\n", handle);
107
	up_write(&EXT4_I(inode)->i_data_sem);
108
	ret = ext4_journal_restart(handle, nblocks);
109
	down_write(&EXT4_I(inode)->i_data_sem);
110
	ext4_discard_preallocations(inode);
111 112

	return ret;
113 114 115 116 117
}

/*
 * Called at the last iput() if i_nlink is zero.
 */
Al Viro's avatar
Al Viro committed
118
void ext4_evict_inode(struct inode *inode)
119 120
{
	handle_t *handle;
121
	int err;
122

123
	trace_ext4_evict_inode(inode);
124 125 126

	ext4_ioend_wait(inode);

Al Viro's avatar
Al Viro committed
127
	if (inode->i_nlink) {
128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146
		/*
		 * When journalling data dirty buffers are tracked only in the
		 * journal. So although mm thinks everything is clean and
		 * ready for reaping the inode might still have some pages to
		 * write in the running transaction or waiting to be
		 * checkpointed. Thus calling jbd2_journal_invalidatepage()
		 * (via truncate_inode_pages()) to discard these buffers can
		 * cause data loss. Also even if we did not discard these
		 * buffers, we would have no way to find them after the inode
		 * is reaped and thus user could see stale data if he tries to
		 * read them before the transaction is checkpointed. So be
		 * careful and force everything to disk here... We use
		 * ei->i_datasync_tid to store the newest transaction
		 * containing inode's data.
		 *
		 * Note that directories do not have this problem because they
		 * don't use page cache.
		 */
		if (ext4_should_journal_data(inode) &&
147 148
		    (S_ISLNK(inode->i_mode) || S_ISREG(inode->i_mode)) &&
		    inode->i_ino != EXT4_JOURNAL_INO) {
149 150 151
			journal_t *journal = EXT4_SB(inode->i_sb)->s_journal;
			tid_t commit_tid = EXT4_I(inode)->i_datasync_tid;

152
			jbd2_complete_transaction(journal, commit_tid);
153 154
			filemap_write_and_wait(&inode->i_data);
		}
Al Viro's avatar
Al Viro committed
155 156 157 158
		truncate_inode_pages(&inode->i_data, 0);
		goto no_delete;
	}

159
	if (!is_bad_inode(inode))
160
		dquot_initialize(inode);
161

162 163
	if (ext4_should_order_data(inode))
		ext4_begin_ordered_truncate(inode, 0);
164 165 166 167 168
	truncate_inode_pages(&inode->i_data, 0);

	if (is_bad_inode(inode))
		goto no_delete;

169
	handle = ext4_journal_start(inode, ext4_blocks_for_truncate(inode)+3);
170
	if (IS_ERR(handle)) {
171
		ext4_std_error(inode->i_sb, PTR_ERR(handle));
172 173 174 175 176
		/*
		 * If we're going to skip the normal cleanup, we still need to
		 * make sure that the in-core orphan linked list is properly
		 * cleaned up.
		 */
177
		ext4_orphan_del(NULL, inode);
178 179 180 181
		goto no_delete;
	}

	if (IS_SYNC(inode))
182
		ext4_handle_sync(handle);
183
	inode->i_size = 0;
184 185
	err = ext4_mark_inode_dirty(handle, inode);
	if (err) {
186
		ext4_warning(inode->i_sb,
187 188 189
			     "couldn't mark inode dirty (err %d)", err);
		goto stop_handle;
	}
190
	if (inode->i_blocks)
191
		ext4_truncate(inode);
192 193 194 195 196 197 198

	/*
	 * ext4_ext_truncate() doesn't reserve any slop when it
	 * restarts journal transactions; therefore there may not be
	 * enough credits left in the handle to remove the inode from
	 * the orphan list and set the dtime field.
	 */
199
	if (!ext4_handle_has_enough_credits(handle, 3)) {
200 201 202 203
		err = ext4_journal_extend(handle, 3);
		if (err > 0)
			err = ext4_journal_restart(handle, 3);
		if (err != 0) {
204
			ext4_warning(inode->i_sb,
205 206 207
				     "couldn't extend journal (err %d)", err);
		stop_handle:
			ext4_journal_stop(handle);
208
			ext4_orphan_del(NULL, inode);
209 210 211 212
			goto no_delete;
		}
	}

213
	/*
214
	 * Kill off the orphan record which ext4_truncate created.
215
	 * AKPM: I think this can be inside the above `if'.
216
	 * Note that ext4_orphan_del() has to be able to cope with the
217
	 * deletion of a non-existent orphan - this is because we don't
218
	 * know if ext4_truncate() actually created an orphan record.
219 220
	 * (Well, we could do this if we need to, but heck - it works)
	 */
221 222
	ext4_orphan_del(handle, inode);
	EXT4_I(inode)->i_dtime	= get_seconds();
223 224 225 226 227 228 229 230

	/*
	 * One subtle ordering requirement: if anything has gone wrong
	 * (transaction abort, IO errors, whatever), then we can still
	 * do these next steps (the fs will already have been marked as
	 * having errors), but we can't free the inode if the mark_dirty
	 * fails.
	 */
231
	if (ext4_mark_inode_dirty(handle, inode))
232
		/* If that failed, just do the required in-core inode clear. */
Al Viro's avatar
Al Viro committed
233
		ext4_clear_inode(inode);
234
	else
235 236
		ext4_free_inode(handle, inode);
	ext4_journal_stop(handle);
237 238
	return;
no_delete:
Al Viro's avatar
Al Viro committed
239
	ext4_clear_inode(inode);	/* We must guarantee clearing of inode... */
240 241
}

242 243
#ifdef CONFIG_QUOTA
qsize_t *ext4_get_reserved_space(struct inode *inode)
244
{
245
	return &EXT4_I(inode)->i_reserved_quota;
246
}
247
#endif
248

249 250
/*
 * Calculate the number of metadata blocks need to reserve
251
 * to allocate a block located at @lblock
252
 */
253
static int ext4_calc_metadata_amount(struct inode *inode, ext4_lblk_t lblock)
254
{
255
	if (ext4_test_inode_flag(inode, EXT4_INODE_EXTENTS))
256
		return ext4_ext_calc_metadata_amount(inode, lblock);
257

258
	return ext4_ind_calc_metadata_amount(inode, lblock);
259 260
}

261 262 263 264
/*
 * Called with i_data_sem down, which is important since we can call
 * ext4_discard_preallocations() from here.
 */
265 266
void ext4_da_update_reserve_space(struct inode *inode,
					int used, int quota_claim)
267 268
{
	struct ext4_sb_info *sbi = EXT4_SB(inode->i_sb);
269 270 271
	struct ext4_inode_info *ei = EXT4_I(inode);

	spin_lock(&ei->i_block_reservation_lock);
272
	trace_ext4_da_update_reserve_space(inode, used, quota_claim);
273 274
	if (unlikely(used > ei->i_reserved_data_blocks)) {
		ext4_msg(inode->i_sb, KERN_NOTICE, "%s: ino %lu, used %d "
275
			 "with only %d reserved data blocks",
276 277 278 279 280
			 __func__, inode->i_ino, used,
			 ei->i_reserved_data_blocks);
		WARN_ON(1);
		used = ei->i_reserved_data_blocks;
	}
281

282 283 284 285 286 287 288 289 290
	if (unlikely(ei->i_allocated_meta_blocks > ei->i_reserved_meta_blocks)) {
		ext4_msg(inode->i_sb, KERN_NOTICE, "%s: ino %lu, allocated %d "
			 "with only %d reserved metadata blocks\n", __func__,
			 inode->i_ino, ei->i_allocated_meta_blocks,
			 ei->i_reserved_meta_blocks);
		WARN_ON(1);
		ei->i_allocated_meta_blocks = ei->i_reserved_meta_blocks;
	}

291 292 293
	/* Update per-inode reservations */
	ei->i_reserved_data_blocks -= used;
	ei->i_reserved_meta_blocks -= ei->i_allocated_meta_blocks;
294
	percpu_counter_sub(&sbi->s_dirtyclusters_counter,
295
			   used + ei->i_allocated_meta_blocks);
296
	ei->i_allocated_meta_blocks = 0;
297

298 299 300 301 302 303
	if (ei->i_reserved_data_blocks == 0) {
		/*
		 * We can release all of the reserved metadata blocks
		 * only when we have written all of the delayed
		 * allocation blocks.
		 */
304
		percpu_counter_sub(&sbi->s_dirtyclusters_counter,
305
				   ei->i_reserved_meta_blocks);
306
		ei->i_reserved_meta_blocks = 0;
307
		ei->i_da_metadata_calc_len = 0;
308
	}
309
	spin_unlock(&EXT4_I(inode)->i_block_reservation_lock);
310

311 312
	/* Update quota subsystem for data blocks */
	if (quota_claim)
313
		dquot_claim_block(inode, EXT4_C2B(sbi, used));
314
	else {
315 316 317
		/*
		 * We did fallocate with an offset that is already delayed
		 * allocated. So on delayed allocated writeback we should
318
		 * not re-claim the quota for fallocated blocks.
319
		 */
320
		dquot_release_reservation_block(inode, EXT4_C2B(sbi, used));
321
	}
322 323 324 325 326 327

	/*
	 * If we have done all the pending block allocations and if
	 * there aren't any writers on the inode, we can discard the
	 * inode's preallocations.
	 */
328 329
	if ((ei->i_reserved_data_blocks == 0) &&
	    (atomic_read(&inode->i_writecount) == 0))
330
		ext4_discard_preallocations(inode);
331 332
}

333
static int __check_block_validity(struct inode *inode, const char *func,
334 335
				unsigned int line,
				struct ext4_map_blocks *map)
336
{
337 338
	if (!ext4_data_block_valid(EXT4_SB(inode->i_sb), map->m_pblk,
				   map->m_len)) {
339 340 341 342
		ext4_error_inode(inode, func, line, map->m_pblk,
				 "lblock %lu mapped to illegal pblock "
				 "(length %d)", (unsigned long) map->m_lblk,
				 map->m_len);
343 344 345 346 347
		return -EIO;
	}
	return 0;
}

348
#define check_block_validity(inode, map)	\
349
	__check_block_validity((inode), __func__, __LINE__, (map))
350

351
/*
352 353
 * Return the number of contiguous dirty pages in a given inode
 * starting at page frame idx.
354 355 356 357 358 359 360 361 362 363 364 365 366 367 368 369 370 371 372 373 374 375 376 377 378 379 380 381 382 383 384 385 386
 */
static pgoff_t ext4_num_dirty_pages(struct inode *inode, pgoff_t idx,
				    unsigned int max_pages)
{
	struct address_space *mapping = inode->i_mapping;
	pgoff_t	index;
	struct pagevec pvec;
	pgoff_t num = 0;
	int i, nr_pages, done = 0;

	if (max_pages == 0)
		return 0;
	pagevec_init(&pvec, 0);
	while (!done) {
		index = idx;
		nr_pages = pagevec_lookup_tag(&pvec, mapping, &index,
					      PAGECACHE_TAG_DIRTY,
					      (pgoff_t)PAGEVEC_SIZE);
		if (nr_pages == 0)
			break;
		for (i = 0; i < nr_pages; i++) {
			struct page *page = pvec.pages[i];
			struct buffer_head *bh, *head;

			lock_page(page);
			if (unlikely(page->mapping != mapping) ||
			    !PageDirty(page) ||
			    PageWriteback(page) ||
			    page->index != idx) {
				done = 1;
				unlock_page(page);
				break;
			}
387 388 389 390 391 392 393 394 395
			if (page_has_buffers(page)) {
				bh = head = page_buffers(page);
				do {
					if (!buffer_delay(bh) &&
					    !buffer_unwritten(bh))
						done = 1;
					bh = bh->b_this_page;
				} while (!done && (bh != head));
			}
396 397 398 399 400
			unlock_page(page);
			if (done)
				break;
			idx++;
			num++;
401 402
			if (num >= max_pages) {
				done = 1;
403
				break;
404
			}
405 406 407 408 409 410
		}
		pagevec_release(&pvec);
	}
	return num;
}

411 412 413 414 415 416 417 418 419 420 421 422 423 424 425 426 427 428 429 430 431 432 433 434 435 436 437 438 439 440 441 442 443 444 445 446 447 448 449 450 451 452 453
/*
 * Sets the BH_Da_Mapped bit on the buffer heads corresponding to the given map.
 */
static void set_buffers_da_mapped(struct inode *inode,
				   struct ext4_map_blocks *map)
{
	struct address_space *mapping = inode->i_mapping;
	struct pagevec pvec;
	int i, nr_pages;
	pgoff_t index, end;

	index = map->m_lblk >> (PAGE_CACHE_SHIFT - inode->i_blkbits);
	end = (map->m_lblk + map->m_len - 1) >>
		(PAGE_CACHE_SHIFT - inode->i_blkbits);

	pagevec_init(&pvec, 0);
	while (index <= end) {
		nr_pages = pagevec_lookup(&pvec, mapping, index,
					  min(end - index + 1,
					      (pgoff_t)PAGEVEC_SIZE));
		if (nr_pages == 0)
			break;
		for (i = 0; i < nr_pages; i++) {
			struct page *page = pvec.pages[i];
			struct buffer_head *bh, *head;

			if (unlikely(page->mapping != mapping) ||
			    !PageDirty(page))
				break;

			if (page_has_buffers(page)) {
				bh = head = page_buffers(page);
				do {
					set_buffer_da_mapped(bh);
					bh = bh->b_this_page;
				} while (bh != head);
			}
			index++;
		}
		pagevec_release(&pvec);
	}
}

454
/*
455
 * The ext4_map_blocks() function tries to look up the requested blocks,
456
 * and returns if the blocks are already mapped.
457 458 459 460 461
 *
 * Otherwise it takes the write lock of the i_data_sem and allocate blocks
 * and store the allocated blocks in the result buffer head and mark it
 * mapped.
 *
462 463
 * If file type is extents based, it will call ext4_ext_map_blocks(),
 * Otherwise, call with ext4_ind_map_blocks() to handle indirect mapping
464 465 466 467 468 469 470 471
 * based files
 *
 * On success, it returns the number of blocks being mapped or allocate.
 * if create==0 and the blocks are pre-allocated and uninitialized block,
 * the result buffer head is unmapped. If the create ==1, it will make sure
 * the buffer head is mapped.
 *
 * It returns 0 if plain look up failed (blocks have not been allocated), in
472
 * that case, buffer head is unmapped
473 474 475
 *
 * It returns the error in case of allocation failure.
 */
476 477
int ext4_map_blocks(handle_t *handle, struct inode *inode,
		    struct ext4_map_blocks *map, int flags)
478 479
{
	int retval;
480

481 482 483 484
	map->m_flags = 0;
	ext_debug("ext4_map_blocks(): inode %lu, flag %d, max_blocks %u,"
		  "logical block %lu\n", inode->i_ino, flags, map->m_len,
		  (unsigned long) map->m_lblk);
485
	/*
486 487
	 * Try to see if we can get the block without requesting a new
	 * file system block.
488 489
	 */
	down_read((&EXT4_I(inode)->i_data_sem));
490
	if (ext4_test_inode_flag(inode, EXT4_INODE_EXTENTS)) {
491 492
		retval = ext4_ext_map_blocks(handle, inode, map, flags &
					     EXT4_GET_BLOCKS_KEEP_SIZE);
493
	} else {
494 495
		retval = ext4_ind_map_blocks(handle, inode, map, flags &
					     EXT4_GET_BLOCKS_KEEP_SIZE);
496
	}
497
	up_read((&EXT4_I(inode)->i_data_sem));
498

499
	if (retval > 0 && map->m_flags & EXT4_MAP_MAPPED) {
500
		int ret = check_block_validity(inode, map);
501 502 503 504
		if (ret != 0)
			return ret;
	}

505
	/* If it is only a block(s) look up */
506
	if ((flags & EXT4_GET_BLOCKS_CREATE) == 0)
507 508 509 510 511 512
		return retval;

	/*
	 * Returns if the blocks have already allocated
	 *
	 * Note that if blocks have been preallocated
513
	 * ext4_ext_get_block() returns the create = 0
514 515
	 * with buffer head unmapped.
	 */
516
	if (retval > 0 && map->m_flags & EXT4_MAP_MAPPED)
517 518
		return retval;

519 520 521 522 523 524 525 526 527 528
	/*
	 * When we call get_blocks without the create flag, the
	 * BH_Unwritten flag could have gotten set if the blocks
	 * requested were part of a uninitialized extent.  We need to
	 * clear this flag now that we are committed to convert all or
	 * part of the uninitialized extent to be an initialized
	 * extent.  This is because we need to avoid the combination
	 * of BH_Unwritten and BH_Mapped flags being simultaneously
	 * set on the buffer_head.
	 */
529
	map->m_flags &= ~EXT4_MAP_UNWRITTEN;
530

531
	/*
532 533 534 535
	 * New blocks allocate and/or writing to uninitialized extent
	 * will possibly result in updating i_data, so we take
	 * the write lock of i_data_sem, and call get_blocks()
	 * with create == 1 flag.
536 537
	 */
	down_write((&EXT4_I(inode)->i_data_sem));
538 539 540 541 542 543 544

	/*
	 * if the caller is from delayed allocation writeout path
	 * we have already reserved fs blocks for allocation
	 * let the underlying get_block() function know to
	 * avoid double accounting
	 */
545
	if (flags & EXT4_GET_BLOCKS_DELALLOC_RESERVE)
546
		ext4_set_inode_state(inode, EXT4_STATE_DELALLOC_RESERVED);
547 548 549 550
	/*
	 * We need to check for EXT4 here because migrate
	 * could have changed the inode type in between
	 */
551
	if (ext4_test_inode_flag(inode, EXT4_INODE_EXTENTS)) {
552
		retval = ext4_ext_map_blocks(handle, inode, map, flags);
553
	} else {
554
		retval = ext4_ind_map_blocks(handle, inode, map, flags);
555

556
		if (retval > 0 && map->m_flags & EXT4_MAP_NEW) {
557 558 559 560 561
			/*
			 * We allocated new blocks which will result in
			 * i_data's format changing.  Force the migrate
			 * to fail by clearing migrate flags
			 */
562
			ext4_clear_inode_state(inode, EXT4_STATE_EXT_MIGRATE);
563
		}
564

565 566 567 568 569 570 571
		/*
		 * Update reserved blocks/metadata blocks after successful
		 * block allocation which had been deferred till now. We don't
		 * support fallocate for non extent files. So we can update
		 * reserve space here.
		 */
		if ((retval > 0) &&
572
			(flags & EXT4_GET_BLOCKS_DELALLOC_RESERVE))
573 574
			ext4_da_update_reserve_space(inode, retval, 1);
	}
575
	if (flags & EXT4_GET_BLOCKS_DELALLOC_RESERVE) {
576
		ext4_clear_inode_state(inode, EXT4_STATE_DELALLOC_RESERVED);
577

578 579 580 581 582 583 584 585
		/* If we have successfully mapped the delayed allocated blocks,
		 * set the BH_Da_Mapped bit on them. Its important to do this
		 * under the protection of i_data_sem.
		 */
		if (retval > 0 && map->m_flags & EXT4_MAP_MAPPED)
			set_buffers_da_mapped(inode, map);
	}

586
	up_write((&EXT4_I(inode)->i_data_sem));
587
	if (retval > 0 && map->m_flags & EXT4_MAP_MAPPED) {
588
		int ret = check_block_validity(inode, map);
589 590 591
		if (ret != 0)
			return ret;
	}
592 593 594
	return retval;
}

595 596 597
/* Maximum number of blocks we map for direct IO at once. */
#define DIO_MAX_BLOCKS 4096

598 599
static int _ext4_get_block(struct inode *inode, sector_t iblock,
			   struct buffer_head *bh, int flags)
600
{
601
	handle_t *handle = ext4_journal_current_handle();
602
	struct ext4_map_blocks map;
Jan Kara's avatar
Jan Kara committed
603
	int ret = 0, started = 0;
604
	int dio_credits;
605

606 607 608 609
	map.m_lblk = iblock;
	map.m_len = bh->b_size >> inode->i_blkbits;

	if (flags && !handle) {
Jan Kara's avatar
Jan Kara committed
610
		/* Direct IO write... */
611 612 613
		if (map.m_len > DIO_MAX_BLOCKS)
			map.m_len = DIO_MAX_BLOCKS;
		dio_credits = ext4_chunk_trans_blocks(inode, map.m_len);
614
		handle = ext4_journal_start(inode, dio_credits);
Jan Kara's avatar
Jan Kara committed
615
		if (IS_ERR(handle)) {
616
			ret = PTR_ERR(handle);
617
			return ret;
618
		}
Jan Kara's avatar
Jan Kara committed
619
		started = 1;
620 621
	}

622
	ret = ext4_map_blocks(handle, inode, &map, flags);
Jan Kara's avatar
Jan Kara committed
623
	if (ret > 0) {
624 625 626
		map_bh(bh, inode->i_sb, map.m_pblk);
		bh->b_state = (bh->b_state & ~EXT4_MAP_FLAGS) | map.m_flags;
		bh->b_size = inode->i_sb->s_blocksize * map.m_len;
Jan Kara's avatar
Jan Kara committed
627
		ret = 0;
628
	}
Jan Kara's avatar
Jan Kara committed
629 630
	if (started)
		ext4_journal_stop(handle);
631 632 633
	return ret;
}

634 635 636 637 638 639 640
int ext4_get_block(struct inode *inode, sector_t iblock,
		   struct buffer_head *bh, int create)
{
	return _ext4_get_block(inode, iblock, bh,
			       create ? EXT4_GET_BLOCKS_CREATE : 0);
}

641 642 643
/*
 * `handle' can be NULL if create is zero
 */
644
struct buffer_head *ext4_getblk(handle_t *handle, struct inode *inode,
645
				ext4_lblk_t block, int create, int *errp)
646
{
647 648
	struct ext4_map_blocks map;
	struct buffer_head *bh;
649 650 651 652
	int fatal = 0, err;

	J_ASSERT(handle != NULL || create == 0);

653 654 655 656
	map.m_lblk = block;
	map.m_len = 1;
	err = ext4_map_blocks(handle, inode, &map,
			      create ? EXT4_GET_BLOCKS_CREATE : 0);
657

658 659 660 661 662 663 664 665 666 667
	if (err < 0)
		*errp = err;
	if (err <= 0)
		return NULL;
	*errp = 0;

	bh = sb_getblk(inode->i_sb, map.m_pblk);
	if (!bh) {
		*errp = -EIO;
		return NULL;
668
	}
669 670 671
	if (map.m_flags & EXT4_MAP_NEW) {
		J_ASSERT(create != 0);
		J_ASSERT(handle != NULL);
672

673 674 675 676 677 678 679 680 681 682 683 684 685
		/*
		 * Now that we do not always journal data, we should
		 * keep in mind whether this should always journal the
		 * new buffer as metadata.  For now, regular file
		 * writes use ext4_get_block instead, so it's not a
		 * problem.
		 */
		lock_buffer(bh);
		BUFFER_TRACE(bh, "call get_create_access");
		fatal = ext4_journal_get_create_access(handle, bh);
		if (!fatal && !buffer_uptodate(bh)) {
			memset(bh->b_data, 0, inode->i_sb->s_blocksize);
			set_buffer_uptodate(bh);
686
		}
687 688 689 690 691 692 693
		unlock_buffer(bh);
		BUFFER_TRACE(bh, "call ext4_handle_dirty_metadata");
		err = ext4_handle_dirty_metadata(handle, inode, bh);
		if (!fatal)
			fatal = err;
	} else {
		BUFFER_TRACE(bh, "not a new buffer");
694
	}
695 696 697 698 699 700
	if (fatal) {
		*errp = fatal;
		brelse(bh);
		bh = NULL;
	}
	return bh;
701 702
}

703
struct buffer_head *ext4_bread(handle_t *handle, struct inode *inode,
704
			       ext4_lblk_t block, int create, int *err)
705
{
706
	struct buffer_head *bh;
707

708
	bh = ext4_getblk(handle, inode, block, create, err);
709 710 711 712
	if (!bh)
		return bh;
	if (buffer_uptodate(bh))
		return bh;
713
	ll_rw_block(READ | REQ_META | REQ_PRIO, 1, &bh);
714 715 716 717 718 719 720 721
	wait_on_buffer(bh);
	if (buffer_uptodate(bh))
		return bh;
	put_bh(bh);
	*err = -EIO;
	return NULL;
}

722 723 724 725 726 727 728
static int walk_page_buffers(handle_t *handle,
			     struct buffer_head *head,
			     unsigned from,
			     unsigned to,
			     int *partial,
			     int (*fn)(handle_t *handle,
				       struct buffer_head *bh))
729 730 731 732 733 734 735
{
	struct buffer_head *bh;
	unsigned block_start, block_end;
	unsigned blocksize = head->b_size;
	int err, ret = 0;
	struct buffer_head *next;

736 737
	for (bh = head, block_start = 0;
	     ret == 0 && (bh != head || !block_start);
738
	     block_start = block_end, bh = next) {
739 740 741 742 743 744 745 746 747 748 749 750 751 752 753 754 755
		next = bh->b_this_page;
		block_end = block_start + blocksize;
		if (block_end <= from || block_start >= to) {
			if (partial && !buffer_uptodate(bh))
				*partial = 1;
			continue;
		}
		err = (*fn)(handle, bh);
		if (!ret)
			ret = err;
	}
	return ret;
}

/*
 * To preserve ordering, it is essential that the hole instantiation and
 * the data write be encapsulated in a single transaction.  We cannot
756
 * close off a transaction and start a new one between the ext4_get_block()
757
 * and the commit_write().  So doing the jbd2_journal_start at the start of
758 759
 * prepare_write() is the right place.
 *
760 761
 * Also, this function can nest inside ext4_writepage() ->
 * block_write_full_page(). In that case, we *know* that ext4_writepage()
762 763 764 765
 * has generated enough buffer credits to do the whole page.  So we won't
 * block on the journal in that case, which is good, because the caller may
 * be PF_MEMALLOC.
 *
766
 * By accident, ext4 can be reentered when a transaction is open via
767 768 769 770 771 772
 * quota file writes.  If we were to commit the transaction while thus
 * reentered, there can be a deadlock - we would be holding a quota
 * lock, and the commit would never complete if another thread had a
 * transaction open and was blocking on the quota lock - a ranking
 * violation.
 *
773
 * So what we do is to rely on the fact that jbd2_journal_stop/journal_start
774 775 776 777 778
 * will _not_ run commit under these circumstances because handle->h_ref
 * is elevated.  We'll still have enough credits for the tiny quotafile
 * write.
 */
static int do_journal_get_write_access(handle_t *handle,
779
				       struct buffer_head *bh)
780
{
781 782 783
	int dirty = buffer_dirty(bh);
	int ret;

784 785
	if (!buffer_mapped(bh) || buffer_freed(bh))
		return 0;
786
	/*
787
	 * __block_write_begin() could have dirtied some buffers. Clean
788 789
	 * the dirty bit as jbd2_journal_get_write_access() could complain
	 * otherwise about fs integrity issues. Setting of the dirty bit
790
	 * by __block_write_begin() isn't a real problem here as we clear
791 792 793 794 795 796 797 798 799
	 * the bit before releasing a page lock and thus writeback cannot
	 * ever write the buffer.
	 */
	if (dirty)
		clear_buffer_dirty(bh);
	ret = ext4_journal_get_write_access(handle, bh);
	if (!ret && dirty)
		ret = ext4_handle_dirty_metadata(handle, NULL, bh);
	return ret;
800 801
}

802 803
static int ext4_get_block_write(struct inode *inode, sector_t iblock,
		   struct buffer_head *bh_result, int create);
Nick Piggin's avatar
Nick Piggin committed
804
static int ext4_write_begin(struct file *file, struct address_space *mapping,
805 806
			    loff_t pos, unsigned len, unsigned flags,
			    struct page **pagep, void **fsdata)
807
{
808
	struct inode *inode = mapping->host;
809
	int ret, needed_blocks;
810 811
	handle_t *handle;
	int retries = 0;
812
	struct page *page;
813
	pgoff_t index;
814
	unsigned from, to;
Nick Piggin's avatar
Nick Piggin committed
815

816
	trace_ext4_write_begin(inode, pos, len, flags);
817 818 819 820 821
	/*
	 * Reserve one block more for addition to orphan list in case
	 * we allocate blocks but write fails for some reason
	 */
	needed_blocks = ext4_writepage_trans_blocks(inode) + 1;
822
	index = pos >> PAGE_CACHE_SHIFT;
823 824
	from = pos & (PAGE_CACHE_SIZE - 1);
	to = from + len;
825 826

retry:
827 828 829 830
	handle = ext4_journal_start(inode, needed_blocks);
	if (IS_ERR(handle)) {
		ret = PTR_ERR(handle);
		goto out;
831
	}
832

833 834 835 836
	/* We cannot recurse into the filesystem as the transaction is already
	 * started */
	flags |= AOP_FLAG_NOFS;

837
	page = grab_cache_page_write_begin(mapping, index, flags);
838 839 840 841 842 843 844
	if (!page) {
		ext4_journal_stop(handle);
		ret = -ENOMEM;
		goto out;
	}
	*pagep = page;

845
	if (ext4_should_dioread_nolock(inode))
846
		ret = __block_write_begin(page, pos, len, ext4_get_block_write);
847
	else
848
		ret = __block_write_begin(page, pos, len, ext4_get_block);
Nick Piggin's avatar
Nick Piggin committed
849 850

	if (!ret && ext4_should_journal_data(inode)) {
851 852 853
		ret = walk_page_buffers(handle, page_buffers(page),
				from, to, NULL, do_journal_get_write_access);
	}
Nick Piggin's avatar
Nick Piggin committed
854 855

	if (ret) {
856 857
		unlock_page(page);
		page_cache_release(page);
858
		/*
859
		 * __block_write_begin may have instantiated a few blocks
860 861
		 * outside i_size.  Trim these off again. Don't need
		 * i_size_read because we hold i_mutex.
862 863 864
		 *
		 * Add inode to orphan list in case we crash before
		 * truncate finishes
865
		 */
866
		if (pos + len > inode->i_size && ext4_can_truncate(inode))
867 868 869 870
			ext4_orphan_add(handle, inode);

		ext4_journal_stop(handle);
		if (pos + len > inode->i_size) {
871
			ext4_truncate_failed_write(inode);
872
			/*
873
			 * If truncate failed early the inode might
874 875 876 877 878 879 880
			 * still be on the orphan list; we need to
			 * make sure the inode is removed from the
			 * orphan list in that case.
			 */
			if (inode->i_nlink)
				ext4_orphan_del(NULL, inode);
		}
Nick Piggin's avatar
Nick Piggin committed
881 882
	}

883
	if (ret == -ENOSPC && ext4_should_retry_alloc(inode->i_sb, &retries))
884
		goto retry;
885
out:
886 887 888
	return ret;
}

Nick Piggin's avatar
Nick Piggin committed
889 890
/* For write_end() in data=journal mode */
static int write_end_fn(handle_t *handle, struct buffer_head *bh)
891 892 893 894
{
	if (!buffer_mapped(bh) || buffer_freed(bh))
		return 0;
	set_buffer_uptodate(bh);
895
	return ext4_handle_dirty_metadata(handle, NULL, bh);
896 897
}

898
static int ext4_generic_write_end(struct file *file,
899 900 901
				  struct address_space *mapping,
				  loff_t pos, unsigned len, unsigned copied,
				  struct page *page, void *fsdata)
902 903 904 905 906 907 908 909 910 911 912 913 914 915 916 917 918 919 920 921 922 923 924 925 926 927 928 929 930 931 932 933 934 935 936 937 938 939 940 941 942 943
{
	int i_size_changed = 0;
	struct inode *inode = mapping->host;
	handle_t *handle = ext4_journal_current_handle();

	copied = block_write_end(file, mapping, pos, len, copied, page, fsdata);

	/*
	 * No need to use i_size_read() here, the i_size
	 * cannot change under us because we hold i_mutex.
	 *
	 * But it's important to update i_size while still holding page lock:
	 * page writeout could otherwise come in and zero beyond i_size.
	 */
	if (pos + copied > inode->i_size) {
		i_size_write(inode, pos + copied);
		i_size_changed = 1;
	}

	if (pos + copied >  EXT4_I(inode)->i_disksize) {
		/* We need to mark inode dirty even if
		 * new_i_size is less that inode->i_size
		 * bu greater than i_disksize.(hint delalloc)
		 */
		ext4_update_i_disksize(inode, (pos + copied));
		i_size_changed = 1;
	}
	unlock_page(page);
	page_cache_release(page);

	/*
	 * Don't mark the inode dirty under page lock. First, it unnecessarily
	 * makes the holding time of page lock longer. Second, it forces lock
	 * ordering of page lock and transaction start for journaling
	 * filesystems.
	 */
	if (i_size_changed)
		ext4_mark_inode_dirty(handle, inode);

	return copied;
}

944 945 946 947
/*
 * We need to pick up the new inode size which generic_commit_write gave us
 * `file' can be NULL - eg, when called from page_symlink().
 *
948
 * ext4 never places buffers on inode->i_mapping->private_list.  metadata
949 950
 * buffers are managed internally.
 */
Nick Piggin's avatar
Nick Piggin committed
951
static int ext4_ordered_write_end(struct file *file,
952 953 954
				  struct address_space *mapping,
				  loff_t pos, unsigned len, unsigned copied,
				  struct page *page, void *fsdata)
955
{
956
	handle_t *handle = ext4_journal_current_handle();
957
	struct inode *inode = mapping->host;
958 959
	int ret = 0, ret2;

960
	trace_ext4_ordered_write_end(inode, pos, len, copied);
961
	ret = ext4_jbd2_file_inode(handle, inode);
962 963

	if (ret == 0) {
964
		ret2 = ext4_generic_write_end(file, mapping, pos, len, copied,
Nick Piggin's avatar
Nick Piggin committed
965
							page, fsdata);
966
		copied = ret2;
967
		if (pos + len > inode->i_size && ext4_can_truncate(inode))
968 969 970 971 972
			/* if we have allocated more blocks and copied
			 * less. We will have blocks allocated outside
			 * inode->i_size. So truncate them
			 */
			ext4_orphan_add(handle, inode);
973 974
		if (ret2 < 0)
			ret = ret2;
975 976 977
	} else {
		unlock_page(page);
		page_cache_release(page);
978
	}
979

980
	ret2 = ext4_journal_stop(handle);
981 982
	if (!ret)
		ret = ret2;
Nick Piggin's avatar
Nick Piggin committed
983

984
	if (pos + len > inode->i_size) {
985
		ext4_truncate_failed_write(inode);
986
		/*
987
		 * If truncate failed early the inode might still be
988 989 990 991 992 993 994 995
		 * on the orphan list; we need to make sure the inode
		 * is removed from the orphan list in that case.
		 */
		if (inode->i_nlink)
			ext4_orphan_del(NULL, inode);
	}


Nick Piggin's avatar
Nick Piggin committed
996
	return ret ? ret : copied;
997 998
}

Nick Piggin's avatar
Nick Piggin committed
999
static int ext4_writeback_write_end(struct file *file,
1000 1001 1002
				    struct address_space *mapping,
				    loff_t pos, unsigned len, unsigned copied,
				    struct page *page, void *fsdata)
1003
{
1004
	handle_t *handle = ext4_journal_current_handle();
1005
	struct inode *inode = mapping->host;
1006 1007
	int ret = 0, ret2;

1008
	trace_ext4_writeback_write_end(inode, pos, len, copied);
1009
	ret2 = ext4_generic_write_end(file, mapping, pos, len, copied,
Nick Piggin's avatar
Nick Piggin committed
1010
							page, fsdata);
1011
	copied = ret2;
1012
	if (pos + len > inode->i_size && ext4_can_truncate(inode))
1013 1014 1015 1016 1017 1018
		/* if we have allocated more blocks and copied
		 * less. We will have blocks allocated outside
		 * inode->i_size. So truncate them
		 */
		ext4_orphan_add(handle, inode);

1019 1020
	if (ret2 < 0)
		ret = ret2;
1021

1022
	ret2 = ext4_journal_stop(handle);
1023 1024
	if (!ret)
		ret = ret2;
Nick Piggin's avatar
Nick Piggin committed
1025

1026
	if (pos + len > inode->i_size) {
1027
		ext4_truncate_failed_write(inode);
1028
		/*
1029
		 * If truncate failed early the inode might still be
1030 1031 1032 1033 1034 1035 1036
		 * on the orphan list; we need to make sure the inode
		 * is removed from the orphan list in that case.
		 */
		if (inode->i_nlink)
			ext4_orphan_del(NULL, inode);
	}

Nick Piggin's avatar
Nick Piggin committed
1037
	return ret ? ret : copied;
1038 1039
}

Nick Piggin's avatar
Nick Piggin committed
1040
static int ext4_journalled_write_end(struct file *file,
1041 1042 1043
				     struct address_space *mapping,
				     loff_t pos, unsigned len, unsigned copied,
				     struct page *page, void *fsdata)
1044
{
1045
	handle_t *handle = ext4_journal_current_handle();
Nick Piggin's avatar
Nick Piggin committed
1046
	struct inode *inode = mapping->host;
1047 1048
	int ret = 0, ret2;
	int partial = 0;
Nick Piggin's avatar
Nick Piggin committed
1049
	unsigned from, to;
1050
	loff_t new_i_size;
1051

1052
	trace_ext4_journalled_write_end(inode, pos, len, copied);
Nick Piggin's avatar
Nick Piggin committed
1053 1054 1055
	from = pos & (PAGE_CACHE_SIZE - 1);
	to = from + len;

1056 1057
	BUG_ON(!ext4_handle_valid(handle));

Nick Piggin's avatar
Nick Piggin committed
1058 1059 1060 1061 1062
	if (copied < len) {
		if (!PageUptodate(page))
			copied = 0;
		page_zero_new_buffers(page, from+copied, to);
	}
1063 1064

	ret = walk_page_buffers(handle, page_buffers(page), from,
Nick Piggin's avatar
Nick Piggin committed
1065
				to, &partial, write_end_fn);
1066 1067
	if (!partial)
		SetPageUptodate(page);
1068 1069
	new_i_size = pos + copied;
	if (new_i_size > inode->i_size)
Nick Piggin's avatar
Nick Piggin committed
1070
		i_size_write(inode, pos+copied);
1071
	ext4_set_inode_state(inode, EXT4_STATE_JDATA);
1072
	EXT4_I(inode)->i_datasync_tid = handle->h_transaction->t_tid;
1073 1074
	if (new_i_size > EXT4_I(inode)->i_disksize) {
		ext4_update_i_disksize(inode, new_i_size);
1075
		ret2 = ext4_mark_inode_dirty(handle, inode);
1076 1077 1078
		if (!ret)
			ret = ret2;
	}
Nick Piggin's avatar
Nick Piggin committed
1079

1080
	unlock_page(page);
1081
	page_cache_release(page);
1082
	if (pos + len > inode->i_size && ext4_can_truncate(inode))
1083 1084 1085 1086 1087 1088
		/* if we have allocated more blocks and copied
		 * less. We will have blocks allocated outside
		 * inode->i_size. So truncate them
		 */
		ext4_orphan_add(handle, inode);

1089
	ret2 = ext4_journal_stop(handle);
1090 1091
	if (!ret)
		ret = ret2;
1092
	if (pos + len > inode->i_size) {
1093
		ext4_truncate_failed_write(inode);
1094
		/*
1095
		 * If truncate failed early the inode might still be
1096 1097 1098 1099 1100 1101
		 * on the orphan list; we need to make sure the inode
		 * is removed from the orphan list in that case.
		 */
		if (inode->i_nlink)
			ext4_orphan_del(NULL, inode);
	}
Nick Piggin's avatar
Nick Piggin committed
1102 1103

	return ret ? ret : copied;
1104
}
1105

1106
/*
1107
 * Reserve a single cluster located at lblock
1108
 */
1109
static int ext4_da_reserve_space(struct inode *inode, ext4_lblk_t lblock)
1110
{
1111
	int retries = 0;
1112
	struct ext4_sb_info *sbi = EXT4_SB(inode->i_sb);
1113
	struct ext4_inode_info *ei = EXT4_I(inode);
1114
	unsigned int md_needed;
1115
	int ret;
1116 1117 1118 1119 1120 1121 1122 1123 1124 1125 1126
	ext4_lblk_t save_last_lblock;
	int save_len;

	/*
	 * We will charge metadata quota at writeout time; this saves
	 * us from metadata over-estimation, though we may go over by
	 * a small amount in the end.  Here we just reserve for data.
	 */
	ret = dquot_reserve_block(inode, EXT4_C2B(sbi, 1));
	if (ret)
		return ret;
1127 1128 1129 1130 1131 1132

	/*
	 * recalculate the amount of metadata blocks to reserve
	 * in order to allocate nrblocks
	 * worse case is one extent per block
	 */
1133
repeat:
1134
	spin_lock(&ei->i_block_reservation_lock);
1135 1136 1137 1138 1139 1140
	/*
	 * ext4_calc_metadata_amount() has side effects, which we have
	 * to be prepared undo if we fail to claim space.
	 */
	save_len = ei->i_da_metadata_calc_len;
	save_last_lblock = ei->i_da_metadata_calc_last_lblock;
1141 1142
	md_needed = EXT4_NUM_B2C(sbi,
				 ext4_calc_metadata_amount(inode, lblock));
1143
	trace_ext4_da_reserve_space(inode, md_needed);
1144

1145 1146 1147 1148
	/*
	 * We do still charge estimated metadata to the sb though;
	 * we cannot afford to run out of free blocks.
	 */
1149
	if (ext4_claim_free_clusters(sbi, md_needed + 1, 0)) {
1150 1151 1152
		ei->i_da_metadata_calc_len = save_len;
		ei->i_da_metadata_calc_last_lblock = save_last_lblock;
		spin_unlock(&ei->i_block_reservation_lock);
1153 1154 1155 1156
		if (ext4_should_retry_alloc(inode->i_sb, &retries)) {
			yield();
			goto repeat;
		}
1157
		dquot_release_reservation_block(inode, EXT4_C2B(sbi, 1));
1158 1159
		return -ENOSPC;
	}
1160
	ei->i_reserved_data_blocks++;
1161 1162
	ei->i_reserved_meta_blocks += md_needed;
	spin_unlock(&ei->i_block_reservation_lock);
1163

1164 1165 1166
	return 0;       /* success */
}

1167
static void ext4_da_release_space(struct inode *inode, int to_free)
1168 1169
{
	struct ext4_sb_info *sbi = EXT4_SB(inode->i_sb);
1170
	struct ext4_inode_info *ei = EXT4_I(inode);
1171

1172 1173 1174
	if (!to_free)
		return;		/* Nothing to release, exit */

1175
	spin_lock(&EXT4_I(inode)->i_block_reservation_lock);
1176

Li Zefan's avatar
Li Zefan committed
1177
	trace_ext4_da_release_space(inode, to_free);
1178
	if (unlikely(to_free > ei->i_reserved_data_blocks)) {
1179
		/*
1180 1181 1182 1183
		 * if there aren't enough reserved blocks, then the
		 * counter is messed up somewhere.  Since this
		 * function is called from invalidate page, it's
		 * harmless to return without any action.
1184
		 */
1185 1186
		ext4_msg(inode->i_sb, KERN_NOTICE, "ext4_da_release_space: "
			 "ino %lu, to_free %d with only %d reserved "
1187
			 "data blocks", inode->i_ino, to_free,
1188 1189 1190
			 ei->i_reserved_data_blocks);
		WARN_ON(1);
		to_free = ei->i_reserved_data_blocks;
1191
	}
1192
	ei->i_reserved_data_blocks -= to_free;
1193

1194 1195 1196 1197 1198
	if (ei->i_reserved_data_blocks == 0) {
		/*
		 * We can release all of the reserved metadata blocks
		 * only when we have written all of the delayed
		 * allocation blocks.
1199 1200
		 * Note that in case of bigalloc, i_reserved_meta_blocks,
		 * i_reserved_data_blocks, etc. refer to number of clusters.
1201
		 */
1202
		percpu_counter_sub(&sbi->s_dirtyclusters_counter,
1203
				   ei->i_reserved_meta_blocks);
1204
		ei->i_reserved_meta_blocks = 0;
1205
		ei->i_da_metadata_calc_len = 0;
1206
	}
1207

1208
	/* update fs dirty data blocks counter */
1209
	percpu_counter_sub(&sbi->s_dirtyclusters_counter, to_free);
1210 1211

	spin_unlock(&EXT4_I(inode)->i_block_reservation_lock);
1212

1213
	dquot_release_reservation_block(inode, EXT4_C2B(sbi, to_free));
1214 1215 1216
}

static void ext4_da_page_release_reservation(struct page *page,
1217
					     unsigned long offset)
1218 1219 1220 1221
{
	int to_release = 0;
	struct buffer_head *head, *bh;
	unsigned int curr_off = 0;
1222 1223 1224
	struct inode *inode = page->mapping->host;
	struct ext4_sb_info *sbi = EXT4_SB(inode->i_sb);
	int num_clusters;
1225 1226 1227 1228 1229 1230 1231 1232 1233

	head = page_buffers(page);
	bh = head;
	do {
		unsigned int next_off = curr_off + bh->b_size;

		if ((offset <= curr_off) && (buffer_delay(bh))) {
			to_release++;
			clear_buffer_delay(bh);
1234
			clear_buffer_da_mapped(bh);
1235 1236 1237
		}
		curr_off = next_off;
	} while ((bh = bh->b_this_page) != head);
1238 1239 1240 1241 1242 1243 1244 1245 1246 1247 1248 1249 1250 1251

	/* If we have released all the blocks belonging to a cluster, then we
	 * need to release the reserved space for that cluster. */
	num_clusters = EXT4_NUM_B2C(sbi, to_release);
	while (num_clusters > 0) {
		ext4_fsblk_t lblk;
		lblk = (page->index << (PAGE_CACHE_SHIFT - inode->i_blkbits)) +
			((num_clusters - 1) << sbi->s_cluster_bits);
		if (sbi->s_cluster_ratio == 1 ||
		    !ext4_find_delalloc_cluster(inode, lblk, 1))
			ext4_da_release_space(inode, 1);

		num_clusters--;
	}
1252
}
1253

1254 1255 1256 1257 1258 1259
/*
 * Delayed allocation stuff
 */

/*
 * mpage_da_submit_io - walks through extent of pages and try to write
1260
 * them with writepage() call back
1261 1262 1263 1264 1265 1266 1267 1268 1269 1270
 *
 * @mpd->inode: inode
 * @mpd->first_page: first page of the extent
 * @mpd->next_page: page after the last page of the extent
 *
 * By the time mpage_da_submit_io() is called we expect all blocks
 * to be allocated. this may be wrong if allocation failed.
 *
 * As pages are already locked by write_cache_pages(), we can't use it
 */
1271 1272
static int mpage_da_submit_io(struct mpage_da_data *mpd,
			      struct ext4_map_blocks *map)
1273
{
1274 1275 1276 1277 1278
	struct pagevec pvec;
	unsigned long index, end;
	int ret = 0, err, nr_pages, i;
	struct inode *inode = mpd->inode;
	struct address_space *mapping = inode->i_mapping;
1279
	loff_t size = i_size_read(inode);
1280 1281
	unsigned int len, block_start;
	struct buffer_head *bh, *page_bufs = NULL;
1282
	int journal_data = ext4_should_journal_data(inode);
1283
	sector_t pblock = 0, cur_logical = 0;
1284
	struct ext4_io_submit io_submit;
1285 1286

	BUG_ON(mpd->next_page <= mpd->first_page);
1287
	memset(&io_submit, 0, sizeof(io_submit));
1288 1289 1290
	/*
	 * We need to start from the first_page to the next_page - 1
	 * to make sure we also write the mapped dirty buffer_heads.
1291
	 * If we look at mpd->b_blocknr we would only be looking
1292 1293
	 * at the currently mapped buffer_heads.
	 */
1294 1295 1296
	index = mpd->first_page;
	end = mpd->next_page - 1;

1297
	pagevec_init(&pvec, 0);
1298
	while (index <= end) {
1299
		nr_pages = pagevec_lookup(&pvec, mapping, index, PAGEVEC_SIZE);
1300 1301 1302
		if (nr_pages == 0)
			break;
		for (i = 0; i < nr_pages; i++) {
1303
			int commit_write = 0, skip_page = 0;
1304 1305
			struct page *page = pvec.pages[i];

1306 1307 1308
			index = page->index;
			if (index > end)
				break;
1309 1310 1311 1312 1313

			if (index == size >> PAGE_CACHE_SHIFT)
				len = size & ~PAGE_CACHE_MASK;
			else
				len = PAGE_CACHE_SIZE;
1314 1315 1316 1317 1318 1319
			if (map) {
				cur_logical = index << (PAGE_CACHE_SHIFT -
							inode->i_blkbits);
				pblock = map->m_pblk + (cur_logical -
							map->m_lblk);
			}
1320 1321 1322 1323 1324
			index++;

			BUG_ON(!PageLocked(page));
			BUG_ON(PageWriteback(page));

1325
			/*
1326 1327
			 * If the page does not have buffers (for
			 * whatever reason), try to create them using
1328
			 * __block_write_begin.  If this fails,
1329
			 * skip the page and move on.
1330
			 */
1331
			if (!page_has_buffers(page)) {
1332
				if (__block_write_begin(page, 0, len,
1333
						noalloc_get_block_write)) {
1334
				skip_page:
1335 1336 1337 1338 1339
					unlock_page(page);
					continue;
				}
				commit_write = 1;
			}
1340

1341 1342
			bh = page_bufs = page_buffers(page);
			block_start = 0;
1343
			do {
1344
				if (!bh)
1345
					goto skip_page;
1346 1347 1348
				if (map && (cur_logical >= map->m_lblk) &&
				    (cur_logical <= (map->m_lblk +
						     (map->m_len - 1)))) {
1349 1350 1351 1352
					if (buffer_delay(bh)) {
						clear_buffer_delay(bh);
						bh->b_blocknr = pblock;
					}
1353 1354
					if (buffer_da_mapped(bh))
						clear_buffer_da_mapped(bh);
1355 1356 1357 1358 1359 1360 1361
					if (buffer_unwritten(bh) ||
					    buffer_mapped(bh))
						BUG_ON(bh->b_blocknr != pblock);
					if (map->m_flags & EXT4_MAP_UNINIT)
						set_buffer_uninit(bh);
					clear_buffer_unwritten(bh);
				}
1362

1363 1364 1365 1366 1367
				/*
				 * skip page if block allocation undone and
				 * block is dirty
				 */
				if (ext4_bh_delay_or_unwritten(NULL, bh))
1368
					skip_page = 1;
1369 1370
				bh = bh->b_this_page;
				block_start += bh->b_size;
1371 1372
				cur_logical++;
				pblock++;
1373 1374
			} while (bh != page_bufs);

1375 1376
			if (skip_page)
				goto skip_page;
1377 1378 1379 1380 1381

			if (commit_write)
				/* mark the buffer_heads as dirty & uptodate */
				block_commit_write(page, 0, len);

1382
			clear_page_dirty_for_io(page);
1383 1384 1385 1386 1387 1388
			/*
			 * Delalloc doesn't support data journalling,
			 * but eventually maybe we'll lift this
			 * restriction.
			 */
			if (unlikely(journal_data && PageChecked(page)))
1389
				err = __ext4_journalled_writepage(page, len);
1390
			else if (test_opt(inode->i_sb, MBLK_IO_SUBMIT))
1391 1392
				err = ext4_bio_write_page(&io_submit, page,
							  len, mpd->wbc);
1393 1394 1395 1396 1397 1398
			else if (buffer_uninit(page_bufs)) {
				ext4_set_bh_endio(page_bufs, inode);
				err = block_write_full_page_endio(page,
					noalloc_get_block_write,
					mpd->wbc, ext4_end_io_buffer_write);
			} else
1399 1400
				err = block_write_full_page(page,
					noalloc_get_block_write, mpd->wbc);
1401 1402

			if (!err)
1403
				mpd->pages_written++;
1404 1405 1406 1407 1408 1409 1410 1411 1412
			/*
			 * In error case, we have to continue because
			 * remaining pages are still locked
			 */
			if (ret == 0)
				ret = err;
		}
		pagevec_release(&pvec);
	}
1413
	ext4_io_submit(&io_submit);
1414 1415 1416
	return ret;
}

1417
static void ext4_da_block_invalidatepages(struct mpage_da_data *mpd)
1418 1419 1420 1421 1422 1423 1424
{
	int nr_pages, i;
	pgoff_t index, end;
	struct pagevec pvec;
	struct inode *inode = mpd->inode;
	struct address_space *mapping = inode->i_mapping;

1425 1426
	index = mpd->first_page;
	end   = mpd->next_page - 1;
1427 1428

	pagevec_init(&pvec, 0);
1429 1430 1431 1432 1433 1434
	while (index <= end) {
		nr_pages = pagevec_lookup(&pvec, mapping, index, PAGEVEC_SIZE);
		if (nr_pages == 0)
			break;
		for (i = 0; i < nr_pages; i++) {
			struct page *page = pvec.pages[i];
1435
			if (page->index > end)
1436 1437 1438 1439 1440 1441 1442
				break;
			BUG_ON(!PageLocked(page));
			BUG_ON(PageWriteback(page));
			block_invalidatepage(page, 0);
			ClearPageUptodate(page);
			unlock_page(page);
		}
1443 1444
		index = pvec.pages[nr_pages - 1]->index + 1;
		pagevec_release(&pvec);
1445 1446 1447 1448
	}
	return;
}

1449 1450 1451
static void ext4_print_free_blocks(struct inode *inode)
{
	struct ext4_sb_info *sbi = EXT4_SB(inode->i_sb);
1452 1453 1454
	struct super_block *sb = inode->i_sb;

	ext4_msg(sb, KERN_CRIT, "Total free blocks count %lld",
1455 1456
	       EXT4_C2B(EXT4_SB(inode->i_sb),
			ext4_count_free_clusters(inode->i_sb)));
1457 1458
	ext4_msg(sb, KERN_CRIT, "Free/Dirty block details");
	ext4_msg(sb, KERN_CRIT, "free_blocks=%lld",
1459 1460
	       (long long) EXT4_C2B(EXT4_SB(inode->i_sb),
		percpu_counter_sum(&sbi->s_freeclusters_counter)));
1461
	ext4_msg(sb, KERN_CRIT, "dirty_blocks=%lld",
1462 1463
	       (long long) EXT4_C2B(EXT4_SB(inode->i_sb),
		percpu_counter_sum(&sbi->s_dirtyclusters_counter)));
1464 1465 1466 1467
	ext4_msg(sb, KERN_CRIT, "Block reservation details");
	ext4_msg(sb, KERN_CRIT, "i_reserved_data_blocks=%u",
		 EXT4_I(inode)->i_reserved_data_blocks);
	ext4_msg(sb, KERN_CRIT, "i_reserved_meta_blocks=%u",
1468
	       EXT4_I(inode)->i_reserved_meta_blocks);
1469 1470 1471
	return;
}

1472
/*
1473 1474
 * mpage_da_map_and_submit - go through given space, map them
 *       if necessary, and then submit them for I/O
1475
 *
1476
 * @mpd - bh describing space
1477 1478 1479 1480
 *
 * The function skips space we know is already mapped to disk blocks.
 *
 */
1481
static void mpage_da_map_and_submit(struct mpage_da_data *mpd)
1482
{
1483
	int err, blks, get_blocks_flags;
1484
	struct ext4_map_blocks map, *mapp = NULL;
1485 1486 1487 1488
	sector_t next = mpd->b_blocknr;
	unsigned max_blocks = mpd->b_size >> mpd->inode->i_blkbits;
	loff_t disksize = EXT4_I(mpd->inode)->i_disksize;
	handle_t *handle = NULL;
1489 1490

	/*
1491 1492
	 * If the blocks are mapped already, or we couldn't accumulate
	 * any blocks, then proceed immediately to the submission stage.
1493
	 */
1494 1495 1496 1497 1498
	if ((mpd->b_size == 0) ||
	    ((mpd->b_state  & (1 << BH_Mapped)) &&
	     !(mpd->b_state & (1 << BH_Delay)) &&
	     !(mpd->b_state & (1 << BH_Unwritten))))
		goto submit_io;
1499 1500 1501 1502

	handle = ext4_journal_current_handle();
	BUG_ON(!handle);

1503
	/*
1504
	 * Call ext4_map_blocks() to allocate any delayed allocation
1505 1506 1507 1508 1509 1510 1511 1512
	 * blocks, or to convert an uninitialized extent to be
	 * initialized (in the case where we have written into
	 * one or more preallocated blocks).
	 *
	 * We pass in the magic EXT4_GET_BLOCKS_DELALLOC_RESERVE to
	 * indicate that we are on the delayed allocation path.  This
	 * affects functions in many different parts of the allocation
	 * call path.  This flag exists primarily because we don't
1513
	 * want to change *many* call functions, so ext4_map_blocks()
1514
	 * will set the EXT4_STATE_DELALLOC_RESERVED flag once the
1515 1516 1517 1518 1519
	 * inode's allocation semaphore is taken.
	 *
	 * If the blocks in questions were delalloc blocks, set
	 * EXT4_GET_BLOCKS_DELALLOC_RESERVE so the delalloc accounting
	 * variables are updated after the blocks have been allocated.
1520
	 */
1521 1522
	map.m_lblk = next;
	map.m_len = max_blocks;
1523
	get_blocks_flags = EXT4_GET_BLOCKS_CREATE;
1524 1525
	if (ext4_should_dioread_nolock(mpd->inode))
		get_blocks_flags |= EXT4_GET_BLOCKS_IO_CREATE_EXT;
1526
	if (mpd->b_state & (1 << BH_Delay))
1527 1528
		get_blocks_flags |= EXT4_GET_BLOCKS_DELALLOC_RESERVE;

1529
	blks = ext4_map_blocks(handle, mpd->inode, &map, get_blocks_flags);
1530
	if (blks < 0) {
1531 1532
		struct super_block *sb = mpd->inode->i_sb;

1533
		err = blks;
1534
		/*
1535
		 * If get block returns EAGAIN or ENOSPC and there
1536 1537
		 * appears to be free blocks we will just let
		 * mpage_da_submit_io() unlock all of the pages.
1538 1539
		 */
		if (err == -EAGAIN)
1540
			goto submit_io;
1541

1542
		if (err == -ENOSPC && ext4_count_free_clusters(sb)) {
1543
			mpd->retval = err;
1544
			goto submit_io;
1545 1546
		}

1547
		/*
1548 1549 1550 1551 1552
		 * get block failure will cause us to loop in
		 * writepages, because a_ops->writepage won't be able
		 * to make progress. The page will be redirtied by
		 * writepage and writepages will again try to write
		 * the same.
1553
		 */
1554 1555 1556 1557 1558 1559 1560 1561 1562 1563 1564
		if (!(EXT4_SB(sb)->s_mount_flags & EXT4_MF_FS_ABORTED)) {
			ext4_msg(sb, KERN_CRIT,
				 "delayed block allocation failed for inode %lu "
				 "at logical offset %llu with max blocks %zd "
				 "with error %d", mpd->inode->i_ino,
				 (unsigned long long) next,
				 mpd->b_size >> mpd->inode->i_blkbits, err);
			ext4_msg(sb, KERN_CRIT,
				"This should not happen!! Data will be lost\n");
			if (err == -ENOSPC)
				ext4_print_free_blocks(mpd->inode);
1565
		}
1566
		/* invalidate all the pages */
1567
		ext4_da_block_invalidatepages(mpd);
1568 1569 1570

		/* Mark this page range as having been completed */
		mpd->io_done = 1;
1571
		return;
1572
	}
1573 1574
	BUG_ON(blks == 0);

1575
	mapp = &map;
1576 1577 1578
	if (map.m_flags & EXT4_MAP_NEW) {
		struct block_device *bdev = mpd->inode->i_sb->s_bdev;
		int i;
1579

1580 1581
		for (i = 0; i < map.m_len; i++)
			unmap_underlying_metadata(bdev, map.m_pblk + i);
1582

1583 1584
		if (ext4_should_order_data(mpd->inode)) {
			err = ext4_jbd2_file_inode(handle, mpd->inode);
1585
			if (err) {
1586
				/* Only if the journal is aborted */
1587 1588 1589
				mpd->retval = err;
				goto submit_io;
			}
1590
		}
1591 1592 1593
	}

	/*
1594
	 * Update on-disk size along with block allocation.
1595 1596 1597 1598 1599 1600
	 */
	disksize = ((loff_t) next + blks) << mpd->inode->i_blkbits;
	if (disksize > i_size_read(mpd->inode))
		disksize = i_size_read(mpd->inode);
	if (disksize > EXT4_I(mpd->inode)->i_disksize) {
		ext4_update_i_disksize(mpd->inode, disksize);
1601 1602 1603 1604 1605
		err = ext4_mark_inode_dirty(handle, mpd->inode);
		if (err)
			ext4_error(mpd->inode->i_sb,
				   "Failed to mark inode %lu dirty",
				   mpd->inode->i_ino);
1606 1607
	}

1608
submit_io:
1609
	mpage_da_submit_io(mpd, mapp);
1610
	mpd->io_done = 1;
1611 1612
}

1613 1614
#define BH_FLAGS ((1 << BH_Uptodate) | (1 << BH_Mapped) | \
		(1 << BH_Delay) | (1 << BH_Unwritten))
1615 1616 1617 1618 1619 1620 1621 1622 1623 1624 1625

/*
 * mpage_add_bh_to_extent - try to add one more block to extent of blocks
 *
 * @mpd->lbh - extent of blocks
 * @logical - logical number of the block in the file
 * @bh - bh of the block (used to access block's state)
 *
 * the function is used to collect contig. blocks in same state
 */
static void mpage_add_bh_to_extent(struct mpage_da_data *mpd,
1626 1627
				   sector_t logical, size_t b_size,
				   unsigned long b_state)
1628 1629
{
	sector_t next;
1630
	int nrblocks = mpd->b_size >> mpd->inode->i_blkbits;
1631

1632 1633 1634 1635
	/*
	 * XXX Don't go larger than mballoc is willing to allocate
	 * This is a stopgap solution.  We eventually need to fold
	 * mpage_da_submit_io() into this function and then call
1636
	 * ext4_map_blocks() multiple times in a loop
1637 1638 1639 1640
	 */
	if (nrblocks >= 8*1024*1024/mpd->inode->i_sb->s_blocksize)
		goto flush_it;

1641
	/* check if thereserved journal credits might overflow */
1642
	if (!(ext4_test_inode_flag(mpd->inode, EXT4_INODE_EXTENTS))) {
1643 1644 1645 1646 1647 1648 1649 1650 1651 1652 1653 1654 1655 1656 1657 1658 1659 1660 1661 1662
		if (nrblocks >= EXT4_MAX_TRANS_DATA) {
			/*
			 * With non-extent format we are limited by the journal
			 * credit available.  Total credit needed to insert
			 * nrblocks contiguous blocks is dependent on the
			 * nrblocks.  So limit nrblocks.
			 */
			goto flush_it;
		} else if ((nrblocks + (b_size >> mpd->inode->i_blkbits)) >
				EXT4_MAX_TRANS_DATA) {
			/*
			 * Adding the new buffer_head would make it cross the
			 * allowed limit for which we have journal credit
			 * reserved. So limit the new bh->b_size
			 */
			b_size = (EXT4_MAX_TRANS_DATA - nrblocks) <<
						mpd->inode->i_blkbits;
			/* we will do mpage_da_submit_io in the next loop */
		}
	}
1663 1664 1665
	/*
	 * First block in the extent
	 */
1666 1667 1668 1669
	if (mpd->b_size == 0) {
		mpd->b_blocknr = logical;
		mpd->b_size = b_size;
		mpd->b_state = b_state & BH_FLAGS;
1670 1671 1672
		return;
	}

1673
	next = mpd->b_blocknr + nrblocks;
1674 1675 1676
	/*
	 * Can we merge the block to our big extent?
	 */
1677 1678
	if (logical == next && (b_state & BH_FLAGS) == mpd->b_state) {
		mpd->b_size += b_size;
1679 1680 1681
		return;
	}

1682
flush_it:
1683 1684 1685 1686
	/*
	 * We couldn't merge the block to our extent, so we
	 * need to flush current  extent and start new one
	 */
1687
	mpage_da_map_and_submit(mpd);
1688
	return;
1689 1690
}

1691
static int ext4_bh_delay_or_unwritten(handle_t *handle, struct buffer_head *bh)
1692
{
1693
	return (buffer_delay(bh) || buffer_unwritten(bh)) && buffer_dirty(bh);
1694 1695
}

1696 1697 1698 1699 1700 1701 1702 1703 1704 1705 1706 1707 1708 1709 1710 1711 1712 1713 1714 1715 1716 1717 1718 1719 1720 1721 1722 1723 1724 1725 1726 1727 1728 1729 1730 1731 1732 1733 1734 1735 1736 1737 1738 1739 1740 1741 1742 1743 1744 1745 1746 1747 1748 1749 1750 1751 1752 1753 1754 1755
/*
 * This function is grabs code from the very beginning of
 * ext4_map_blocks, but assumes that the caller is from delayed write
 * time. This function looks up the requested blocks and sets the
 * buffer delay bit under the protection of i_data_sem.
 */
static int ext4_da_map_blocks(struct inode *inode, sector_t iblock,
			      struct ext4_map_blocks *map,
			      struct buffer_head *bh)
{
	int retval;
	sector_t invalid_block = ~((sector_t) 0xffff);

	if (invalid_block < ext4_blocks_count(EXT4_SB(inode->i_sb)->s_es))
		invalid_block = ~0;

	map->m_flags = 0;
	ext_debug("ext4_da_map_blocks(): inode %lu, max_blocks %u,"
		  "logical block %lu\n", inode->i_ino, map->m_len,
		  (unsigned long) map->m_lblk);
	/*
	 * Try to see if we can get the block without requesting a new
	 * file system block.
	 */
	down_read((&EXT4_I(inode)->i_data_sem));
	if (ext4_test_inode_flag(inode, EXT4_INODE_EXTENTS))
		retval = ext4_ext_map_blocks(NULL, inode, map, 0);
	else
		retval = ext4_ind_map_blocks(NULL, inode, map, 0);

	if (retval == 0) {
		/*
		 * XXX: __block_prepare_write() unmaps passed block,
		 * is it OK?
		 */
		/* If the block was allocated from previously allocated cluster,
		 * then we dont need to reserve it again. */
		if (!(map->m_flags & EXT4_MAP_FROM_CLUSTER)) {
			retval = ext4_da_reserve_space(inode, iblock);
			if (retval)
				/* not enough space to reserve */
				goto out_unlock;
		}

		/* Clear EXT4_MAP_FROM_CLUSTER flag since its purpose is served
		 * and it should not appear on the bh->b_state.
		 */
		map->m_flags &= ~EXT4_MAP_FROM_CLUSTER;

		map_bh(bh, inode->i_sb, invalid_block);
		set_buffer_new(bh);
		set_buffer_delay(bh);
	}

out_unlock:
	up_read((&EXT4_I(inode)->i_data_sem));

	return retval;
}

1756
/*
1757 1758 1759
 * This is a special get_blocks_t callback which is used by
 * ext4_da_write_begin().  It will either return mapped block or
 * reserve space for a single block.
1760 1761 1762 1763 1764 1765 1766
 *
 * For delayed buffer_head we have BH_Mapped, BH_New, BH_Delay set.
 * We also have b_blocknr = -1 and b_bdev initialized properly
 *
 * For unwritten buffer_head we have BH_Mapped, BH_New, BH_Unwritten set.
 * We also have b_blocknr = physicalblock mapping unwritten extent and b_bdev
 * initialized properly.
1767 1768
 */
static int ext4_da_get_block_prep(struct inode *inode, sector_t iblock,
1769
				  struct buffer_head *bh, int create)
1770
{
1771
	struct ext4_map_blocks map;
1772 1773 1774
	int ret = 0;

	BUG_ON(create == 0);
1775 1776 1777 1778
	BUG_ON(bh->b_size != inode->i_sb->s_blocksize);

	map.m_lblk = iblock;
	map.m_len = 1;
1779 1780 1781 1782 1783 1784

	/*
	 * first, we need to know whether the block is allocated already
	 * preallocated blocks are unmapped but should treated
	 * the same as allocated blocks.
	 */
1785 1786
	ret = ext4_da_map_blocks(inode, iblock, &map, bh);
	if (ret <= 0)
1787
		return ret;
1788

1789 1790 1791 1792 1793 1794 1795 1796 1797 1798 1799
	map_bh(bh, inode->i_sb, map.m_pblk);
	bh->b_state = (bh->b_state & ~EXT4_MAP_FLAGS) | map.m_flags;

	if (buffer_unwritten(bh)) {
		/* A delayed write to unwritten bh should be marked
		 * new and mapped.  Mapped ensures that we don't do
		 * get_block multiple times when we write to the same
		 * offset and new ensures that we do proper zero out
		 * for partial write.
		 */
		set_buffer_new(bh);
1800
		set_buffer_mapped(bh);
1801 1802
	}
	return 0;
1803
}
1804

1805 1806 1807
/*
 * This function is used as a standard get_block_t calback function
 * when there is no desire to allocate any blocks.  It is used as a
1808
 * callback function for block_write_begin() and block_write_full_page().
1809
 * These functions should only try to map a single block at a time.
1810 1811 1812 1813 1814
 *
 * Since this function doesn't do block allocations even if the caller
 * requests it by passing in create=1, it is critically important that
 * any caller checks to make sure that any buffer heads are returned
 * by this function are either all already mapped or marked for
1815 1816 1817
 * delayed allocation before calling  block_write_full_page().  Otherwise,
 * b_blocknr could be left unitialized, and the page write functions will
 * be taken by surprise.
1818 1819
 */
static int noalloc_get_block_write(struct inode *inode, sector_t iblock,
1820 1821
				   struct buffer_head *bh_result, int create)
{
1822
	BUG_ON(bh_result->b_size != inode->i_sb->s_blocksize);
1823
	return _ext4_get_block(inode, iblock, bh_result, 0);
1824 1825
}

1826 1827 1828 1829 1830 1831 1832 1833 1834 1835 1836 1837 1838 1839 1840 1841 1842 1843 1844 1845 1846 1847
static int bget_one(handle_t *handle, struct buffer_head *bh)
{
	get_bh(bh);
	return 0;
}

static int bput_one(handle_t *handle, struct buffer_head *bh)
{
	put_bh(bh);
	return 0;
}

static int __ext4_journalled_writepage(struct page *page,
				       unsigned int len)
{
	struct address_space *mapping = page->mapping;
	struct inode *inode = mapping->host;
	struct buffer_head *page_bufs;
	handle_t *handle = NULL;
	int ret = 0;
	int err;

1848
	ClearPageChecked(page);
1849 1850 1851 1852 1853 1854 1855 1856 1857 1858 1859 1860 1861
	page_bufs = page_buffers(page);
	BUG_ON(!page_bufs);
	walk_page_buffers(handle, page_bufs, 0, len, NULL, bget_one);
	/* As soon as we unlock the page, it can go away, but we have
	 * references to buffers so we are safe */
	unlock_page(page);

	handle = ext4_journal_start(inode, ext4_writepage_trans_blocks(inode));
	if (IS_ERR(handle)) {
		ret = PTR_ERR(handle);
		goto out;
	}

1862 1863
	BUG_ON(!ext4_handle_valid(handle));

1864 1865 1866 1867 1868 1869 1870
	ret = walk_page_buffers(handle, page_bufs, 0, len, NULL,
				do_journal_get_write_access);

	err = walk_page_buffers(handle, page_bufs, 0, len, NULL,
				write_end_fn);
	if (ret == 0)
		ret = err;
1871
	EXT4_I(inode)->i_datasync_tid = handle->h_transaction->t_tid;
1872 1873 1874 1875 1876
	err = ext4_journal_stop(handle);
	if (!ret)
		ret = err;

	walk_page_buffers(handle, page_bufs, 0, len, NULL, bput_one);
1877
	ext4_set_inode_state(inode, EXT4_STATE_JDATA);
1878 1879 1880 1881
out:
	return ret;
}

1882 1883 1884
static int ext4_set_bh_endio(struct buffer_head *bh, struct inode *inode);
static void ext4_end_io_buffer_write(struct buffer_head *bh, int uptodate);

1885
/*
1886 1887 1888 1889
 * Note that we don't need to start a transaction unless we're journaling data
 * because we should have holes filled from ext4_page_mkwrite(). We even don't
 * need to file the inode to the transaction's list in ordered mode because if
 * we are writing back data added by write(), the inode is already there and if
Lucas De Marchi's avatar
Lucas De Marchi committed
1890
 * we are writing back data modified via mmap(), no one guarantees in which
1891 1892 1893 1894
 * transaction the data will hit the disk. In case we are journaling data, we
 * cannot start transaction directly because transaction start ranks above page
 * lock so we have to do some magic.
 *
1895 1896 1897 1898 1899
 * This function can get called via...
 *   - ext4_da_writepages after taking page lock (have journal handle)
 *   - journal_submit_inode_data_buffers (no journal handle)
 *   - shrink_page_list via pdflush (no journal handle)
 *   - grab_page_cache when doing write_begin (have journal handle)
1900 1901 1902 1903 1904 1905 1906 1907 1908
 *
 * We don't do any block allocation in this function. If we have page with
 * multiple blocks we need to write those buffer_heads that are mapped. This
 * is important for mmaped based write. So if we do with blocksize 1K
 * truncate(f, 1024);
 * a = mmap(f, 0, 4096);
 * a[0] = 'a';
 * truncate(f, 4096);
 * we have in the page first buffer_head mapped via page_mkwrite call back
1909
 * but other buffer_heads would be unmapped but dirty (dirty done via the
1910 1911 1912 1913 1914 1915 1916 1917 1918 1919 1920 1921 1922 1923 1924
 * do_wp_page). So writepage should write the first block. If we modify
 * the mmap area beyond 1024 we will again get a page_fault and the
 * page_mkwrite callback will do the block allocation and mark the
 * buffer_heads mapped.
 *
 * We redirty the page if we have any buffer_heads that is either delay or
 * unwritten in the page.
 *
 * We can get recursively called as show below.
 *
 *	ext4_writepage() -> kmalloc() -> __alloc_pages() -> page_launder() ->
 *		ext4_writepage()
 *
 * But since we don't do any block allocation we should not deadlock.
 * Page also have the dirty flag cleared so we don't get recurive page_lock.
1925
 */
1926
static int ext4_writepage(struct page *page,
1927
			  struct writeback_control *wbc)
1928
{
1929
	int ret = 0, commit_write = 0;
1930
	loff_t size;
1931
	unsigned int len;
1932
	struct buffer_head *page_bufs = NULL;
1933 1934
	struct inode *inode = page->mapping->host;

1935
	trace_ext4_writepage(page);
1936 1937 1938 1939 1940
	size = i_size_read(inode);
	if (page->index == size >> PAGE_CACHE_SHIFT)
		len = size & ~PAGE_CACHE_MASK;
	else
		len = PAGE_CACHE_SIZE;
1941

1942 1943
	/*
	 * If the page does not have buffers (for whatever reason),
1944
	 * try to create them using __block_write_begin.  If this
1945 1946
	 * fails, redirty the page and move on.
	 */
1947
	if (!page_has_buffers(page)) {
1948
		if (__block_write_begin(page, 0, len,
1949 1950
					noalloc_get_block_write)) {
		redirty_page:
1951 1952 1953 1954
			redirty_page_for_writepage(wbc, page);
			unlock_page(page);
			return 0;
		}
1955 1956 1957 1958 1959
		commit_write = 1;
	}
	page_bufs = page_buffers(page);
	if (walk_page_buffers(NULL, page_bufs, 0, len, NULL,
			      ext4_bh_delay_or_unwritten)) {
1960
		/*
1961 1962 1963
		 * We don't want to do block allocation, so redirty
		 * the page and return.  We may reach here when we do
		 * a journal commit via journal_submit_inode_data_buffers.
1964 1965 1966
		 * We can also reach here via shrink_page_list but it
		 * should never be for direct reclaim so warn if that
		 * happens
1967
		 */
1968 1969
		WARN_ON_ONCE((current->flags & (PF_MEMALLOC|PF_KSWAPD)) ==
								PF_MEMALLOC);
1970 1971 1972
		goto redirty_page;
	}
	if (commit_write)
1973
		/* now mark the buffer_heads as dirty and uptodate */
1974
		block_commit_write(page, 0, len);
1975

1976
	if (PageChecked(page) && ext4_should_journal_data(inode))
1977 1978 1979 1980
		/*
		 * It's mmapped pagecache.  Add buffers and journal it.  There
		 * doesn't seem much point in redirtying the page here.
		 */
1981
		return __ext4_journalled_writepage(page, len);
1982

1983
	if (buffer_uninit(page_bufs)) {
1984 1985 1986 1987
		ext4_set_bh_endio(page_bufs, inode);
		ret = block_write_full_page_endio(page, noalloc_get_block_write,
					    wbc, ext4_end_io_buffer_write);
	} else
1988 1989
		ret = block_write_full_page(page, noalloc_get_block_write,
					    wbc);
1990 1991 1992 1993

	return ret;
}

1994
/*
1995
 * This is called via ext4_da_writepages() to
Lucas De Marchi's avatar
Lucas De Marchi committed
1996
 * calculate the total number of credits to reserve to fit
1997 1998 1999
 * a single extent allocation into a single transaction,
 * ext4_da_writpeages() will loop calling this before
 * the block allocation.
2000
 */
2001 2002 2003 2004 2005 2006 2007 2008 2009 2010 2011

static int ext4_da_writepages_trans_blocks(struct inode *inode)
{
	int max_blocks = EXT4_I(inode)->i_reserved_data_blocks;

	/*
	 * With non-extent format the journal credit needed to
	 * insert nrblocks contiguous block is dependent on
	 * number of contiguous block. So we will limit
	 * number of contiguous block to a sane value
	 */
2012
	if (!(ext4_test_inode_flag(inode, EXT4_INODE_EXTENTS)) &&
2013 2014 2015 2016 2017
	    (max_blocks > EXT4_MAX_TRANS_DATA))
		max_blocks = EXT4_MAX_TRANS_DATA;

	return ext4_chunk_trans_blocks(inode, max_blocks);
}
2018

2019 2020
/*
 * write_cache_pages_da - walk the list of dirty pages of the given
2021
 * address space and accumulate pages that need writing, and call
2022 2023
 * mpage_da_map_and_submit to map a single contiguous memory region
 * and then write them.
2024 2025 2026
 */
static int write_cache_pages_da(struct address_space *mapping,
				struct writeback_control *wbc,
2027 2028
				struct mpage_da_data *mpd,
				pgoff_t *done_index)
2029
{
2030
	struct buffer_head	*bh, *head;
2031
	struct inode		*inode = mapping->host;
2032 2033 2034 2035 2036 2037
	struct pagevec		pvec;
	unsigned int		nr_pages;
	sector_t		logical;
	pgoff_t			index, end;
	long			nr_to_write = wbc->nr_to_write;
	int			i, tag, ret = 0;
2038

2039 2040 2041
	memset(mpd, 0, sizeof(struct mpage_da_data));
	mpd->wbc = wbc;
	mpd->inode = inode;
2042 2043 2044 2045
	pagevec_init(&pvec, 0);
	index = wbc->range_start >> PAGE_CACHE_SHIFT;
	end = wbc->range_end >> PAGE_CACHE_SHIFT;

2046
	if (wbc->sync_mode == WB_SYNC_ALL || wbc->tagged_writepages)
2047 2048 2049 2050
		tag = PAGECACHE_TAG_TOWRITE;
	else
		tag = PAGECACHE_TAG_DIRTY;

2051
	*done_index = index;
2052
	while (index <= end) {
2053
		nr_pages = pagevec_lookup_tag(&pvec, mapping, &index, tag,
2054 2055
			      min(end - index, (pgoff_t)PAGEVEC_SIZE-1) + 1);
		if (nr_pages == 0)
2056
			return 0;
2057 2058 2059 2060 2061 2062 2063 2064 2065 2066 2067

		for (i = 0; i < nr_pages; i++) {
			struct page *page = pvec.pages[i];

			/*
			 * At this point, the page may be truncated or
			 * invalidated (changing page->mapping to NULL), or
			 * even swizzled back from swapper_space to tmpfs file
			 * mapping. However, page->index will not change
			 * because we have a reference on the page.
			 */
2068 2069
			if (page->index > end)
				goto out;
2070

2071 2072
			*done_index = page->index + 1;

2073 2074 2075 2076 2077 2078 2079 2080 2081 2082
			/*
			 * If we can't merge this page, and we have
			 * accumulated an contiguous region, write it
			 */
			if ((mpd->next_page != page->index) &&
			    (mpd->next_page != mpd->first_page)) {
				mpage_da_map_and_submit(mpd);
				goto ret_extent_tail;
			}

2083 2084 2085
			lock_page(page);

			/*
2086 2087 2088 2089 2090 2091
			 * If the page is no longer dirty, or its
			 * mapping no longer corresponds to inode we
			 * are writing (which means it has been
			 * truncated or invalidated), or the page is
			 * already under writeback and we are not
			 * doing a data integrity writeback, skip the page
2092
			 */
2093 2094 2095 2096
			if (!PageDirty(page) ||
			    (PageWriteback(page) &&
			     (wbc->sync_mode == WB_SYNC_NONE)) ||
			    unlikely(page->mapping != mapping)) {
2097 2098 2099 2100
				unlock_page(page);
				continue;
			}

2101
			wait_on_page_writeback(page);
2102 2103
			BUG_ON(PageWriteback(page));

2104
			if (mpd->next_page != page->index)
2105 2106 2107 2108 2109 2110
				mpd->first_page = page->index;
			mpd->next_page = page->index + 1;
			logical = (sector_t) page->index <<
				(PAGE_CACHE_SHIFT - inode->i_blkbits);

			if (!page_has_buffers(page)) {
2111 2112
				mpage_add_bh_to_extent(mpd, logical,
						       PAGE_CACHE_SIZE,
2113
						       (1 << BH_Dirty) | (1 << BH_Uptodate));
2114 2115
				if (mpd->io_done)
					goto ret_extent_tail;
2116 2117
			} else {
				/*
2118 2119
				 * Page with regular buffer heads,
				 * just add all dirty ones
2120 2121 2122 2123 2124 2125 2126 2127 2128 2129 2130 2131 2132 2133 2134
				 */
				head = page_buffers(page);
				bh = head;
				do {
					BUG_ON(buffer_locked(bh));
					/*
					 * We need to try to allocate
					 * unmapped blocks in the same page.
					 * Otherwise we won't make progress
					 * with the page in ext4_writepage
					 */
					if (ext4_bh_delay_or_unwritten(NULL, bh)) {
						mpage_add_bh_to_extent(mpd, logical,
								       bh->b_size,
								       bh->b_state);
2135 2136
						if (mpd->io_done)
							goto ret_extent_tail;
2137 2138
					} else if (buffer_dirty(bh) && (buffer_mapped(bh))) {
						/*
2139 2140 2141 2142 2143 2144 2145 2146 2147
						 * mapped dirty buffer. We need
						 * to update the b_state
						 * because we look at b_state
						 * in mpage_da_map_blocks.  We
						 * don't update b_size because
						 * if we find an unmapped
						 * buffer_head later we need to
						 * use the b_state flag of that
						 * buffer_head.
2148 2149 2150 2151 2152 2153
						 */
						if (mpd->b_size == 0)
							mpd->b_state = bh->b_state & BH_FLAGS;
					}
					logical++;
				} while ((bh = bh->b_this_page) != head);
2154 2155 2156 2157 2158
			}

			if (nr_to_write > 0) {
				nr_to_write--;
				if (nr_to_write == 0 &&
2159
				    wbc->sync_mode == WB_SYNC_NONE)
2160 2161 2162 2163 2164 2165 2166 2167 2168 2169
					/*
					 * We stop writing back only if we are
					 * not doing integrity sync. In case of
					 * integrity sync we have to keep going
					 * because someone may be concurrently
					 * dirtying pages, and we might have
					 * synced a lot of newly appeared dirty
					 * pages, but have not synced all of the
					 * old dirty pages.
					 */
2170
					goto out;
2171 2172 2173 2174 2175
			}
		}
		pagevec_release(&pvec);
		cond_resched();
	}
2176 2177 2178
	return 0;
ret_extent_tail:
	ret = MPAGE_DA_EXTENT_TAIL;
2179 2180 2181
out:
	pagevec_release(&pvec);
	cond_resched();
2182 2183 2184 2185
	return ret;
}


2186
static int ext4_da_writepages(struct address_space *mapping,
2187
			      struct writeback_control *wbc)
2188
{
2189 2190
	pgoff_t	index;
	int range_whole = 0;
2191
	handle_t *handle = NULL;
2192
	struct mpage_da_data mpd;
2193
	struct inode *inode = mapping->host;
2194
	int pages_written = 0;
2195
	unsigned int max_pages;
2196
	int range_cyclic, cycled = 1, io_done = 0;
2197 2198
	int needed_blocks, ret = 0;
	long desired_nr_to_write, nr_to_writebump = 0;
2199
	loff_t range_start = wbc->range_start;
2200
	struct ext4_sb_info *sbi = EXT4_SB(mapping->host->i_sb);
2201
	pgoff_t done_index = 0;
2202
	pgoff_t end;
2203
	struct blk_plug plug;
2204

2205
	trace_ext4_da_writepages(inode, wbc);
2206

2207 2208 2209 2210 2211
	/*
	 * No pages to write? This is mainly a kludge to avoid starting
	 * a transaction for special inodes like journal inode on last iput()
	 * because that could violate lock ordering on umount
	 */
2212
	if (!mapping->nrpages || !mapping_tagged(mapping, PAGECACHE_TAG_DIRTY))
2213
		return 0;
2214 2215 2216 2217 2218

	/*
	 * If the filesystem has aborted, it is read-only, so return
	 * right away instead of dumping stack traces later on that
	 * will obscure the real source of the problem.  We test
2219
	 * EXT4_MF_FS_ABORTED instead of sb->s_flag's MS_RDONLY because
2220 2221 2222 2223 2224
	 * the latter could be true if the filesystem is mounted
	 * read-only, and in that case, ext4_da_writepages should
	 * *never* be called, so if that ever happens, we would want
	 * the stack trace.
	 */
2225
	if (unlikely(sbi->s_mount_flags & EXT4_MF_FS_ABORTED))
2226 2227
		return -EROFS;

2228 2229
	if (wbc->range_start == 0 && wbc->range_end == LLONG_MAX)
		range_whole = 1;
2230

2231 2232
	range_cyclic = wbc->range_cyclic;
	if (wbc->range_cyclic) {
2233
		index = mapping->writeback_index;
2234 2235 2236 2237 2238
		if (index)
			cycled = 0;
		wbc->range_start = index << PAGE_CACHE_SHIFT;
		wbc->range_end  = LLONG_MAX;
		wbc->range_cyclic = 0;
2239 2240
		end = -1;
	} else {
2241
		index = wbc->range_start >> PAGE_CACHE_SHIFT;
2242 2243
		end = wbc->range_end >> PAGE_CACHE_SHIFT;
	}
2244

2245 2246 2247 2248 2249 2250 2251 2252 2253 2254 2255 2256 2257 2258 2259 2260 2261
	/*
	 * This works around two forms of stupidity.  The first is in
	 * the writeback code, which caps the maximum number of pages
	 * written to be 1024 pages.  This is wrong on multiple
	 * levels; different architectues have a different page size,
	 * which changes the maximum amount of data which gets
	 * written.  Secondly, 4 megabytes is way too small.  XFS
	 * forces this value to be 16 megabytes by multiplying
	 * nr_to_write parameter by four, and then relies on its
	 * allocator to allocate larger extents to make them
	 * contiguous.  Unfortunately this brings us to the second
	 * stupidity, which is that ext4's mballoc code only allocates
	 * at most 2048 blocks.  So we force contiguous writes up to
	 * the number of dirty blocks in the inode, or
	 * sbi->max_writeback_mb_bump whichever is smaller.
	 */
	max_pages = sbi->s_max_writeback_mb_bump << (20 - PAGE_CACHE_SHIFT);
2262 2263 2264 2265 2266 2267
	if (!range_cyclic && range_whole) {
		if (wbc->nr_to_write == LONG_MAX)
			desired_nr_to_write = wbc->nr_to_write;
		else
			desired_nr_to_write = wbc->nr_to_write * 8;
	} else
2268 2269 2270 2271 2272 2273 2274 2275 2276 2277
		desired_nr_to_write = ext4_num_dirty_pages(inode, index,
							   max_pages);
	if (desired_nr_to_write > max_pages)
		desired_nr_to_write = max_pages;

	if (wbc->nr_to_write < desired_nr_to_write) {
		nr_to_writebump = desired_nr_to_write - wbc->nr_to_write;
		wbc->nr_to_write = desired_nr_to_write;
	}

2278
retry:
2279
	if (wbc->sync_mode == WB_SYNC_ALL || wbc->tagged_writepages)
2280 2281
		tag_pages_for_writeback(mapping, index, end);

2282
	blk_start_plug(&plug);
2283
	while (!ret && wbc->nr_to_write > 0) {
2284 2285 2286 2287 2288 2289 2290 2291

		/*
		 * we  insert one extent at a time. So we need
		 * credit needed for single extent allocation.
		 * journalled mode is currently not supported
		 * by delalloc
		 */
		BUG_ON(ext4_should_journal_data(inode));
2292
		needed_blocks = ext4_da_writepages_trans_blocks(inode);
2293

2294 2295 2296 2297
		/* start a new transaction*/
		handle = ext4_journal_start(inode, needed_blocks);
		if (IS_ERR(handle)) {
			ret = PTR_ERR(handle);
2298
			ext4_msg(inode->i_sb, KERN_CRIT, "%s: jbd2_start: "
2299
			       "%ld pages, ino %lu; err %d", __func__,
2300
				wbc->nr_to_write, inode->i_ino, ret);
2301
			blk_finish_plug(&plug);
2302 2303
			goto out_writepages;
		}
2304 2305

		/*
2306
		 * Now call write_cache_pages_da() to find the next
2307
		 * contiguous region of logical blocks that need
2308
		 * blocks to be allocated by ext4 and submit them.
2309
		 */
2310
		ret = write_cache_pages_da(mapping, wbc, &mpd, &done_index);
2311
		/*
2312
		 * If we have a contiguous extent of pages and we
2313 2314 2315 2316
		 * haven't done the I/O yet, map the blocks and submit
		 * them for I/O.
		 */
		if (!mpd.io_done && mpd.next_page != mpd.first_page) {
2317
			mpage_da_map_and_submit(&mpd);
2318 2319
			ret = MPAGE_DA_EXTENT_TAIL;
		}
2320
		trace_ext4_da_write_pages(inode, &mpd);
2321
		wbc->nr_to_write -= mpd.pages_written;
2322

2323
		ext4_journal_stop(handle);
2324

2325
		if ((mpd.retval == -ENOSPC) && sbi->s_journal) {
2326 2327 2328 2329
			/* commit the transaction which would
			 * free blocks released in the transaction
			 * and try again
			 */
2330
			jbd2_journal_force_commit_nested(sbi->s_journal);
2331 2332
			ret = 0;
		} else if (ret == MPAGE_DA_EXTENT_TAIL) {
2333
			/*
2334 2335 2336
			 * Got one extent now try with rest of the pages.
			 * If mpd.retval is set -EIO, journal is aborted.
			 * So we don't need to write any more.
2337
			 */
2338
			pages_written += mpd.pages_written;
2339
			ret = mpd.retval;
2340
			io_done = 1;
2341
		} else if (wbc->nr_to_write)
2342 2343 2344 2345 2346 2347
			/*
			 * There is no more writeout needed
			 * or we requested for a noblocking writeout
			 * and we found the device congested
			 */
			break;
2348
	}
2349
	blk_finish_plug(&plug);
2350 2351 2352 2353 2354 2355 2356
	if (!io_done && !cycled) {
		cycled = 1;
		index = 0;
		wbc->range_start = index << PAGE_CACHE_SHIFT;
		wbc->range_end  = mapping->writeback_index - 1;
		goto retry;
	}
2357 2358

	/* Update index */
2359
	wbc->range_cyclic = range_cyclic;
2360 2361 2362 2363 2364
	if (wbc->range_cyclic || (range_whole && wbc->nr_to_write > 0))
		/*
		 * set the writeback_index so that range_cyclic
		 * mode will write it back later
		 */
2365
		mapping->writeback_index = done_index;
2366

2367
out_writepages:
2368
	wbc->nr_to_write -= nr_to_writebump;
2369
	wbc->range_start = range_start;
2370
	trace_ext4_da_writepages_result(inode, wbc, ret, pages_written);
2371
	return ret;
2372 2373
}

2374 2375 2376 2377 2378 2379 2380 2381 2382
#define FALL_BACK_TO_NONDELALLOC 1
static int ext4_nonda_switch(struct super_block *sb)
{
	s64 free_blocks, dirty_blocks;
	struct ext4_sb_info *sbi = EXT4_SB(sb);

	/*
	 * switch to non delalloc mode if we are running low
	 * on free block. The free block accounting via percpu
2383
	 * counters can get slightly wrong with percpu_counter_batch getting
2384 2385 2386 2387
	 * accumulated on each CPU without updating global counters
	 * Delalloc need an accurate free block accounting. So switch
	 * to non delalloc when we are near to error range.
	 */
2388 2389 2390
	free_blocks  = EXT4_C2B(sbi,
		percpu_counter_read_positive(&sbi->s_freeclusters_counter));
	dirty_blocks = percpu_counter_read_positive(&sbi->s_dirtyclusters_counter);
2391 2392 2393 2394 2395 2396 2397 2398 2399 2400
	/*
	 * Start pushing delalloc when 1/2 of free blocks are dirty.
	 */
	if (dirty_blocks && (free_blocks < 2 * dirty_blocks) &&
	    !writeback_in_progress(sb->s_bdi) &&
	    down_read_trylock(&sb->s_umount)) {
		writeback_inodes_sb(sb, WB_REASON_FS_FREE_SPACE);
		up_read(&sb->s_umount);
	}

2401
	if (2 * free_blocks < 3 * dirty_blocks ||
2402
		free_blocks < (dirty_blocks + EXT4_FREECLUSTERS_WATERMARK)) {
2403
		/*
2404 2405
		 * free block count is less than 150% of dirty blocks
		 * or free blocks is less than watermark
2406 2407 2408 2409 2410 2411
		 */
		return 1;
	}
	return 0;
}

2412
static int ext4_da_write_begin(struct file *file, struct address_space *mapping,
2413 2414
			       loff_t pos, unsigned len, unsigned flags,
			       struct page **pagep, void **fsdata)
2415
{
2416
	int ret, retries = 0;
2417 2418 2419 2420 2421 2422
	struct page *page;
	pgoff_t index;
	struct inode *inode = mapping->host;
	handle_t *handle;

	index = pos >> PAGE_CACHE_SHIFT;
2423 2424 2425 2426 2427 2428 2429

	if (ext4_nonda_switch(inode->i_sb)) {
		*fsdata = (void *)FALL_BACK_TO_NONDELALLOC;
		return ext4_write_begin(file, mapping, pos,
					len, flags, pagep, fsdata);
	}
	*fsdata = (void *)0;
2430
	trace_ext4_da_write_begin(inode, pos, len, flags);
2431
retry:
2432 2433 2434 2435 2436 2437 2438 2439 2440 2441 2442
	/*
	 * With delayed allocation, we don't log the i_disksize update
	 * if there is delayed block allocation. But we still need
	 * to journalling the i_disksize update if writes to the end
	 * of file which has an already mapped buffer.
	 */
	handle = ext4_journal_start(inode, 1);
	if (IS_ERR(handle)) {
		ret = PTR_ERR(handle);
		goto out;
	}
2443 2444 2445
	/* We cannot recurse into the filesystem as the transaction is already
	 * started */
	flags |= AOP_FLAG_NOFS;
2446

2447
	page = grab_cache_page_write_begin(mapping, index, flags);
2448 2449 2450 2451 2452
	if (!page) {
		ext4_journal_stop(handle);
		ret = -ENOMEM;
		goto out;
	}
2453 2454
	*pagep = page;

2455
	ret = __block_write_begin(page, pos, len, ext4_da_get_block_prep);
2456 2457 2458 2459
	if (ret < 0) {
		unlock_page(page);
		ext4_journal_stop(handle);
		page_cache_release(page);
2460 2461 2462 2463 2464 2465
		/*
		 * block_write_begin may have instantiated a few blocks
		 * outside i_size.  Trim these off again. Don't need
		 * i_size_read because we hold i_mutex.
		 */
		if (pos + len > inode->i_size)
2466
			ext4_truncate_failed_write(inode);
2467 2468
	}

2469 2470
	if (ret == -ENOSPC && ext4_should_retry_alloc(inode->i_sb, &retries))
		goto retry;
2471 2472 2473 2474
out:
	return ret;
}

2475 2476 2477 2478 2479
/*
 * Check if we should update i_disksize
 * when write to the end of file but not require block allocation
 */
static int ext4_da_should_update_i_disksize(struct page *page,
2480
					    unsigned long offset)
2481 2482 2483 2484 2485 2486 2487 2488 2489
{
	struct buffer_head *bh;
	struct inode *inode = page->mapping->host;
	unsigned int idx;
	int i;

	bh = page_buffers(page);
	idx = offset >> inode->i_blkbits;

2490
	for (i = 0; i < idx; i++)
2491 2492
		bh = bh->b_this_page;

2493
	if (!buffer_mapped(bh) || (buffer_delay(bh)) || buffer_unwritten(bh))
2494 2495 2496 2497
		return 0;
	return 1;
}

2498
static int ext4_da_write_end(struct file *file,
2499 2500 2501
			     struct address_space *mapping,
			     loff_t pos, unsigned len, unsigned copied,
			     struct page *page, void *fsdata)
2502 2503 2504 2505 2506
{
	struct inode *inode = mapping->host;
	int ret = 0, ret2;
	handle_t *handle = ext4_journal_current_handle();
	loff_t new_i_size;
2507
	unsigned long start, end;
2508 2509 2510
	int write_mode = (int)(unsigned long)fsdata;

	if (write_mode == FALL_BACK_TO_NONDELALLOC) {
2511 2512
		switch (ext4_inode_journal_mode(inode)) {
		case EXT4_INODE_ORDERED_DATA_MODE:
2513 2514
			return ext4_ordered_write_end(file, mapping, pos,
					len, copied, page, fsdata);
2515
		case EXT4_INODE_WRITEBACK_DATA_MODE:
2516 2517
			return ext4_writeback_write_end(file, mapping, pos,
					len, copied, page, fsdata);
2518
		default:
2519 2520 2521
			BUG();
		}
	}
2522

2523
	trace_ext4_da_write_end(inode, pos, len, copied);
2524
	start = pos & (PAGE_CACHE_SIZE - 1);
2525
	end = start + copied - 1;
2526 2527 2528 2529 2530 2531 2532 2533

	/*
	 * generic_write_end() will run mark_inode_dirty() if i_size
	 * changes.  So let's piggyback the i_disksize mark_inode_dirty
	 * into that.
	 */

	new_i_size = pos + copied;
2534
	if (copied && new_i_size > EXT4_I(inode)->i_disksize) {
2535 2536 2537 2538 2539 2540 2541 2542 2543 2544
		if (ext4_da_should_update_i_disksize(page, end)) {
			down_write(&EXT4_I(inode)->i_data_sem);
			if (new_i_size > EXT4_I(inode)->i_disksize) {
				/*
				 * Updating i_disksize when extending file
				 * without needing block allocation
				 */
				if (ext4_should_order_data(inode))
					ret = ext4_jbd2_file_inode(handle,
								   inode);
2545

2546 2547 2548
				EXT4_I(inode)->i_disksize = new_i_size;
			}
			up_write(&EXT4_I(inode)->i_data_sem);
2549 2550 2551 2552 2553
			/* We need to mark inode dirty even if
			 * new_i_size is less that inode->i_size
			 * bu greater than i_disksize.(hint delalloc)
			 */
			ext4_mark_inode_dirty(handle, inode);
2554
		}
2555
	}
2556 2557 2558 2559 2560 2561 2562 2563 2564 2565 2566 2567 2568 2569 2570 2571 2572 2573 2574 2575 2576
	ret2 = generic_write_end(file, mapping, pos, len, copied,
							page, fsdata);
	copied = ret2;
	if (ret2 < 0)
		ret = ret2;
	ret2 = ext4_journal_stop(handle);
	if (!ret)
		ret = ret2;

	return ret ? ret : copied;
}

static void ext4_da_invalidatepage(struct page *page, unsigned long offset)
{
	/*
	 * Drop reserved blocks
	 */
	BUG_ON(!PageLocked(page));
	if (!page_has_buffers(page))
		goto out;

2577
	ext4_da_page_release_reservation(page, offset);
2578 2579 2580 2581 2582 2583 2584

out:
	ext4_invalidatepage(page, offset);

	return;
}

2585 2586 2587 2588 2589
/*
 * Force all delayed allocation blocks to be allocated for a given inode.
 */
int ext4_alloc_da_blocks(struct inode *inode)
{
2590 2591
	trace_ext4_alloc_da_blocks(inode);

2592 2593 2594 2595 2596 2597 2598 2599 2600 2601
	if (!EXT4_I(inode)->i_reserved_data_blocks &&
	    !EXT4_I(inode)->i_reserved_meta_blocks)
		return 0;

	/*
	 * We do something simple for now.  The filemap_flush() will
	 * also start triggering a write of the data blocks, which is
	 * not strictly speaking necessary (and for users of
	 * laptop_mode, not even desirable).  However, to do otherwise
	 * would require replicating code paths in:
2602
	 *
2603 2604 2605 2606 2607 2608 2609 2610 2611 2612 2613 2614
	 * ext4_da_writepages() ->
	 *    write_cache_pages() ---> (via passed in callback function)
	 *        __mpage_da_writepage() -->
	 *           mpage_add_bh_to_extent()
	 *           mpage_da_map_blocks()
	 *
	 * The problem is that write_cache_pages(), located in
	 * mm/page-writeback.c, marks pages clean in preparation for
	 * doing I/O, which is not desirable if we're not planning on
	 * doing I/O at all.
	 *
	 * We could call write_cache_pages(), and then redirty all of
2615
	 * the pages by calling redirty_page_for_writepage() but that
2616 2617
	 * would be ugly in the extreme.  So instead we would need to
	 * replicate parts of the code in the above functions,
Lucas De Marchi's avatar
Lucas De Marchi committed
2618
	 * simplifying them because we wouldn't actually intend to
2619 2620 2621
	 * write out the pages, but rather only collect contiguous
	 * logical block extents, call the multi-block allocator, and
	 * then update the buffer heads with the block allocations.
2622
	 *
2623 2624 2625 2626 2627 2628
	 * For now, though, we'll cheat by calling filemap_flush(),
	 * which will map the blocks, and start the I/O, but not
	 * actually wait for the I/O to complete.
	 */
	return filemap_flush(inode->i_mapping);
}
2629

2630 2631 2632 2633 2634
/*
 * bmap() is special.  It gets used by applications such as lilo and by
 * the swapper to find the on-disk block of a specific piece of data.
 *
 * Naturally, this is dangerous if the block concerned is still in the
2635
 * journal.  If somebody makes a swapfile on an ext4 data-journaling
2636 2637 2638 2639 2640 2641 2642 2643
 * filesystem and enables swap, then they may get a nasty shock when the
 * data getting swapped to that swapfile suddenly gets overwritten by
 * the original zero's written out previously to the journal and
 * awaiting writeback in the kernel's buffer cache.
 *
 * So, if we see any bmap calls here on a modified, data-journaled file,
 * take extra steps to flush any blocks which might be in the cache.
 */
2644
static sector_t ext4_bmap(struct address_space *mapping, sector_t block)
2645 2646 2647 2648 2649
{
	struct inode *inode = mapping->host;
	journal_t *journal;
	int err;

2650 2651 2652 2653 2654 2655 2656 2657 2658 2659
	if (mapping_tagged(mapping, PAGECACHE_TAG_DIRTY) &&
			test_opt(inode->i_sb, DELALLOC)) {
		/*
		 * With delalloc we want to sync the file
		 * so that we can make sure we allocate
		 * blocks for file
		 */
		filemap_write_and_wait(mapping);
	}

2660 2661
	if (EXT4_JOURNAL(inode) &&
	    ext4_test_inode_state(inode, EXT4_STATE_JDATA)) {
2662 2663 2664 2665 2666 2667 2668 2669 2670 2671 2672
		/*
		 * This is a REALLY heavyweight approach, but the use of
		 * bmap on dirty files is expected to be extremely rare:
		 * only if we run lilo or swapon on a freshly made file
		 * do we expect this to happen.
		 *
		 * (bmap requires CAP_SYS_RAWIO so this does not
		 * represent an unprivileged user DOS attack --- we'd be
		 * in trouble if mortal users could trigger this path at
		 * will.)
		 *
2673
		 * NB. EXT4_STATE_JDATA is not set on files other than
2674 2675 2676 2677 2678 2679
		 * regular files.  If somebody wants to bmap a directory
		 * or symlink and gets confused because the buffer
		 * hasn't yet been flushed to disk, they deserve
		 * everything they get.
		 */

2680
		ext4_clear_inode_state(inode, EXT4_STATE_JDATA);
2681
		journal = EXT4_JOURNAL(inode);
2682 2683 2684
		jbd2_journal_lock_updates(journal);
		err = jbd2_journal_flush(journal);
		jbd2_journal_unlock_updates(journal);
2685 2686 2687 2688 2689

		if (err)
			return 0;
	}

2690
	return generic_block_bmap(mapping, block, ext4_get_block);
2691 2692
}

2693
static int ext4_readpage(struct file *file, struct page *page)
2694
{
2695
	trace_ext4_readpage(page);
2696
	return mpage_readpage(page, ext4_get_block);
2697 2698 2699
}

static int
2700
ext4_readpages(struct file *file, struct address_space *mapping,
2701 2702
		struct list_head *pages, unsigned nr_pages)
{
2703
	return mpage_readpages(mapping, pages, nr_pages, ext4_get_block);
2704 2705
}

2706 2707 2708 2709 2710 2711 2712 2713 2714 2715 2716 2717 2718 2719 2720 2721 2722 2723 2724 2725
static void ext4_invalidatepage_free_endio(struct page *page, unsigned long offset)
{
	struct buffer_head *head, *bh;
	unsigned int curr_off = 0;

	if (!page_has_buffers(page))
		return;
	head = bh = page_buffers(page);
	do {
		if (offset <= curr_off && test_clear_buffer_uninit(bh)
					&& bh->b_private) {
			ext4_free_io_end(bh->b_private);
			bh->b_private = NULL;
			bh->b_end_io = NULL;
		}
		curr_off = curr_off + bh->b_size;
		bh = bh->b_this_page;
	} while (bh != head);
}

2726
static void ext4_invalidatepage(struct page *page, unsigned long offset)
2727
{
2728
	journal_t *journal = EXT4_JOURNAL(page->mapping->host);
2729

2730 2731
	trace_ext4_invalidatepage(page, offset);

2732 2733 2734 2735 2736
	/*
	 * free any io_end structure allocated for buffers to be discarded
	 */
	if (ext4_should_dioread_nolock(page->mapping->host))
		ext4_invalidatepage_free_endio(page, offset);
2737 2738 2739 2740 2741 2742
	/*
	 * If it's a full truncate we just forget about the pending dirtying
	 */
	if (offset == 0)
		ClearPageChecked(page);

2743 2744 2745 2746
	if (journal)
		jbd2_journal_invalidatepage(journal, page, offset);
	else
		block_invalidatepage(page, offset);
2747 2748
}

2749
static int ext4_releasepage(struct page *page, gfp_t wait)
2750
{
2751
	journal_t *journal = EXT4_JOURNAL(page->mapping->host);
2752

2753 2754
	trace_ext4_releasepage(page);

2755 2756 2757
	WARN_ON(PageChecked(page));
	if (!page_has_buffers(page))
		return 0;
2758 2759 2760 2761
	if (journal)
		return jbd2_journal_try_to_free_buffers(journal, page, wait);
	else
		return try_to_free_buffers(page);
2762 2763
}

2764 2765 2766 2767 2768
/*
 * ext4_get_block used when preparing for a DIO write or buffer write.
 * We allocate an uinitialized extent if blocks haven't been allocated.
 * The extent will be converted to initialized after the IO is complete.
 */
2769
static int ext4_get_block_write(struct inode *inode, sector_t iblock,
2770 2771
		   struct buffer_head *bh_result, int create)
{
2772
	ext4_debug("ext4_get_block_write: inode %lu, create flag %d\n",
2773
		   inode->i_ino, create);
2774 2775
	return _ext4_get_block(inode, iblock, bh_result,
			       EXT4_GET_BLOCKS_IO_CREATE_EXT);
2776 2777 2778
}

static void ext4_end_io_dio(struct kiocb *iocb, loff_t offset,
2779 2780
			    ssize_t size, void *private, int ret,
			    bool is_async)
2781
{
2782
	struct inode *inode = iocb->ki_filp->f_path.dentry->d_inode;
2783 2784
        ext4_io_end_t *io_end = iocb->private;
	struct workqueue_struct *wq;
2785 2786
	unsigned long flags;
	struct ext4_inode_info *ei;
2787

2788 2789
	/* if not async direct IO or dio with 0 bytes write, just return */
	if (!io_end || !size)
2790
		goto out;
2791

2792
	ext_debug("ext4_end_io_dio(): io_end 0x%p "
2793
		  "for inode %lu, iocb 0x%p, offset %llu, size %zd\n",
2794 2795 2796
 		  iocb->private, io_end->inode->i_ino, iocb, offset,
		  size);

2797 2798
	iocb->private = NULL;

2799
	/* if not aio dio with unwritten extents, just free io and return */
2800
	if (!(io_end->flag & EXT4_IO_END_UNWRITTEN)) {
2801
		ext4_free_io_end(io_end);
2802
out:
2803
		inode_dio_done(inode);
2804 2805 2806
		if (is_async)
			aio_complete(iocb, ret, 0);
		return;
2807 2808
	}

2809 2810
	io_end->offset = offset;
	io_end->size = size;
2811 2812 2813 2814
	if (is_async) {
		io_end->iocb = iocb;
		io_end->result = ret;
	}
2815 2816
	wq = EXT4_SB(io_end->inode->i_sb)->dio_unwritten_wq;

2817
	/* Add the io_end to per-inode completed aio dio list*/
2818 2819 2820 2821
	ei = EXT4_I(io_end->inode);
	spin_lock_irqsave(&ei->i_completed_io_lock, flags);
	list_add_tail(&io_end->list, &ei->i_completed_io_list);
	spin_unlock_irqrestore(&ei->i_completed_io_lock, flags);
2822 2823

	/* queue the work to convert unwritten extents to written */
2824
	queue_work(wq, &io_end->work);
2825
}
2826

2827 2828 2829 2830 2831 2832 2833 2834 2835 2836 2837
static void ext4_end_io_buffer_write(struct buffer_head *bh, int uptodate)
{
	ext4_io_end_t *io_end = bh->b_private;
	struct workqueue_struct *wq;
	struct inode *inode;
	unsigned long flags;

	if (!test_clear_buffer_uninit(bh) || !io_end)
		goto out;

	if (!(io_end->inode->i_sb->s_flags & MS_ACTIVE)) {
2838 2839 2840
		ext4_msg(io_end->inode->i_sb, KERN_INFO,
			 "sb umounted, discard end_io request for inode %lu",
			 io_end->inode->i_ino);
2841 2842 2843 2844
		ext4_free_io_end(io_end);
		goto out;
	}

2845 2846 2847 2848
	/*
	 * It may be over-defensive here to check EXT4_IO_END_UNWRITTEN now,
	 * but being more careful is always safe for the future change.
	 */
2849
	inode = io_end->inode;
2850
	ext4_set_io_unwritten_flag(inode, io_end);
2851 2852 2853 2854 2855 2856 2857 2858 2859 2860 2861 2862 2863 2864 2865 2866 2867 2868 2869 2870 2871 2872 2873 2874 2875 2876

	/* Add the io_end to per-inode completed io list*/
	spin_lock_irqsave(&EXT4_I(inode)->i_completed_io_lock, flags);
	list_add_tail(&io_end->list, &EXT4_I(inode)->i_completed_io_list);
	spin_unlock_irqrestore(&EXT4_I(inode)->i_completed_io_lock, flags);

	wq = EXT4_SB(inode->i_sb)->dio_unwritten_wq;
	/* queue the work to convert unwritten extents to written */
	queue_work(wq, &io_end->work);
out:
	bh->b_private = NULL;
	bh->b_end_io = NULL;
	clear_buffer_uninit(bh);
	end_buffer_async_write(bh, uptodate);
}

static int ext4_set_bh_endio(struct buffer_head *bh, struct inode *inode)
{
	ext4_io_end_t *io_end;
	struct page *page = bh->b_page;
	loff_t offset = (sector_t)page->index << PAGE_CACHE_SHIFT;
	size_t size = bh->b_size;

retry:
	io_end = ext4_init_io_end(inode, GFP_ATOMIC);
	if (!io_end) {
2877
		pr_warn_ratelimited("%s: allocation fail\n", __func__);
2878 2879 2880 2881 2882 2883 2884 2885 2886 2887 2888 2889 2890 2891 2892 2893 2894 2895
		schedule();
		goto retry;
	}
	io_end->offset = offset;
	io_end->size = size;
	/*
	 * We need to hold a reference to the page to make sure it
	 * doesn't get evicted before ext4_end_io_work() has a chance
	 * to convert the extent from written to unwritten.
	 */
	io_end->page = page;
	get_page(io_end->page);

	bh->b_private = io_end;
	bh->b_end_io = ext4_end_io_buffer_write;
	return 0;
}

2896 2897 2898 2899 2900
/*
 * For ext4 extent files, ext4 will do direct-io write to holes,
 * preallocated extents, and those write extend the file, no need to
 * fall back to buffered IO.
 *
2901
 * For holes, we fallocate those blocks, mark them as uninitialized
2902
 * If those blocks were preallocated, we mark sure they are splited, but
2903
 * still keep the range to write as uninitialized.
2904
 *
2905 2906
 * The unwrritten extents will be converted to written when DIO is completed.
 * For async direct IO, since the IO may still pending when return, we
Lucas De Marchi's avatar
Lucas De Marchi committed
2907
 * set up an end_io call back function, which will do the conversion
2908
 * when async direct IO completed.
2909 2910 2911 2912 2913 2914 2915 2916 2917 2918 2919 2920 2921 2922 2923 2924 2925 2926
 *
 * If the O_DIRECT write will extend the file then add this inode to the
 * orphan list.  So recovery will truncate it back to the original size
 * if the machine crashes during the write.
 *
 */
static ssize_t ext4_ext_direct_IO(int rw, struct kiocb *iocb,
			      const struct iovec *iov, loff_t offset,
			      unsigned long nr_segs)
{
	struct file *file = iocb->ki_filp;
	struct inode *inode = file->f_mapping->host;
	ssize_t ret;
	size_t count = iov_length(iov, nr_segs);

	loff_t final_size = offset + count;
	if (rw == WRITE && final_size <= inode->i_size) {
		/*
2927 2928 2929
 		 * We could direct write to holes and fallocate.
		 *
 		 * Allocated blocks to fill the hole are marked as uninitialized
Lucas De Marchi's avatar
Lucas De Marchi committed
2930
 		 * to prevent parallel buffered read to expose the stale data
2931
 		 * before DIO complete the data IO.
2932 2933
		 *
 		 * As to previously fallocated extents, ext4 get_block
2934 2935 2936
 		 * will just simply mark the buffer mapped but still
 		 * keep the extents uninitialized.
 		 *
2937 2938 2939 2940 2941 2942 2943 2944
		 * for non AIO case, we will convert those unwritten extents
		 * to written after return back from blockdev_direct_IO.
		 *
		 * for async DIO, the conversion needs to be defered when
		 * the IO is completed. The ext4 end_io callback function
		 * will be called to take care of the conversion work.
		 * Here for async case, we allocate an io_end structure to
		 * hook to the iocb.
2945
 		 */
2946 2947 2948
		iocb->private = NULL;
		EXT4_I(inode)->cur_aio_dio = NULL;
		if (!is_sync_kiocb(iocb)) {
2949 2950 2951
			ext4_io_end_t *io_end =
				ext4_init_io_end(inode, GFP_NOFS);
			if (!io_end)
2952
				return -ENOMEM;
2953 2954
			io_end->flag |= EXT4_IO_END_DIRECT;
			iocb->private = io_end;
2955 2956
			/*
			 * we save the io structure for current async
2957
			 * direct IO, so that later ext4_map_blocks()
2958 2959 2960 2961 2962 2963 2964
			 * could flag the io structure whether there
			 * is a unwritten extents needs to be converted
			 * when IO is completed.
			 */
			EXT4_I(inode)->cur_aio_dio = iocb->private;
		}

2965
		ret = __blockdev_direct_IO(rw, iocb, inode,
2966 2967
					 inode->i_sb->s_bdev, iov,
					 offset, nr_segs,
2968
					 ext4_get_block_write,
2969 2970
					 ext4_end_io_dio,
					 NULL,
2971
					 DIO_LOCKING);
2972 2973 2974 2975 2976 2977 2978 2979 2980 2981 2982 2983 2984 2985 2986 2987 2988 2989 2990
		if (iocb->private)
			EXT4_I(inode)->cur_aio_dio = NULL;
		/*
		 * The io_end structure takes a reference to the inode,
		 * that structure needs to be destroyed and the
		 * reference to the inode need to be dropped, when IO is
		 * complete, even with 0 byte write, or failed.
		 *
		 * In the successful AIO DIO case, the io_end structure will be
		 * desctroyed and the reference to the inode will be dropped
		 * after the end_io call back function is called.
		 *
		 * In the case there is 0 byte write, or error case, since
		 * VFS direct IO won't invoke the end_io call back function,
		 * we need to free the end_io structure here.
		 */
		if (ret != -EIOCBQUEUED && ret <= 0 && iocb->private) {
			ext4_free_io_end(iocb->private);
			iocb->private = NULL;
2991 2992
		} else if (ret > 0 && ext4_test_inode_state(inode,
						EXT4_STATE_DIO_UNWRITTEN)) {
2993
			int err;
2994 2995
			/*
			 * for non AIO case, since the IO is already
Lucas De Marchi's avatar
Lucas De Marchi committed
2996
			 * completed, we could do the conversion right here
2997
			 */
2998 2999 3000 3001
			err = ext4_convert_unwritten_extents(inode,
							     offset, ret);
			if (err < 0)
				ret = err;
3002
			ext4_clear_inode_state(inode, EXT4_STATE_DIO_UNWRITTEN);
3003
		}
3004 3005
		return ret;
	}
3006 3007

	/* for write the the end of file case, we fall back to old way */
3008 3009 3010 3011 3012 3013 3014 3015 3016
	return ext4_ind_direct_IO(rw, iocb, iov, offset, nr_segs);
}

static ssize_t ext4_direct_IO(int rw, struct kiocb *iocb,
			      const struct iovec *iov, loff_t offset,
			      unsigned long nr_segs)
{
	struct file *file = iocb->ki_filp;
	struct inode *inode = file->f_mapping->host;
3017
	ssize_t ret;
3018

3019 3020 3021 3022 3023 3024
	/*
	 * If we are doing data journalling we don't support O_DIRECT
	 */
	if (ext4_should_journal_data(inode))
		return 0;

3025
	trace_ext4_direct_IO_enter(inode, offset, iov_length(iov, nr_segs), rw);
3026
	if (ext4_test_inode_flag(inode, EXT4_INODE_EXTENTS))
3027 3028 3029 3030 3031 3032
		ret = ext4_ext_direct_IO(rw, iocb, iov, offset, nr_segs);
	else
		ret = ext4_ind_direct_IO(rw, iocb, iov, offset, nr_segs);
	trace_ext4_direct_IO_exit(inode, offset,
				iov_length(iov, nr_segs), rw, ret);
	return ret;
3033 3034
}

3035
/*
3036
 * Pages can be marked dirty completely asynchronously from ext4's journalling
3037 3038 3039 3040 3041 3042 3043 3044 3045 3046 3047
 * activity.  By filemap_sync_pte(), try_to_unmap_one(), etc.  We cannot do
 * much here because ->set_page_dirty is called under VFS locks.  The page is
 * not necessarily locked.
 *
 * We cannot just dirty the page and leave attached buffers clean, because the
 * buffers' dirty state is "definitive".  We cannot just set the buffers dirty
 * or jbddirty because all the journalling code will explode.
 *
 * So what we do is to mark the page "pending dirty" and next time writepage
 * is called, propagate that into the buffers appropriately.
 */
3048
static int ext4_journalled_set_page_dirty(struct page *page)
3049 3050 3051 3052 3053
{
	SetPageChecked(page);
	return __set_page_dirty_nobuffers(page);
}

3054
static const struct address_space_operations ext4_ordered_aops = {
3055 3056
	.readpage		= ext4_readpage,
	.readpages		= ext4_readpages,
3057
	.writepage		= ext4_writepage,
3058 3059 3060 3061 3062 3063 3064 3065
	.write_begin		= ext4_write_begin,
	.write_end		= ext4_ordered_write_end,
	.bmap			= ext4_bmap,
	.invalidatepage		= ext4_invalidatepage,
	.releasepage		= ext4_releasepage,
	.direct_IO		= ext4_direct_IO,
	.migratepage		= buffer_migrate_page,
	.is_partially_uptodate  = block_is_partially_uptodate,
3066
	.error_remove_page	= generic_error_remove_page,
3067 3068
};

3069
static const struct address_space_operations ext4_writeback_aops = {
3070 3071
	.readpage		= ext4_readpage,
	.readpages		= ext4_readpages,
3072
	.writepage		= ext4_writepage,
3073 3074 3075 3076 3077 3078 3079 3080
	.write_begin		= ext4_write_begin,
	.write_end		= ext4_writeback_write_end,
	.bmap			= ext4_bmap,
	.invalidatepage		= ext4_invalidatepage,
	.releasepage		= ext4_releasepage,
	.direct_IO		= ext4_direct_IO,
	.migratepage		= buffer_migrate_page,
	.is_partially_uptodate  = block_is_partially_uptodate,
3081
	.error_remove_page	= generic_error_remove_page,
3082 3083
};

3084
static const struct address_space_operations ext4_journalled_aops = {
3085 3086
	.readpage		= ext4_readpage,
	.readpages		= ext4_readpages,
3087
	.writepage		= ext4_writepage,
3088 3089 3090 3091 3092 3093
	.write_begin		= ext4_write_begin,
	.write_end		= ext4_journalled_write_end,
	.set_page_dirty		= ext4_journalled_set_page_dirty,
	.bmap			= ext4_bmap,
	.invalidatepage		= ext4_invalidatepage,
	.releasepage		= ext4_releasepage,
3094
	.direct_IO		= ext4_direct_IO,
3095
	.is_partially_uptodate  = block_is_partially_uptodate,
3096
	.error_remove_page	= generic_error_remove_page,
3097 3098
};

3099
static const struct address_space_operations ext4_da_aops = {
3100 3101
	.readpage		= ext4_readpage,
	.readpages		= ext4_readpages,
3102
	.writepage		= ext4_writepage,
3103 3104 3105 3106 3107 3108 3109 3110 3111
	.writepages		= ext4_da_writepages,
	.write_begin		= ext4_da_write_begin,
	.write_end		= ext4_da_write_end,
	.bmap			= ext4_bmap,
	.invalidatepage		= ext4_da_invalidatepage,
	.releasepage		= ext4_releasepage,
	.direct_IO		= ext4_direct_IO,
	.migratepage		= buffer_migrate_page,
	.is_partially_uptodate  = block_is_partially_uptodate,
3112
	.error_remove_page	= generic_error_remove_page,
3113 3114
};

3115
void ext4_set_aops(struct inode *inode)
3116
{
3117 3118 3119 3120 3121 3122 3123 3124 3125 3126 3127 3128 3129 3130
	switch (ext4_inode_journal_mode(inode)) {
	case EXT4_INODE_ORDERED_DATA_MODE:
		if (test_opt(inode->i_sb, DELALLOC))
			inode->i_mapping->a_ops = &ext4_da_aops;
		else
			inode->i_mapping->a_ops = &ext4_ordered_aops;
		break;
	case EXT4_INODE_WRITEBACK_DATA_MODE:
		if (test_opt(inode->i_sb, DELALLOC))
			inode->i_mapping->a_ops = &ext4_da_aops;
		else
			inode->i_mapping->a_ops = &ext4_writeback_aops;
		break;
	case EXT4_INODE_JOURNAL_DATA_MODE:
3131
		inode->i_mapping->a_ops = &ext4_journalled_aops;
3132 3133 3134 3135
		break;
	default:
		BUG();
	}
3136 3137
}

3138 3139 3140 3141 3142 3143 3144 3145 3146 3147 3148 3149 3150 3151 3152 3153 3154 3155 3156 3157

/*
 * ext4_discard_partial_page_buffers()
 * Wrapper function for ext4_discard_partial_page_buffers_no_lock.
 * This function finds and locks the page containing the offset
 * "from" and passes it to ext4_discard_partial_page_buffers_no_lock.
 * Calling functions that already have the page locked should call
 * ext4_discard_partial_page_buffers_no_lock directly.
 */
int ext4_discard_partial_page_buffers(handle_t *handle,
		struct address_space *mapping, loff_t from,
		loff_t length, int flags)
{
	struct inode *inode = mapping->host;
	struct page *page;
	int err = 0;

	page = find_or_create_page(mapping, from >> PAGE_CACHE_SHIFT,
				   mapping_gfp_mask(mapping) & ~__GFP_FS);
	if (!page)
3158
		return -ENOMEM;
3159 3160 3161 3162 3163 3164 3165 3166 3167 3168 3169 3170 3171 3172 3173 3174 3175 3176 3177 3178 3179 3180 3181 3182 3183 3184 3185 3186 3187 3188 3189 3190 3191 3192 3193 3194 3195 3196 3197 3198 3199 3200

	err = ext4_discard_partial_page_buffers_no_lock(handle, inode, page,
		from, length, flags);

	unlock_page(page);
	page_cache_release(page);
	return err;
}

/*
 * ext4_discard_partial_page_buffers_no_lock()
 * Zeros a page range of length 'length' starting from offset 'from'.
 * Buffer heads that correspond to the block aligned regions of the
 * zeroed range will be unmapped.  Unblock aligned regions
 * will have the corresponding buffer head mapped if needed so that
 * that region of the page can be updated with the partial zero out.
 *
 * This function assumes that the page has already been  locked.  The
 * The range to be discarded must be contained with in the given page.
 * If the specified range exceeds the end of the page it will be shortened
 * to the end of the page that corresponds to 'from'.  This function is
 * appropriate for updating a page and it buffer heads to be unmapped and
 * zeroed for blocks that have been either released, or are going to be
 * released.
 *
 * handle: The journal handle
 * inode:  The files inode
 * page:   A locked page that contains the offset "from"
 * from:   The starting byte offset (from the begining of the file)
 *         to begin discarding
 * len:    The length of bytes to discard
 * flags:  Optional flags that may be used:
 *
 *         EXT4_DISCARD_PARTIAL_PG_ZERO_UNMAPPED
 *         Only zero the regions of the page whose buffer heads
 *         have already been unmapped.  This flag is appropriate
 *         for updateing the contents of a page whose blocks may
 *         have already been released, and we only want to zero
 *         out the regions that correspond to those released blocks.
 *
 * Returns zero on sucess or negative on failure.
 */
3201
static int ext4_discard_partial_page_buffers_no_lock(handle_t *handle,
3202 3203 3204 3205 3206 3207 3208 3209 3210 3211 3212 3213 3214 3215 3216 3217 3218 3219 3220 3221 3222 3223 3224 3225 3226
		struct inode *inode, struct page *page, loff_t from,
		loff_t length, int flags)
{
	ext4_fsblk_t index = from >> PAGE_CACHE_SHIFT;
	unsigned int offset = from & (PAGE_CACHE_SIZE-1);
	unsigned int blocksize, max, pos;
	ext4_lblk_t iblock;
	struct buffer_head *bh;
	int err = 0;

	blocksize = inode->i_sb->s_blocksize;
	max = PAGE_CACHE_SIZE - offset;

	if (index != page->index)
		return -EINVAL;

	/*
	 * correct length if it does not fall between
	 * 'from' and the end of the page
	 */
	if (length > max || length < 0)
		length = max;

	iblock = index << (PAGE_CACHE_SHIFT - inode->i_sb->s_blocksize_bits);

3227 3228
	if (!page_has_buffers(page))
		create_empty_buffers(page, blocksize, 0);
3229 3230 3231 3232 3233 3234 3235 3236 3237 3238 3239 3240

	/* Find the buffer that contains "offset" */
	bh = page_buffers(page);
	pos = blocksize;
	while (offset >= pos) {
		bh = bh->b_this_page;
		iblock++;
		pos += blocksize;
	}

	pos = offset;
	while (pos < offset + length) {
3241 3242
		unsigned int end_of_block, range_to_discard;

3243 3244 3245 3246 3247 3248 3249 3250 3251 3252 3253 3254 3255 3256 3257 3258 3259 3260 3261 3262 3263 3264 3265 3266 3267 3268 3269 3270 3271 3272 3273 3274 3275 3276 3277 3278 3279 3280 3281 3282 3283 3284 3285 3286 3287 3288 3289 3290 3291 3292 3293 3294 3295 3296 3297 3298 3299 3300 3301 3302 3303 3304 3305 3306 3307 3308 3309 3310 3311 3312 3313 3314 3315 3316 3317 3318 3319 3320 3321 3322 3323 3324 3325 3326 3327
		err = 0;

		/* The length of space left to zero and unmap */
		range_to_discard = offset + length - pos;

		/* The length of space until the end of the block */
		end_of_block = blocksize - (pos & (blocksize-1));

		/*
		 * Do not unmap or zero past end of block
		 * for this buffer head
		 */
		if (range_to_discard > end_of_block)
			range_to_discard = end_of_block;


		/*
		 * Skip this buffer head if we are only zeroing unampped
		 * regions of the page
		 */
		if (flags & EXT4_DISCARD_PARTIAL_PG_ZERO_UNMAPPED &&
			buffer_mapped(bh))
				goto next;

		/* If the range is block aligned, unmap */
		if (range_to_discard == blocksize) {
			clear_buffer_dirty(bh);
			bh->b_bdev = NULL;
			clear_buffer_mapped(bh);
			clear_buffer_req(bh);
			clear_buffer_new(bh);
			clear_buffer_delay(bh);
			clear_buffer_unwritten(bh);
			clear_buffer_uptodate(bh);
			zero_user(page, pos, range_to_discard);
			BUFFER_TRACE(bh, "Buffer discarded");
			goto next;
		}

		/*
		 * If this block is not completely contained in the range
		 * to be discarded, then it is not going to be released. Because
		 * we need to keep this block, we need to make sure this part
		 * of the page is uptodate before we modify it by writeing
		 * partial zeros on it.
		 */
		if (!buffer_mapped(bh)) {
			/*
			 * Buffer head must be mapped before we can read
			 * from the block
			 */
			BUFFER_TRACE(bh, "unmapped");
			ext4_get_block(inode, iblock, bh, 0);
			/* unmapped? It's a hole - nothing to do */
			if (!buffer_mapped(bh)) {
				BUFFER_TRACE(bh, "still unmapped");
				goto next;
			}
		}

		/* Ok, it's mapped. Make sure it's up-to-date */
		if (PageUptodate(page))
			set_buffer_uptodate(bh);

		if (!buffer_uptodate(bh)) {
			err = -EIO;
			ll_rw_block(READ, 1, &bh);
			wait_on_buffer(bh);
			/* Uhhuh. Read error. Complain and punt.*/
			if (!buffer_uptodate(bh))
				goto next;
		}

		if (ext4_should_journal_data(inode)) {
			BUFFER_TRACE(bh, "get write access");
			err = ext4_journal_get_write_access(handle, bh);
			if (err)
				goto next;
		}

		zero_user(page, pos, range_to_discard);

		err = 0;
		if (ext4_should_journal_data(inode)) {
			err = ext4_handle_dirty_metadata(handle, inode, bh);
3328
		} else
3329 3330 3331 3332 3333 3334 3335 3336 3337 3338 3339 3340
			mark_buffer_dirty(bh);

		BUFFER_TRACE(bh, "Partial buffer zeroed");
next:
		bh = bh->b_this_page;
		iblock++;
		pos += range_to_discard;
	}

	return err;
}

3341 3342 3343 3344 3345 3346 3347 3348 3349 3350 3351
int ext4_can_truncate(struct inode *inode)
{
	if (S_ISREG(inode->i_mode))
		return 1;
	if (S_ISDIR(inode->i_mode))
		return 1;
	if (S_ISLNK(inode->i_mode))
		return !ext4_inode_is_fast_symlink(inode);
	return 0;
}

3352 3353 3354 3355 3356 3357 3358 3359 3360 3361 3362 3363 3364 3365 3366
/*
 * ext4_punch_hole: punches a hole in a file by releaseing the blocks
 * associated with the given offset and length
 *
 * @inode:  File inode
 * @offset: The offset where the hole will begin
 * @len:    The length of the hole
 *
 * Returns: 0 on sucess or negative on failure
 */

int ext4_punch_hole(struct file *file, loff_t offset, loff_t length)
{
	struct inode *inode = file->f_path.dentry->d_inode;
	if (!S_ISREG(inode->i_mode))
3367
		return -EOPNOTSUPP;
3368 3369 3370

	if (!ext4_test_inode_flag(inode, EXT4_INODE_EXTENTS)) {
		/* TODO: Add support for non extent hole punching */
3371
		return -EOPNOTSUPP;
3372 3373
	}

3374 3375
	if (EXT4_SB(inode->i_sb)->s_cluster_ratio > 1) {
		/* TODO: Add support for bigalloc file systems */
3376
		return -EOPNOTSUPP;
3377 3378
	}

3379 3380 3381
	return ext4_ext_punch_hole(file, offset, length);
}

3382
/*
3383
 * ext4_truncate()
3384
 *
3385 3386
 * We block out ext4_get_block() block instantiations across the entire
 * transaction, and VFS/VM ensures that ext4_truncate() cannot run
3387 3388
 * simultaneously on behalf of the same inode.
 *
3389
 * As we work through the truncate and commit bits of it to the journal there
3390 3391 3392 3393 3394 3395 3396 3397 3398 3399 3400 3401 3402
 * is one core, guiding principle: the file's tree must always be consistent on
 * disk.  We must be able to restart the truncate after a crash.
 *
 * The file's tree may be transiently inconsistent in memory (although it
 * probably isn't), but whenever we close off and commit a journal transaction,
 * the contents of (the filesystem + the journal) must be consistent and
 * restartable.  It's pretty simple, really: bottom up, right to left (although
 * left-to-right works OK too).
 *
 * Note that at recovery time, journal replay occurs *before* the restart of
 * truncate against the orphan inode list.
 *
 * The committed inode has the new, desired i_size (which is the same as
3403
 * i_disksize in this case).  After a crash, ext4_orphan_cleanup() will see
3404
 * that this inode's truncate did not complete and it will again call
3405 3406
 * ext4_truncate() to have another go.  So there will be instantiated blocks
 * to the right of the truncation point in a crashed ext4 filesystem.  But
3407
 * that's fine - as long as they are linked from the inode, the post-crash
3408
 * ext4_truncate() run will find them and release them.
3409
 */
3410
void ext4_truncate(struct inode *inode)
3411
{
3412 3413
	trace_ext4_truncate_enter(inode);

3414
	if (!ext4_can_truncate(inode))
3415 3416
		return;

3417
	ext4_clear_inode_flag(inode, EXT4_INODE_EOFBLOCKS);
3418

3419
	if (inode->i_size == 0 && !test_opt(inode->i_sb, NO_AUTO_DA_ALLOC))
3420
		ext4_set_inode_state(inode, EXT4_STATE_DA_ALLOC_CLOSE);
3421

3422
	if (ext4_test_inode_flag(inode, EXT4_INODE_EXTENTS))
3423
		ext4_ext_truncate(inode);
3424 3425
	else
		ext4_ind_truncate(inode);
3426

3427
	trace_ext4_truncate_exit(inode);
3428 3429 3430
}

/*
3431
 * ext4_get_inode_loc returns with an extra refcount against the inode's
3432 3433 3434 3435
 * underlying buffer_head on success. If 'in_mem' is true, we have all
 * data in memory that is needed to recreate the on-disk version of this
 * inode.
 */
3436 3437
static int __ext4_get_inode_loc(struct inode *inode,
				struct ext4_iloc *iloc, int in_mem)
3438
{
3439 3440 3441 3442 3443 3444
	struct ext4_group_desc	*gdp;
	struct buffer_head	*bh;
	struct super_block	*sb = inode->i_sb;
	ext4_fsblk_t		block;
	int			inodes_per_block, inode_offset;

Aneesh Kumar K.V's avatar
Aneesh Kumar K.V committed
3445
	iloc->bh = NULL;
3446 3447
	if (!ext4_valid_inum(sb, inode->i_ino))
		return -EIO;
3448

3449 3450 3451
	iloc->block_group = (inode->i_ino - 1) / EXT4_INODES_PER_GROUP(sb);
	gdp = ext4_get_group_desc(sb, iloc->block_group, NULL);
	if (!gdp)
3452 3453
		return -EIO;

3454 3455 3456
	/*
	 * Figure out the offset within the block group inode table
	 */
3457
	inodes_per_block = EXT4_SB(sb)->s_inodes_per_block;
3458 3459 3460 3461 3462 3463
	inode_offset = ((inode->i_ino - 1) %
			EXT4_INODES_PER_GROUP(sb));
	block = ext4_inode_table(sb, gdp) + (inode_offset / inodes_per_block);
	iloc->offset = (inode_offset % inodes_per_block) * EXT4_INODE_SIZE(sb);

	bh = sb_getblk(sb, block);
3464
	if (!bh) {
3465 3466
		EXT4_ERROR_INODE_BLOCK(inode, block,
				       "unable to read itable block");
3467 3468 3469 3470
		return -EIO;
	}
	if (!buffer_uptodate(bh)) {
		lock_buffer(bh);
3471 3472 3473 3474 3475 3476 3477 3478 3479 3480

		/*
		 * If the buffer has the write error flag, we have failed
		 * to write out another inode in the same block.  In this
		 * case, we don't have to read the block because we may
		 * read the old inode data successfully.
		 */
		if (buffer_write_io_error(bh) && !buffer_uptodate(bh))
			set_buffer_uptodate(bh);

3481 3482 3483 3484 3485 3486 3487 3488 3489 3490 3491 3492 3493
		if (buffer_uptodate(bh)) {
			/* someone brought it uptodate while we waited */
			unlock_buffer(bh);
			goto has_buffer;
		}

		/*
		 * If we have all information of the inode in memory and this
		 * is the only valid inode in the block, we need not read the
		 * block.
		 */
		if (in_mem) {
			struct buffer_head *bitmap_bh;
3494
			int i, start;
3495

3496
			start = inode_offset & ~(inodes_per_block - 1);
3497

3498 3499
			/* Is the inode bitmap in cache? */
			bitmap_bh = sb_getblk(sb, ext4_inode_bitmap(sb, gdp));
3500 3501 3502 3503 3504 3505 3506 3507 3508 3509 3510 3511
			if (!bitmap_bh)
				goto make_io;

			/*
			 * If the inode bitmap isn't in cache then the
			 * optimisation may end up performing two reads instead
			 * of one, so skip it.
			 */
			if (!buffer_uptodate(bitmap_bh)) {
				brelse(bitmap_bh);
				goto make_io;
			}
3512
			for (i = start; i < start + inodes_per_block; i++) {
3513 3514
				if (i == inode_offset)
					continue;
3515
				if (ext4_test_bit(i, bitmap_bh->b_data))
3516 3517 3518
					break;
			}
			brelse(bitmap_bh);
3519
			if (i == start + inodes_per_block) {
3520 3521 3522 3523 3524 3525 3526 3527 3528
				/* all other inodes are free, so skip I/O */
				memset(bh->b_data, 0, bh->b_size);
				set_buffer_uptodate(bh);
				unlock_buffer(bh);
				goto has_buffer;
			}
		}

make_io:
3529 3530 3531 3532 3533 3534 3535 3536 3537
		/*
		 * If we need to do any I/O, try to pre-readahead extra
		 * blocks from the inode table.
		 */
		if (EXT4_SB(sb)->s_inode_readahead_blks) {
			ext4_fsblk_t b, end, table;
			unsigned num;

			table = ext4_inode_table(sb, gdp);
3538
			/* s_inode_readahead_blks is always a power of 2 */
3539 3540 3541 3542 3543 3544 3545
			b = block & ~(EXT4_SB(sb)->s_inode_readahead_blks-1);
			if (table > b)
				b = table;
			end = b + EXT4_SB(sb)->s_inode_readahead_blks;
			num = EXT4_INODES_PER_GROUP(sb);
			if (EXT4_HAS_RO_COMPAT_FEATURE(sb,
				       EXT4_FEATURE_RO_COMPAT_GDT_CSUM))
3546
				num -= ext4_itable_unused_count(sb, gdp);
3547 3548 3549 3550 3551 3552 3553
			table += num / inodes_per_block;
			if (end > table)
				end = table;
			while (b <= end)
				sb_breadahead(sb, b++);
		}

3554 3555 3556 3557 3558
		/*
		 * There are other valid inodes in the buffer, this inode
		 * has in-inode xattrs, or we don't have this inode in memory.
		 * Read the block from disk.
		 */
3559
		trace_ext4_load_inode(inode);
3560 3561
		get_bh(bh);
		bh->b_end_io = end_buffer_read_sync;
3562
		submit_bh(READ | REQ_META | REQ_PRIO, bh);
3563 3564
		wait_on_buffer(bh);
		if (!buffer_uptodate(bh)) {
3565 3566
			EXT4_ERROR_INODE_BLOCK(inode, block,
					       "unable to read itable block");
3567 3568 3569 3570 3571 3572 3573 3574 3575
			brelse(bh);
			return -EIO;
		}
	}
has_buffer:
	iloc->bh = bh;
	return 0;
}

3576
int ext4_get_inode_loc(struct inode *inode, struct ext4_iloc *iloc)
3577 3578
{
	/* We have all inode data except xattrs in memory here. */
3579
	return __ext4_get_inode_loc(inode, iloc,
3580
		!ext4_test_inode_state(inode, EXT4_STATE_XATTR));
3581 3582
}

3583
void ext4_set_inode_flags(struct inode *inode)
3584
{
3585
	unsigned int flags = EXT4_I(inode)->i_flags;
3586 3587

	inode->i_flags &= ~(S_SYNC|S_APPEND|S_IMMUTABLE|S_NOATIME|S_DIRSYNC);
3588
	if (flags & EXT4_SYNC_FL)
3589
		inode->i_flags |= S_SYNC;
3590
	if (flags & EXT4_APPEND_FL)
3591
		inode->i_flags |= S_APPEND;
3592
	if (flags & EXT4_IMMUTABLE_FL)
3593
		inode->i_flags |= S_IMMUTABLE;
3594
	if (flags & EXT4_NOATIME_FL)
3595
		inode->i_flags |= S_NOATIME;
3596
	if (flags & EXT4_DIRSYNC_FL)
3597 3598 3599
		inode->i_flags |= S_DIRSYNC;
}

3600 3601 3602
/* Propagate flags from i_flags to EXT4_I(inode)->i_flags */
void ext4_get_inode_flags(struct ext4_inode_info *ei)
{
3603 3604 3605 3606 3607 3608 3609 3610 3611 3612 3613 3614 3615 3616 3617 3618 3619 3620 3621 3622
	unsigned int vfs_fl;
	unsigned long old_fl, new_fl;

	do {
		vfs_fl = ei->vfs_inode.i_flags;
		old_fl = ei->i_flags;
		new_fl = old_fl & ~(EXT4_SYNC_FL|EXT4_APPEND_FL|
				EXT4_IMMUTABLE_FL|EXT4_NOATIME_FL|
				EXT4_DIRSYNC_FL);
		if (vfs_fl & S_SYNC)
			new_fl |= EXT4_SYNC_FL;
		if (vfs_fl & S_APPEND)
			new_fl |= EXT4_APPEND_FL;
		if (vfs_fl & S_IMMUTABLE)
			new_fl |= EXT4_IMMUTABLE_FL;
		if (vfs_fl & S_NOATIME)
			new_fl |= EXT4_NOATIME_FL;
		if (vfs_fl & S_DIRSYNC)
			new_fl |= EXT4_DIRSYNC_FL;
	} while (cmpxchg(&ei->i_flags, old_fl, new_fl) != old_fl);
3623
}
3624

3625
static blkcnt_t ext4_inode_blocks(struct ext4_inode *raw_inode,
3626
				  struct ext4_inode_info *ei)
3627 3628
{
	blkcnt_t i_blocks ;
Aneesh Kumar K.V's avatar
Aneesh Kumar K.V committed
3629 3630
	struct inode *inode = &(ei->vfs_inode);
	struct super_block *sb = inode->i_sb;
3631 3632 3633 3634 3635 3636

	if (EXT4_HAS_RO_COMPAT_FEATURE(sb,
				EXT4_FEATURE_RO_COMPAT_HUGE_FILE)) {
		/* we are using combined 48 bit field */
		i_blocks = ((u64)le16_to_cpu(raw_inode->i_blocks_high)) << 32 |
					le32_to_cpu(raw_inode->i_blocks_lo);
3637
		if (ext4_test_inode_flag(inode, EXT4_INODE_HUGE_FILE)) {
Aneesh Kumar K.V's avatar
Aneesh Kumar K.V committed
3638 3639 3640 3641 3642
			/* i_blocks represent file system block size */
			return i_blocks  << (inode->i_blkbits - 9);
		} else {
			return i_blocks;
		}
3643 3644 3645 3646
	} else {
		return le32_to_cpu(raw_inode->i_blocks_lo);
	}
}
3647

3648
struct inode *ext4_iget(struct super_block *sb, unsigned long ino)
3649
{
3650 3651
	struct ext4_iloc iloc;
	struct ext4_inode *raw_inode;
3652 3653
	struct ext4_inode_info *ei;
	struct inode *inode;
3654
	journal_t *journal = EXT4_SB(sb)->s_journal;
3655
	long ret;
3656 3657
	int block;

3658 3659 3660 3661 3662 3663 3664
	inode = iget_locked(sb, ino);
	if (!inode)
		return ERR_PTR(-ENOMEM);
	if (!(inode->i_state & I_NEW))
		return inode;

	ei = EXT4_I(inode);
3665
	iloc.bh = NULL;
3666

3667 3668
	ret = __ext4_get_inode_loc(inode, &iloc, 0);
	if (ret < 0)
3669
		goto bad_inode;
3670
	raw_inode = ext4_raw_inode(&iloc);
3671 3672 3673
	inode->i_mode = le16_to_cpu(raw_inode->i_mode);
	inode->i_uid = (uid_t)le16_to_cpu(raw_inode->i_uid_low);
	inode->i_gid = (gid_t)le16_to_cpu(raw_inode->i_gid_low);
3674
	if (!(test_opt(inode->i_sb, NO_UID32))) {
3675 3676 3677
		inode->i_uid |= le16_to_cpu(raw_inode->i_uid_high) << 16;
		inode->i_gid |= le16_to_cpu(raw_inode->i_gid_high) << 16;
	}
3678
	set_nlink(inode, le16_to_cpu(raw_inode->i_links_count));
3679

3680
	ext4_clear_state_flags(ei);	/* Only relevant on 32-bit archs */
3681 3682 3683 3684 3685 3686 3687 3688 3689
	ei->i_dir_start_lookup = 0;
	ei->i_dtime = le32_to_cpu(raw_inode->i_dtime);
	/* We now have enough fields to check if the inode was active or not.
	 * This is needed because nfsd might try to access dead inodes
	 * the test is that same one that e2fsck uses
	 * NeilBrown 1999oct15
	 */
	if (inode->i_nlink == 0) {
		if (inode->i_mode == 0 ||
3690
		    !(EXT4_SB(inode->i_sb)->s_mount_state & EXT4_ORPHAN_FS)) {
3691
			/* this inode is deleted */
3692
			ret = -ESTALE;
3693 3694 3695 3696 3697 3698 3699 3700
			goto bad_inode;
		}
		/* The only unlinked inodes we let through here have
		 * valid i_mode and are being read by the orphan
		 * recovery code: that's fine, we're about to complete
		 * the process of deleting those. */
	}
	ei->i_flags = le32_to_cpu(raw_inode->i_flags);
3701
	inode->i_blocks = ext4_inode_blocks(raw_inode, ei);
3702
	ei->i_file_acl = le32_to_cpu(raw_inode->i_file_acl_lo);
3703
	if (EXT4_HAS_INCOMPAT_FEATURE(sb, EXT4_FEATURE_INCOMPAT_64BIT))
3704 3705
		ei->i_file_acl |=
			((__u64)le16_to_cpu(raw_inode->i_file_acl_high)) << 32;
3706
	inode->i_size = ext4_isize(raw_inode);
3707
	ei->i_disksize = inode->i_size;
3708 3709 3710
#ifdef CONFIG_QUOTA
	ei->i_reserved_quota = 0;
#endif
3711 3712
	inode->i_generation = le32_to_cpu(raw_inode->i_generation);
	ei->i_block_group = iloc.block_group;
3713
	ei->i_last_alloc_group = ~0;
3714 3715 3716 3717
	/*
	 * NOTE! The in-memory inode i_data array is in little-endian order
	 * even on big-endian machines: we do NOT byteswap the block numbers!
	 */
3718
	for (block = 0; block < EXT4_N_BLOCKS; block++)
3719 3720 3721
		ei->i_data[block] = raw_inode->i_block[block];
	INIT_LIST_HEAD(&ei->i_orphan);

3722 3723 3724 3725 3726 3727 3728 3729 3730 3731 3732
	/*
	 * Set transaction id's of transactions that have to be committed
	 * to finish f[data]sync. We set them to currently running transaction
	 * as we cannot be sure that the inode or some of its metadata isn't
	 * part of the transaction - the inode could have been reclaimed and
	 * now it is reread from disk.
	 */
	if (journal) {
		transaction_t *transaction;
		tid_t tid;

3733
		read_lock(&journal->j_state_lock);
3734 3735 3736 3737 3738 3739 3740 3741
		if (journal->j_running_transaction)
			transaction = journal->j_running_transaction;
		else
			transaction = journal->j_committing_transaction;
		if (transaction)
			tid = transaction->t_tid;
		else
			tid = journal->j_commit_sequence;
3742
		read_unlock(&journal->j_state_lock);
3743 3744 3745 3746
		ei->i_sync_tid = tid;
		ei->i_datasync_tid = tid;
	}

3747
	if (EXT4_INODE_SIZE(inode->i_sb) > EXT4_GOOD_OLD_INODE_SIZE) {
3748
		ei->i_extra_isize = le16_to_cpu(raw_inode->i_extra_isize);
3749
		if (EXT4_GOOD_OLD_INODE_SIZE + ei->i_extra_isize >
3750
		    EXT4_INODE_SIZE(inode->i_sb)) {
3751
			ret = -EIO;
3752
			goto bad_inode;
3753
		}
3754 3755
		if (ei->i_extra_isize == 0) {
			/* The extra space is currently unused. Use it. */
3756 3757
			ei->i_extra_isize = sizeof(struct ext4_inode) -
					    EXT4_GOOD_OLD_INODE_SIZE;
3758 3759
		} else {
			__le32 *magic = (void *)raw_inode +
3760
					EXT4_GOOD_OLD_INODE_SIZE +
3761
					ei->i_extra_isize;
3762
			if (*magic == cpu_to_le32(EXT4_XATTR_MAGIC))
3763
				ext4_set_inode_state(inode, EXT4_STATE_XATTR);
3764 3765 3766 3767
		}
	} else
		ei->i_extra_isize = 0;

3768 3769 3770 3771 3772
	EXT4_INODE_GET_XTIME(i_ctime, inode, raw_inode);
	EXT4_INODE_GET_XTIME(i_mtime, inode, raw_inode);
	EXT4_INODE_GET_XTIME(i_atime, inode, raw_inode);
	EXT4_EINODE_GET_XTIME(i_crtime, ei, raw_inode);

3773 3774 3775 3776 3777 3778 3779
	inode->i_version = le32_to_cpu(raw_inode->i_disk_version);
	if (EXT4_INODE_SIZE(inode->i_sb) > EXT4_GOOD_OLD_INODE_SIZE) {
		if (EXT4_FITS_IN_INODE(raw_inode, ei, i_version_hi))
			inode->i_version |=
			(__u64)(le32_to_cpu(raw_inode->i_version_hi)) << 32;
	}

3780
	ret = 0;
3781
	if (ei->i_file_acl &&
3782
	    !ext4_data_block_valid(EXT4_SB(sb), ei->i_file_acl, 1)) {
3783 3784
		EXT4_ERROR_INODE(inode, "bad extended attribute block %llu",
				 ei->i_file_acl);
3785 3786
		ret = -EIO;
		goto bad_inode;
3787
	} else if (ext4_test_inode_flag(inode, EXT4_INODE_EXTENTS)) {
3788 3789 3790 3791 3792
		if (S_ISREG(inode->i_mode) || S_ISDIR(inode->i_mode) ||
		    (S_ISLNK(inode->i_mode) &&
		     !ext4_inode_is_fast_symlink(inode)))
			/* Validate extent which is part of inode */
			ret = ext4_ext_check_inode(inode);
3793
	} else if (S_ISREG(inode->i_mode) || S_ISDIR(inode->i_mode) ||
3794 3795
		   (S_ISLNK(inode->i_mode) &&
		    !ext4_inode_is_fast_symlink(inode))) {
3796
		/* Validate block references which are part of inode */
3797
		ret = ext4_ind_check_inode(inode);
3798
	}
3799
	if (ret)
3800
		goto bad_inode;
3801

3802
	if (S_ISREG(inode->i_mode)) {
3803 3804 3805
		inode->i_op = &ext4_file_inode_operations;
		inode->i_fop = &ext4_file_operations;
		ext4_set_aops(inode);
3806
	} else if (S_ISDIR(inode->i_mode)) {
3807 3808
		inode->i_op = &ext4_dir_inode_operations;
		inode->i_fop = &ext4_dir_operations;
3809
	} else if (S_ISLNK(inode->i_mode)) {
3810
		if (ext4_inode_is_fast_symlink(inode)) {
3811
			inode->i_op = &ext4_fast_symlink_inode_operations;
3812 3813 3814
			nd_terminate_link(ei->i_data, inode->i_size,
				sizeof(ei->i_data) - 1);
		} else {
3815 3816
			inode->i_op = &ext4_symlink_inode_operations;
			ext4_set_aops(inode);
3817
		}
3818 3819
	} else if (S_ISCHR(inode->i_mode) || S_ISBLK(inode->i_mode) ||
	      S_ISFIFO(inode->i_mode) || S_ISSOCK(inode->i_mode)) {
3820
		inode->i_op = &ext4_special_inode_operations;
3821 3822 3823 3824 3825 3826
		if (raw_inode->i_block[0])
			init_special_inode(inode, inode->i_mode,
			   old_decode_dev(le32_to_cpu(raw_inode->i_block[0])));
		else
			init_special_inode(inode, inode->i_mode,
			   new_decode_dev(le32_to_cpu(raw_inode->i_block[1])));
3827 3828
	} else {
		ret = -EIO;
3829
		EXT4_ERROR_INODE(inode, "bogus i_mode (%o)", inode->i_mode);
3830
		goto bad_inode;
3831
	}
3832
	brelse(iloc.bh);
3833
	ext4_set_inode_flags(inode);
3834 3835
	unlock_new_inode(inode);
	return inode;
3836 3837

bad_inode:
3838
	brelse(iloc.bh);
3839 3840
	iget_failed(inode);
	return ERR_PTR(ret);
3841 3842
}

3843 3844 3845 3846 3847 3848 3849 3850 3851 3852 3853 3854 3855
static int ext4_inode_blocks_set(handle_t *handle,
				struct ext4_inode *raw_inode,
				struct ext4_inode_info *ei)
{
	struct inode *inode = &(ei->vfs_inode);
	u64 i_blocks = inode->i_blocks;
	struct super_block *sb = inode->i_sb;

	if (i_blocks <= ~0U) {
		/*
		 * i_blocks can be represnted in a 32 bit variable
		 * as multiple of 512 bytes
		 */
Aneesh Kumar K.V's avatar
Aneesh Kumar K.V committed
3856
		raw_inode->i_blocks_lo   = cpu_to_le32(i_blocks);
3857
		raw_inode->i_blocks_high = 0;
3858
		ext4_clear_inode_flag(inode, EXT4_INODE_HUGE_FILE);
3859 3860 3861 3862 3863 3864
		return 0;
	}
	if (!EXT4_HAS_RO_COMPAT_FEATURE(sb, EXT4_FEATURE_RO_COMPAT_HUGE_FILE))
		return -EFBIG;

	if (i_blocks <= 0xffffffffffffULL) {
3865 3866 3867 3868
		/*
		 * i_blocks can be represented in a 48 bit variable
		 * as multiple of 512 bytes
		 */
Aneesh Kumar K.V's avatar
Aneesh Kumar K.V committed
3869
		raw_inode->i_blocks_lo   = cpu_to_le32(i_blocks);
3870
		raw_inode->i_blocks_high = cpu_to_le16(i_blocks >> 32);
3871
		ext4_clear_inode_flag(inode, EXT4_INODE_HUGE_FILE);
3872
	} else {
3873
		ext4_set_inode_flag(inode, EXT4_INODE_HUGE_FILE);
Aneesh Kumar K.V's avatar
Aneesh Kumar K.V committed
3874 3875 3876 3877
		/* i_block is stored in file system block size */
		i_blocks = i_blocks >> (inode->i_blkbits - 9);
		raw_inode->i_blocks_lo   = cpu_to_le32(i_blocks);
		raw_inode->i_blocks_high = cpu_to_le16(i_blocks >> 32);
3878
	}
3879
	return 0;
3880 3881
}

3882 3883 3884 3885 3886 3887 3888
/*
 * Post the struct inode info into an on-disk inode location in the
 * buffer-cache.  This gobbles the caller's reference to the
 * buffer_head in the inode location struct.
 *
 * The caller must have write access to iloc->bh.
 */
3889
static int ext4_do_update_inode(handle_t *handle,
3890
				struct inode *inode,
3891
				struct ext4_iloc *iloc)
3892
{
3893 3894
	struct ext4_inode *raw_inode = ext4_raw_inode(iloc);
	struct ext4_inode_info *ei = EXT4_I(inode);
3895 3896
	struct buffer_head *bh = iloc->bh;
	int err = 0, rc, block;
3897
	int need_datasync = 0;
3898 3899 3900

	/* For fields not not tracking in the in-memory inode,
	 * initialise them to zero for new inodes. */
3901
	if (ext4_test_inode_state(inode, EXT4_STATE_NEW))
3902
		memset(raw_inode, 0, EXT4_SB(inode->i_sb)->s_inode_size);
3903

3904
	ext4_get_inode_flags(ei);
3905
	raw_inode->i_mode = cpu_to_le16(inode->i_mode);
3906
	if (!(test_opt(inode->i_sb, NO_UID32))) {
3907 3908 3909 3910 3911 3912
		raw_inode->i_uid_low = cpu_to_le16(low_16_bits(inode->i_uid));
		raw_inode->i_gid_low = cpu_to_le16(low_16_bits(inode->i_gid));
/*
 * Fix up interoperability with old kernels. Otherwise, old inodes get
 * re-used with the upper 16 bits of the uid/gid intact
 */
3913
		if (!ei->i_dtime) {
3914 3915 3916 3917 3918 3919 3920 3921 3922 3923 3924 3925 3926 3927 3928 3929 3930
			raw_inode->i_uid_high =
				cpu_to_le16(high_16_bits(inode->i_uid));
			raw_inode->i_gid_high =
				cpu_to_le16(high_16_bits(inode->i_gid));
		} else {
			raw_inode->i_uid_high = 0;
			raw_inode->i_gid_high = 0;
		}
	} else {
		raw_inode->i_uid_low =
			cpu_to_le16(fs_high2lowuid(inode->i_uid));
		raw_inode->i_gid_low =
			cpu_to_le16(fs_high2lowgid(inode->i_gid));
		raw_inode->i_uid_high = 0;
		raw_inode->i_gid_high = 0;
	}
	raw_inode->i_links_count = cpu_to_le16(inode->i_nlink);
3931 3932 3933 3934 3935 3936

	EXT4_INODE_SET_XTIME(i_ctime, inode, raw_inode);
	EXT4_INODE_SET_XTIME(i_mtime, inode, raw_inode);
	EXT4_INODE_SET_XTIME(i_atime, inode, raw_inode);
	EXT4_EINODE_SET_XTIME(i_crtime, ei, raw_inode);

3937 3938
	if (ext4_inode_blocks_set(handle, raw_inode, ei))
		goto out_brelse;
3939
	raw_inode->i_dtime = cpu_to_le32(ei->i_dtime);
3940
	raw_inode->i_flags = cpu_to_le32(ei->i_flags & 0xFFFFFFFF);
3941 3942
	if (EXT4_SB(inode->i_sb)->s_es->s_creator_os !=
	    cpu_to_le32(EXT4_OS_HURD))
3943 3944
		raw_inode->i_file_acl_high =
			cpu_to_le16(ei->i_file_acl >> 32);
3945
	raw_inode->i_file_acl_lo = cpu_to_le32(ei->i_file_acl);
3946 3947 3948 3949
	if (ei->i_disksize != ext4_isize(raw_inode)) {
		ext4_isize_set(raw_inode, ei->i_disksize);
		need_datasync = 1;
	}
3950 3951 3952 3953 3954 3955 3956 3957 3958 3959 3960 3961 3962 3963 3964
	if (ei->i_disksize > 0x7fffffffULL) {
		struct super_block *sb = inode->i_sb;
		if (!EXT4_HAS_RO_COMPAT_FEATURE(sb,
				EXT4_FEATURE_RO_COMPAT_LARGE_FILE) ||
				EXT4_SB(sb)->s_es->s_rev_level ==
				cpu_to_le32(EXT4_GOOD_OLD_REV)) {
			/* If this is the first large file
			 * created, add a flag to the superblock.
			 */
			err = ext4_journal_get_write_access(handle,
					EXT4_SB(sb)->s_sbh);
			if (err)
				goto out_brelse;
			ext4_update_dynamic_rev(sb);
			EXT4_SET_RO_COMPAT_FEATURE(sb,
3965
					EXT4_FEATURE_RO_COMPAT_LARGE_FILE);
3966
			ext4_handle_sync(handle);
3967
			err = ext4_handle_dirty_super(handle, sb);
3968 3969 3970 3971 3972 3973 3974 3975 3976 3977 3978 3979 3980 3981
		}
	}
	raw_inode->i_generation = cpu_to_le32(inode->i_generation);
	if (S_ISCHR(inode->i_mode) || S_ISBLK(inode->i_mode)) {
		if (old_valid_dev(inode->i_rdev)) {
			raw_inode->i_block[0] =
				cpu_to_le32(old_encode_dev(inode->i_rdev));
			raw_inode->i_block[1] = 0;
		} else {
			raw_inode->i_block[0] = 0;
			raw_inode->i_block[1] =
				cpu_to_le32(new_encode_dev(inode->i_rdev));
			raw_inode->i_block[2] = 0;
		}
3982 3983 3984
	} else
		for (block = 0; block < EXT4_N_BLOCKS; block++)
			raw_inode->i_block[block] = ei->i_data[block];
3985

3986 3987 3988 3989 3990
	raw_inode->i_disk_version = cpu_to_le32(inode->i_version);
	if (ei->i_extra_isize) {
		if (EXT4_FITS_IN_INODE(raw_inode, ei, i_version_hi))
			raw_inode->i_version_hi =
			cpu_to_le32(inode->i_version >> 32);
3991
		raw_inode->i_extra_isize = cpu_to_le16(ei->i_extra_isize);
3992 3993
	}

3994
	BUFFER_TRACE(bh, "call ext4_handle_dirty_metadata");
3995
	rc = ext4_handle_dirty_metadata(handle, NULL, bh);
3996 3997
	if (!err)
		err = rc;
3998
	ext4_clear_inode_state(inode, EXT4_STATE_NEW);
3999

4000
	ext4_update_inode_fsync_trans(handle, inode, need_datasync);
4001
out_brelse:
4002
	brelse(bh);
4003
	ext4_std_error(inode->i_sb, err);
4004 4005 4006 4007
	return err;
}

/*
4008
 * ext4_write_inode()
4009 4010 4011 4012 4013 4014 4015 4016 4017 4018 4019 4020 4021 4022 4023 4024
 *
 * We are called from a few places:
 *
 * - Within generic_file_write() for O_SYNC files.
 *   Here, there will be no transaction running. We wait for any running
 *   trasnaction to commit.
 *
 * - Within sys_sync(), kupdate and such.
 *   We wait on commit, if tol to.
 *
 * - Within prune_icache() (PF_MEMALLOC == true)
 *   Here we simply return.  We can't afford to block kswapd on the
 *   journal commit.
 *
 * In all cases it is actually safe for us to return without doing anything,
 * because the inode has been copied into a raw inode buffer in
4025
 * ext4_mark_inode_dirty().  This is a correctness thing for O_SYNC and for
4026 4027 4028 4029 4030 4031 4032 4033 4034 4035 4036 4037 4038 4039 4040 4041
 * knfsd.
 *
 * Note that we are absolutely dependent upon all inode dirtiers doing the
 * right thing: they *must* call mark_inode_dirty() after dirtying info in
 * which we are interested.
 *
 * It would be a bug for them to not do this.  The code:
 *
 *	mark_inode_dirty(inode)
 *	stuff();
 *	inode->i_size = expr;
 *
 * is in error because a kswapd-driven write_inode() could occur while
 * `stuff()' is running, and the new i_size will be lost.  Plus the inode
 * will no longer be on the superblock's dirty inode list.
 */
4042
int ext4_write_inode(struct inode *inode, struct writeback_control *wbc)
4043
{
4044 4045
	int err;

4046 4047 4048
	if (current->flags & PF_MEMALLOC)
		return 0;

4049 4050 4051 4052 4053 4054
	if (EXT4_SB(inode->i_sb)->s_journal) {
		if (ext4_journal_current_handle()) {
			jbd_debug(1, "called recursively, non-PF_MEMALLOC!\n");
			dump_stack();
			return -EIO;
		}
4055

4056
		if (wbc->sync_mode != WB_SYNC_ALL)
4057 4058 4059 4060 4061
			return 0;

		err = ext4_force_commit(inode->i_sb);
	} else {
		struct ext4_iloc iloc;
4062

4063
		err = __ext4_get_inode_loc(inode, &iloc, 0);
4064 4065
		if (err)
			return err;
4066
		if (wbc->sync_mode == WB_SYNC_ALL)
4067 4068
			sync_dirty_buffer(iloc.bh);
		if (buffer_req(iloc.bh) && !buffer_uptodate(iloc.bh)) {
4069 4070
			EXT4_ERROR_INODE_BLOCK(inode, iloc.bh->b_blocknr,
					 "IO error syncing inode");
4071 4072
			err = -EIO;
		}
4073
		brelse(iloc.bh);
4074 4075
	}
	return err;
4076 4077 4078
}

/*
4079
 * ext4_setattr()
4080 4081 4082 4083 4084 4085 4086 4087 4088 4089 4090 4091 4092
 *
 * Called from notify_change.
 *
 * We want to trap VFS attempts to truncate the file as soon as
 * possible.  In particular, we want to make sure that when the VFS
 * shrinks i_size, we put the inode on the orphan list and modify
 * i_disksize immediately, so that during the subsequent flushing of
 * dirty pages and freeing of disk blocks, we can guarantee that any
 * commit will leave the blocks being flushed in an unused state on
 * disk.  (On recovery, the inode will get truncated and the blocks will
 * be freed, so we have a strong guarantee that no future commit will
 * leave these blocks visible to the user.)
 *
4093 4094 4095 4096 4097 4098 4099 4100
 * Another thing we have to assure is that if we are in ordered mode
 * and inode is still attached to the committing transaction, we must
 * we start writeout of all the dirty pages which are being truncated.
 * This way we are sure that all the data written in the previous
 * transaction are already on disk (truncate waits for pages under
 * writeback).
 *
 * Called with inode->i_mutex down.
4101
 */
4102
int ext4_setattr(struct dentry *dentry, struct iattr *attr)
4103 4104 4105
{
	struct inode *inode = dentry->d_inode;
	int error, rc = 0;
4106
	int orphan = 0;
4107 4108 4109 4110 4111 4112
	const unsigned int ia_valid = attr->ia_valid;

	error = inode_change_ok(inode, attr);
	if (error)
		return error;

4113
	if (is_quota_modification(inode, attr))
4114
		dquot_initialize(inode);
4115 4116 4117 4118 4119 4120
	if ((ia_valid & ATTR_UID && attr->ia_uid != inode->i_uid) ||
		(ia_valid & ATTR_GID && attr->ia_gid != inode->i_gid)) {
		handle_t *handle;

		/* (user+group)*(old+new) structure, inode write (sb,
		 * inode block, ? - but truncate inode update has it) */
Dmitry Monakhov's avatar
Dmitry Monakhov committed
4121
		handle = ext4_journal_start(inode, (EXT4_MAXQUOTAS_INIT_BLOCKS(inode->i_sb)+
4122
					EXT4_MAXQUOTAS_DEL_BLOCKS(inode->i_sb))+3);
4123 4124 4125 4126
		if (IS_ERR(handle)) {
			error = PTR_ERR(handle);
			goto err_out;
		}
4127
		error = dquot_transfer(inode, attr);
4128
		if (error) {
4129
			ext4_journal_stop(handle);
4130 4131 4132 4133 4134 4135 4136 4137
			return error;
		}
		/* Update corresponding info in inode so that everything is in
		 * one transaction */
		if (attr->ia_valid & ATTR_UID)
			inode->i_uid = attr->ia_uid;
		if (attr->ia_valid & ATTR_GID)
			inode->i_gid = attr->ia_gid;
4138 4139
		error = ext4_mark_inode_dirty(handle, inode);
		ext4_journal_stop(handle);
4140 4141
	}

4142
	if (attr->ia_valid & ATTR_SIZE) {
4143 4144
		inode_dio_wait(inode);

4145
		if (!(ext4_test_inode_flag(inode, EXT4_INODE_EXTENTS))) {
4146 4147
			struct ext4_sb_info *sbi = EXT4_SB(inode->i_sb);

4148 4149
			if (attr->ia_size > sbi->s_bitmap_maxbytes)
				return -EFBIG;
4150 4151 4152
		}
	}

4153
	if (S_ISREG(inode->i_mode) &&
4154
	    attr->ia_valid & ATTR_SIZE &&
4155
	    (attr->ia_size < inode->i_size)) {
4156 4157
		handle_t *handle;

4158
		handle = ext4_journal_start(inode, 3);
4159 4160 4161 4162
		if (IS_ERR(handle)) {
			error = PTR_ERR(handle);
			goto err_out;
		}
4163 4164 4165 4166
		if (ext4_handle_valid(handle)) {
			error = ext4_orphan_add(handle, inode);
			orphan = 1;
		}
4167 4168
		EXT4_I(inode)->i_disksize = attr->ia_size;
		rc = ext4_mark_inode_dirty(handle, inode);
4169 4170
		if (!error)
			error = rc;
4171
		ext4_journal_stop(handle);
4172 4173 4174 4175 4176 4177 4178 4179 4180 4181 4182 4183

		if (ext4_should_order_data(inode)) {
			error = ext4_begin_ordered_truncate(inode,
							    attr->ia_size);
			if (error) {
				/* Do as much error cleanup as possible */
				handle = ext4_journal_start(inode, 3);
				if (IS_ERR(handle)) {
					ext4_orphan_del(NULL, inode);
					goto err_out;
				}
				ext4_orphan_del(handle, inode);
4184
				orphan = 0;
4185 4186 4187 4188
				ext4_journal_stop(handle);
				goto err_out;
			}
		}
4189 4190
	}

4191
	if (attr->ia_valid & ATTR_SIZE) {
4192
		if (attr->ia_size != i_size_read(inode))
4193
			truncate_setsize(inode, attr->ia_size);
4194
		ext4_truncate(inode);
4195
	}
4196

Christoph Hellwig's avatar
Christoph Hellwig committed
4197 4198 4199 4200 4201 4202 4203 4204 4205
	if (!rc) {
		setattr_copy(inode, attr);
		mark_inode_dirty(inode);
	}

	/*
	 * If the call to ext4_truncate failed to get a transaction handle at
	 * all, we need to clean up the in-core orphan list manually.
	 */
4206
	if (orphan && inode->i_nlink)
4207
		ext4_orphan_del(NULL, inode);
4208 4209

	if (!rc && (ia_valid & ATTR_MODE))
4210
		rc = ext4_acl_chmod(inode);
4211 4212

err_out:
4213
	ext4_std_error(inode->i_sb, error);
4214 4215 4216 4217 4218
	if (!error)
		error = rc;
	return error;
}

4219 4220 4221 4222
int ext4_getattr(struct vfsmount *mnt, struct dentry *dentry,
		 struct kstat *stat)
{
	struct inode *inode;
4223
	unsigned long long delalloc_blocks;
4224 4225 4226 4227 4228 4229 4230 4231 4232 4233 4234 4235 4236 4237 4238 4239

	inode = dentry->d_inode;
	generic_fillattr(inode, stat);

	/*
	 * We can't update i_blocks if the block allocation is delayed
	 * otherwise in the case of system crash before the real block
	 * allocation is done, we will have i_blocks inconsistent with
	 * on-disk file blocks.
	 * We always keep i_blocks updated together with real
	 * allocation. But to not confuse with user, stat
	 * will return the blocks that include the delayed allocation
	 * blocks for this file.
	 */
	delalloc_blocks = EXT4_I(inode)->i_reserved_data_blocks;

4240
	stat->blocks += delalloc_blocks << (inode->i_sb->s_blocksize_bits-9);
4241 4242
	return 0;
}
4243

4244 4245
static int ext4_index_trans_blocks(struct inode *inode, int nrblocks, int chunk)
{
4246
	if (!(ext4_test_inode_flag(inode, EXT4_INODE_EXTENTS)))
4247
		return ext4_ind_trans_blocks(inode, nrblocks, chunk);
4248
	return ext4_ext_index_trans_blocks(inode, nrblocks, chunk);
4249
}
4250

4251
/*
4252 4253 4254
 * Account for index blocks, block groups bitmaps and block group
 * descriptor blocks if modify datablocks and index blocks
 * worse case, the indexs blocks spread over different block groups
4255
 *
4256
 * If datablocks are discontiguous, they are possible to spread over
4257
 * different block groups too. If they are contiuguous, with flexbg,
4258
 * they could still across block group boundary.
4259
 *
4260 4261
 * Also account for superblock, inode, quota and xattr blocks
 */
4262
static int ext4_meta_trans_blocks(struct inode *inode, int nrblocks, int chunk)
4263
{
4264 4265
	ext4_group_t groups, ngroups = ext4_get_groups_count(inode->i_sb);
	int gdpblocks;
4266 4267 4268 4269 4270 4271 4272 4273 4274 4275 4276 4277 4278 4279 4280 4281 4282 4283 4284 4285 4286 4287 4288 4289 4290 4291
	int idxblocks;
	int ret = 0;

	/*
	 * How many index blocks need to touch to modify nrblocks?
	 * The "Chunk" flag indicating whether the nrblocks is
	 * physically contiguous on disk
	 *
	 * For Direct IO and fallocate, they calls get_block to allocate
	 * one single extent at a time, so they could set the "Chunk" flag
	 */
	idxblocks = ext4_index_trans_blocks(inode, nrblocks, chunk);

	ret = idxblocks;

	/*
	 * Now let's see how many group bitmaps and group descriptors need
	 * to account
	 */
	groups = idxblocks;
	if (chunk)
		groups += 1;
	else
		groups += nrblocks;

	gdpblocks = groups;
4292 4293
	if (groups > ngroups)
		groups = ngroups;
4294 4295 4296 4297 4298 4299 4300 4301 4302 4303 4304 4305 4306
	if (groups > EXT4_SB(inode->i_sb)->s_gdb_count)
		gdpblocks = EXT4_SB(inode->i_sb)->s_gdb_count;

	/* bitmaps and block group descriptor blocks */
	ret += groups + gdpblocks;

	/* Blocks for super block, inode, quota and xattr blocks */
	ret += EXT4_META_TRANS_BLOCKS(inode->i_sb);

	return ret;
}

/*
Lucas De Marchi's avatar
Lucas De Marchi committed
4307
 * Calculate the total number of credits to reserve to fit
4308 4309
 * the modification of a single pages into a single transaction,
 * which may include multiple chunks of block allocations.
4310
 *
4311
 * This could be called via ext4_write_begin()
4312
 *
4313
 * We need to consider the worse case, when
4314
 * one new block per extent.
4315
 */
4316
int ext4_writepage_trans_blocks(struct inode *inode)
4317
{
4318
	int bpp = ext4_journal_blocks_per_page(inode);
4319 4320
	int ret;

4321
	ret = ext4_meta_trans_blocks(inode, bpp, 0);
4322

4323
	/* Account for data blocks for journalled mode */
4324
	if (ext4_should_journal_data(inode))
4325
		ret += bpp;
4326 4327
	return ret;
}
4328 4329 4330 4331 4332

/*
 * Calculate the journal credits for a chunk of data modification.
 *
 * This is called from DIO, fallocate or whoever calling
4333
 * ext4_map_blocks() to map/allocate a chunk of contiguous disk blocks.
4334 4335 4336 4337 4338 4339 4340 4341 4342
 *
 * journal buffers for data blocks are not included here, as DIO
 * and fallocate do no need to journal data buffers.
 */
int ext4_chunk_trans_blocks(struct inode *inode, int nrblocks)
{
	return ext4_meta_trans_blocks(inode, nrblocks, 1);
}

4343
/*
4344
 * The caller must have previously called ext4_reserve_inode_write().
4345 4346
 * Give this, we know that the caller already has write access to iloc->bh.
 */
4347
int ext4_mark_iloc_dirty(handle_t *handle,
4348
			 struct inode *inode, struct ext4_iloc *iloc)
4349 4350 4351
{
	int err = 0;

4352
	if (IS_I_VERSION(inode))
4353 4354
		inode_inc_iversion(inode);

4355 4356 4357
	/* the do_update_inode consumes one bh->b_count */
	get_bh(iloc->bh);

4358
	/* ext4_do_update_inode() does jbd2_journal_dirty_metadata */
4359
	err = ext4_do_update_inode(handle, inode, iloc);
4360 4361 4362 4363 4364 4365 4366 4367 4368 4369
	put_bh(iloc->bh);
	return err;
}

/*
 * On success, We end up with an outstanding reference count against
 * iloc->bh.  This _must_ be cleaned up later.
 */

int
4370 4371
ext4_reserve_inode_write(handle_t *handle, struct inode *inode,
			 struct ext4_iloc *iloc)
4372
{
4373 4374 4375 4376 4377 4378 4379 4380 4381
	int err;

	err = ext4_get_inode_loc(inode, iloc);
	if (!err) {
		BUFFER_TRACE(iloc->bh, "get_write_access");
		err = ext4_journal_get_write_access(handle, iloc->bh);
		if (err) {
			brelse(iloc->bh);
			iloc->bh = NULL;
4382 4383
		}
	}
4384
	ext4_std_error(inode->i_sb, err);
4385 4386 4387
	return err;
}

4388 4389 4390 4391
/*
 * Expand an inode by new_extra_isize bytes.
 * Returns 0 on success or negative error number on failure.
 */
4392 4393 4394 4395
static int ext4_expand_extra_isize(struct inode *inode,
				   unsigned int new_extra_isize,
				   struct ext4_iloc iloc,
				   handle_t *handle)
4396 4397 4398 4399 4400 4401 4402 4403 4404 4405 4406 4407
{
	struct ext4_inode *raw_inode;
	struct ext4_xattr_ibody_header *header;

	if (EXT4_I(inode)->i_extra_isize >= new_extra_isize)
		return 0;

	raw_inode = ext4_raw_inode(&iloc);

	header = IHDR(inode, raw_inode);

	/* No extended attributes present */
4408 4409
	if (!ext4_test_inode_state(inode, EXT4_STATE_XATTR) ||
	    header->h_magic != cpu_to_le32(EXT4_XATTR_MAGIC)) {
4410 4411 4412 4413 4414 4415 4416 4417 4418 4419 4420
		memset((void *)raw_inode + EXT4_GOOD_OLD_INODE_SIZE, 0,
			new_extra_isize);
		EXT4_I(inode)->i_extra_isize = new_extra_isize;
		return 0;
	}

	/* try to expand with EAs present */
	return ext4_expand_extra_isize_ea(inode, new_extra_isize,
					  raw_inode, handle);
}

4421 4422 4423 4424 4425 4426 4427 4428 4429 4430 4431 4432 4433 4434 4435 4436 4437 4438 4439 4440 4441
/*
 * What we do here is to mark the in-core inode as clean with respect to inode
 * dirtiness (it may still be data-dirty).
 * This means that the in-core inode may be reaped by prune_icache
 * without having to perform any I/O.  This is a very good thing,
 * because *any* task may call prune_icache - even ones which
 * have a transaction open against a different journal.
 *
 * Is this cheating?  Not really.  Sure, we haven't written the
 * inode out, but prune_icache isn't a user-visible syncing function.
 * Whenever the user wants stuff synced (sys_sync, sys_msync, sys_fsync)
 * we start and wait on commits.
 *
 * Is this efficient/effective?  Well, we're being nice to the system
 * by cleaning up our inodes proactively so they can be reaped
 * without I/O.  But we are potentially leaving up to five seconds'
 * worth of inodes floating about which prune_icache wants us to
 * write out.  One way to fix that would be to get prune_icache()
 * to do a write_super() to free up some memory.  It has the desired
 * effect.
 */
4442
int ext4_mark_inode_dirty(handle_t *handle, struct inode *inode)
4443
{
4444
	struct ext4_iloc iloc;
4445 4446 4447
	struct ext4_sb_info *sbi = EXT4_SB(inode->i_sb);
	static unsigned int mnt_count;
	int err, ret;
4448 4449

	might_sleep();
4450
	trace_ext4_mark_inode_dirty(inode, _RET_IP_);
4451
	err = ext4_reserve_inode_write(handle, inode, &iloc);
4452 4453
	if (ext4_handle_valid(handle) &&
	    EXT4_I(inode)->i_extra_isize < sbi->s_want_extra_isize &&
4454
	    !ext4_test_inode_state(inode, EXT4_STATE_NO_EXPAND)) {
4455 4456 4457 4458 4459 4460 4461 4462 4463 4464 4465 4466 4467
		/*
		 * We need extra buffer credits since we may write into EA block
		 * with this same handle. If journal_extend fails, then it will
		 * only result in a minor loss of functionality for that inode.
		 * If this is felt to be critical, then e2fsck should be run to
		 * force a large enough s_min_extra_isize.
		 */
		if ((jbd2_journal_extend(handle,
			     EXT4_DATA_TRANS_BLOCKS(inode->i_sb))) == 0) {
			ret = ext4_expand_extra_isize(inode,
						      sbi->s_want_extra_isize,
						      iloc, handle);
			if (ret) {
4468 4469
				ext4_set_inode_state(inode,
						     EXT4_STATE_NO_EXPAND);
Aneesh Kumar K.V's avatar
Aneesh Kumar K.V committed
4470 4471
				if (mnt_count !=
					le16_to_cpu(sbi->s_es->s_mnt_count)) {
4472
					ext4_warning(inode->i_sb,
4473 4474 4475
					"Unable to expand inode %lu. Delete"
					" some EAs or run e2fsck.",
					inode->i_ino);
Aneesh Kumar K.V's avatar
Aneesh Kumar K.V committed
4476 4477
					mnt_count =
					  le16_to_cpu(sbi->s_es->s_mnt_count);
4478 4479 4480 4481
				}
			}
		}
	}
4482
	if (!err)
4483
		err = ext4_mark_iloc_dirty(handle, inode, &iloc);
4484 4485 4486 4487
	return err;
}

/*
4488
 * ext4_dirty_inode() is called from __mark_inode_dirty()
4489 4490 4491 4492 4493
 *
 * We're really interested in the case where a file is being extended.
 * i_size has been changed by generic_commit_write() and we thus need
 * to include the updated inode in the current transaction.
 *
4494
 * Also, dquot_alloc_block() will always dirty the inode when blocks
4495 4496 4497 4498 4499 4500
 * are allocated to the file.
 *
 * If the inode is marked synchronous, we don't honour that here - doing
 * so would cause a commit on atime updates, which we don't bother doing.
 * We handle synchronous inodes at the highest possible level.
 */
4501
void ext4_dirty_inode(struct inode *inode, int flags)
4502 4503 4504
{
	handle_t *handle;

4505
	handle = ext4_journal_start(inode, 2);
4506 4507
	if (IS_ERR(handle))
		goto out;
4508 4509 4510

	ext4_mark_inode_dirty(handle, inode);

4511
	ext4_journal_stop(handle);
4512 4513 4514 4515 4516 4517 4518 4519
out:
	return;
}

#if 0
/*
 * Bind an inode's backing buffer_head into this transaction, to prevent
 * it from being flushed to disk early.  Unlike
4520
 * ext4_reserve_inode_write, this leaves behind no bh reference and
4521 4522 4523
 * returns no iloc structure, so the caller needs to repeat the iloc
 * lookup to mark the inode dirty later.
 */
4524
static int ext4_pin_inode(handle_t *handle, struct inode *inode)
4525
{
4526
	struct ext4_iloc iloc;
4527 4528 4529

	int err = 0;
	if (handle) {
4530
		err = ext4_get_inode_loc(inode, &iloc);
4531 4532
		if (!err) {
			BUFFER_TRACE(iloc.bh, "get_write_access");
4533
			err = jbd2_journal_get_write_access(handle, iloc.bh);
4534
			if (!err)
4535
				err = ext4_handle_dirty_metadata(handle,
4536
								 NULL,
4537
								 iloc.bh);
4538 4539 4540
			brelse(iloc.bh);
		}
	}
4541
	ext4_std_error(inode->i_sb, err);
4542 4543 4544 4545
	return err;
}
#endif

4546
int ext4_change_inode_journal_flag(struct inode *inode, int val)
4547 4548 4549 4550 4551 4552 4553 4554 4555 4556 4557 4558 4559 4560 4561
{
	journal_t *journal;
	handle_t *handle;
	int err;

	/*
	 * We have to be very careful here: changing a data block's
	 * journaling status dynamically is dangerous.  If we write a
	 * data block to the journal, change the status and then delete
	 * that block, we risk forgetting to revoke the old log record
	 * from the journal and so a subsequent replay can corrupt data.
	 * So, first we make sure that the journal is empty and that
	 * nobody is changing anything.
	 */

4562
	journal = EXT4_JOURNAL(inode);
4563 4564
	if (!journal)
		return 0;
4565
	if (is_journal_aborted(journal))
4566
		return -EROFS;
4567 4568 4569 4570 4571 4572 4573 4574 4575 4576 4577
	/* We have to allocate physical blocks for delalloc blocks
	 * before flushing journal. otherwise delalloc blocks can not
	 * be allocated any more. even more truncate on delalloc blocks
	 * could trigger BUG by flushing delalloc blocks in journal.
	 * There is no delalloc block in non-journal data mode.
	 */
	if (val && test_opt(inode->i_sb, DELALLOC)) {
		err = ext4_alloc_da_blocks(inode);
		if (err < 0)
			return err;
	}
4578

4579
	jbd2_journal_lock_updates(journal);
4580 4581 4582 4583 4584 4585 4586 4587 4588 4589

	/*
	 * OK, there are no updates running now, and all cached data is
	 * synced to disk.  We are now in a completely consistent state
	 * which doesn't have anything in the journal, and we know that
	 * no filesystem updates are running, so it is safe to modify
	 * the inode's in-core data-journaling state flag now.
	 */

	if (val)
4590
		ext4_set_inode_flag(inode, EXT4_INODE_JOURNAL_DATA);
4591 4592
	else {
		jbd2_journal_flush(journal);
4593
		ext4_clear_inode_flag(inode, EXT4_INODE_JOURNAL_DATA);
4594
	}
4595
	ext4_set_aops(inode);
4596

4597
	jbd2_journal_unlock_updates(journal);
4598 4599 4600

	/* Finally we can mark the inode as dirty. */

4601
	handle = ext4_journal_start(inode, 1);
4602 4603 4604
	if (IS_ERR(handle))
		return PTR_ERR(handle);

4605
	err = ext4_mark_inode_dirty(handle, inode);
4606
	ext4_handle_sync(handle);
4607 4608
	ext4_journal_stop(handle);
	ext4_std_error(inode->i_sb, err);
4609 4610 4611

	return err;
}
4612 4613 4614 4615 4616 4617

static int ext4_bh_unmapped(handle_t *handle, struct buffer_head *bh)
{
	return !buffer_mapped(bh);
}

4618
int ext4_page_mkwrite(struct vm_area_struct *vma, struct vm_fault *vmf)
4619
{
4620
	struct page *page = vmf->page;
4621 4622
	loff_t size;
	unsigned long len;
4623
	int ret;
4624 4625 4626
	struct file *file = vma->vm_file;
	struct inode *inode = file->f_path.dentry->d_inode;
	struct address_space *mapping = inode->i_mapping;
4627 4628 4629
	handle_t *handle;
	get_block_t *get_block;
	int retries = 0;
4630 4631

	/*
4632 4633
	 * This check is racy but catches the common case. We rely on
	 * __block_page_mkwrite() to do a reliable check.
4634
	 */
4635 4636 4637 4638 4639 4640 4641 4642 4643 4644 4645
	vfs_check_frozen(inode->i_sb, SB_FREEZE_WRITE);
	/* Delalloc case is easy... */
	if (test_opt(inode->i_sb, DELALLOC) &&
	    !ext4_should_journal_data(inode) &&
	    !ext4_nonda_switch(inode->i_sb)) {
		do {
			ret = __block_page_mkwrite(vma, vmf,
						   ext4_da_get_block_prep);
		} while (ret == -ENOSPC &&
		       ext4_should_retry_alloc(inode->i_sb, &retries));
		goto out_ret;
4646
	}
4647 4648

	lock_page(page);
4649 4650 4651 4652 4653 4654
	size = i_size_read(inode);
	/* Page got truncated from under us? */
	if (page->mapping != mapping || page_offset(page) > size) {
		unlock_page(page);
		ret = VM_FAULT_NOPAGE;
		goto out;
4655
	}
4656 4657 4658 4659 4660

	if (page->index == size >> PAGE_CACHE_SHIFT)
		len = size & ~PAGE_CACHE_MASK;
	else
		len = PAGE_CACHE_SIZE;
4661
	/*
4662 4663
	 * Return if we have all the buffers mapped. This avoids the need to do
	 * journal_start/journal_stop which can block and take a long time
4664
	 */
4665 4666
	if (page_has_buffers(page)) {
		if (!walk_page_buffers(NULL, page_buffers(page), 0, len, NULL,
4667
					ext4_bh_unmapped)) {
4668 4669 4670 4671
			/* Wait so that we don't change page under IO */
			wait_on_page_writeback(page);
			ret = VM_FAULT_LOCKED;
			goto out;
4672
		}
4673
	}
4674
	unlock_page(page);
4675 4676 4677 4678 4679 4680 4681 4682
	/* OK, we need to fill the hole... */
	if (ext4_should_dioread_nolock(inode))
		get_block = ext4_get_block_write;
	else
		get_block = ext4_get_block;
retry_alloc:
	handle = ext4_journal_start(inode, ext4_writepage_trans_blocks(inode));
	if (IS_ERR(handle)) {
4683
		ret = VM_FAULT_SIGBUS;
4684 4685 4686 4687 4688 4689 4690 4691
		goto out;
	}
	ret = __block_page_mkwrite(vma, vmf, get_block);
	if (!ret && ext4_should_journal_data(inode)) {
		if (walk_page_buffers(handle, page_buffers(page), 0,
			  PAGE_CACHE_SIZE, NULL, do_journal_get_write_access)) {
			unlock_page(page);
			ret = VM_FAULT_SIGBUS;
4692
			ext4_journal_stop(handle);
4693 4694 4695 4696 4697 4698 4699 4700 4701 4702
			goto out;
		}
		ext4_set_inode_state(inode, EXT4_STATE_JDATA);
	}
	ext4_journal_stop(handle);
	if (ret == -ENOSPC && ext4_should_retry_alloc(inode->i_sb, &retries))
		goto retry_alloc;
out_ret:
	ret = block_page_mkwrite_return(ret);
out:
4703 4704
	return ret;
}