filemap.c 45.6 KB
Newer Older
Linus Torvalds's avatar
Linus Torvalds committed
1 2 3 4 5 6 7 8 9 10 11
/*
 *	linux/mm/filemap.c
 *
 * Copyright (C) 1994-1999  Linus Torvalds
 */

/*
 * This file handles the generic file mmap semantics used by
 * most "normal" filesystems (but you don't /have/ to use this:
 * the NFS filesystem used to do this differently, for example)
 */
12
#include <linux/config.h>
Linus Torvalds's avatar
Linus Torvalds committed
13
#include <linux/module.h>
Linus Torvalds's avatar
Linus Torvalds committed
14
#include <linux/slab.h>
15 16
#include <linux/compiler.h>
#include <linux/fs.h>
17
#include <linux/aio.h>
Andrew Morton's avatar
Andrew Morton committed
18
#include <linux/kernel_stat.h>
19
#include <linux/mm.h>
Linus Torvalds's avatar
Linus Torvalds committed
20 21 22
#include <linux/mman.h>
#include <linux/pagemap.h>
#include <linux/file.h>
Andrew Morton's avatar
Andrew Morton committed
23
#include <linux/uio.h>
24
#include <linux/hash.h>
25
#include <linux/writeback.h>
26
#include <linux/pagevec.h>
27
#include <linux/blkdev.h>
28
#include <linux/security.h>
29 30 31
/*
 * This is needed for the following functions:
 *  - try_to_release_page
32
 *  - block_invalidatepage
33 34 35 36 37 38
 *  - page_has_buffers
 *  - generic_osync_inode
 *
 * FIXME: remove all knowledge of the buffer layer from this file
 */
#include <linux/buffer_head.h>
Linus Torvalds's avatar
Linus Torvalds committed
39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55

#include <asm/uaccess.h>
#include <asm/mman.h>

/*
 * Shared mappings implemented 30.11.1994. It's not fully working yet,
 * though.
 *
 * Shared mappings now work. 15.8.1995  Bruno.
 *
 * finished 'unifying' the page and buffer cache and SMP-threaded the
 * page-cache, 21.05.1999, Ingo Molnar <mingo@redhat.com>
 *
 * SMP-threaded pagemap-LRU 1999, Andrea Arcangeli <andrea@suse.de>
 */

/*
56
 * Lock ordering:
57 58
 *
 *  ->i_shared_lock		(vmtruncate)
59
 *    ->private_lock		(__free_pte->__set_page_dirty_buffers)
Andrew Morton's avatar
Andrew Morton committed
60 61 62
 *      ->swap_list_lock
 *        ->swap_device_lock	(exclusive_swap_page, others)
 *          ->mapping->page_lock
63 64
 *  ->inode_lock
 *    ->sb_lock			(fs/fs-writeback.c)
Andrew Morton's avatar
Andrew Morton committed
65
 *    ->mapping->page_lock	(__sync_single_inode)
Andrew Morton's avatar
Andrew Morton committed
66 67 68 69
 *  ->page_table_lock
 *    ->swap_device_lock	(try_to_unmap_one)
 *    ->private_lock		(try_to_unmap_one)
 *    ->page_lock		(try_to_unmap_one)
Linus Torvalds's avatar
Linus Torvalds committed
70 71
 */

72 73 74 75 76
/*
 * Remove a page from the page cache and free it. Caller has to make
 * sure the page is locked and that nobody else uses it - or that usage
 * is safe.  The caller must hold a write_lock on the mapping's page_lock.
 */
77
void __remove_from_page_cache(struct page *page)
Linus Torvalds's avatar
Linus Torvalds committed
78
{
79
	struct address_space *mapping = page->mapping;
Linus Torvalds's avatar
Linus Torvalds committed
80

81
	BUG_ON(PageDirty(page) && !PageSwapCache(page));
Linus Torvalds's avatar
Linus Torvalds committed
82

83
	radix_tree_delete(&mapping->page_tree, page->index);
Linus Torvalds's avatar
Linus Torvalds committed
84 85 86
	list_del(&page->list);
	page->mapping = NULL;

87
	mapping->nrpages--;
Andrew Morton's avatar
Andrew Morton committed
88
	dec_page_state(nr_pagecache);
Linus Torvalds's avatar
Linus Torvalds committed
89 90
}

91
void remove_from_page_cache(struct page *page)
Linus Torvalds's avatar
Linus Torvalds committed
92
{
93 94 95
	struct address_space *mapping = page->mapping;

	if (unlikely(!PageLocked(page)))
Linus Torvalds's avatar
Linus Torvalds committed
96 97
		PAGE_BUG(page);

98
	write_lock(&mapping->page_lock);
99
	__remove_from_page_cache(page);
100
	write_unlock(&mapping->page_lock);
Linus Torvalds's avatar
Linus Torvalds committed
101 102 103 104 105 106 107 108 109 110 111 112
}

static inline int sync_page(struct page *page)
{
	struct address_space *mapping = page->mapping;

	if (mapping && mapping->a_ops && mapping->a_ops->sync_page)
		return mapping->a_ops->sync_page(page);
	return 0;
}

/**
113 114
 * filemap_fdatawrite - start writeback against all of a mapping's dirty pages
 * @mapping: address space structure to write
115
 *
116 117 118 119
 * This is a "data integrity" operation, as opposed to a regular memory
 * cleansing writeback.  The difference between these two operations is that
 * if a dirty page/buffer is encountered, it must be waited upon, and not just
 * skipped over.
Linus Torvalds's avatar
Linus Torvalds committed
120
 */
121
int filemap_fdatawrite(struct address_space *mapping)
Linus Torvalds's avatar
Linus Torvalds committed
122
{
123
	int ret;
124 125 126 127
	struct writeback_control wbc = {
		.sync_mode = WB_SYNC_ALL,
		.nr_to_write = mapping->nrpages * 2,
	};
128

129 130 131
	if (mapping->backing_dev_info->memory_backed)
		return 0;

Andrew Morton's avatar
Andrew Morton committed
132 133 134
	write_lock(&mapping->page_lock);
	list_splice_init(&mapping->dirty_pages, &mapping->io_pages);
	write_unlock(&mapping->page_lock);
135
	ret = do_writepages(mapping, &wbc);
136
	return ret;
Linus Torvalds's avatar
Linus Torvalds committed
137 138 139
}

/**
140 141 142
 * filemap_fdatawait - walk the list of locked pages of the given address
 *                     space and wait for all of them.
 * @mapping: address space structure to wait for
Linus Torvalds's avatar
Linus Torvalds committed
143
 */
Andrew Morton's avatar
Andrew Morton committed
144
int filemap_fdatawait(struct address_space * mapping)
Linus Torvalds's avatar
Linus Torvalds committed
145
{
Andrew Morton's avatar
Andrew Morton committed
146
	int ret = 0;
147
	int progress;
Andrew Morton's avatar
Andrew Morton committed
148

149 150
restart:
	progress = 0;
151
	write_lock(&mapping->page_lock);
Linus Torvalds's avatar
Linus Torvalds committed
152
        while (!list_empty(&mapping->locked_pages)) {
153
		struct page *page;
Linus Torvalds's avatar
Linus Torvalds committed
154

155
		page = list_entry(mapping->locked_pages.next,struct page,list);
Linus Torvalds's avatar
Linus Torvalds committed
156
		list_del(&page->list);
157 158 159 160
		if (PageDirty(page))
			list_add(&page->list, &mapping->dirty_pages);
		else
			list_add(&page->list, &mapping->clean_pages);
Linus Torvalds's avatar
Linus Torvalds committed
161

162 163 164 165 166 167 168 169
		if (!PageWriteback(page)) {
			if (++progress > 32) {
				if (need_resched()) {
					write_unlock(&mapping->page_lock);
					__cond_resched();
					goto restart;
				}
			}
Linus Torvalds's avatar
Linus Torvalds committed
170
			continue;
171
		}
Linus Torvalds's avatar
Linus Torvalds committed
172

173
		progress = 0;
Linus Torvalds's avatar
Linus Torvalds committed
174
		page_cache_get(page);
175
		write_unlock(&mapping->page_lock);
Linus Torvalds's avatar
Linus Torvalds committed
176

177
		wait_on_page_writeback(page);
Andrew Morton's avatar
Andrew Morton committed
178 179
		if (PageError(page))
			ret = -EIO;
Linus Torvalds's avatar
Linus Torvalds committed
180 181

		page_cache_release(page);
182
		write_lock(&mapping->page_lock);
Linus Torvalds's avatar
Linus Torvalds committed
183
	}
184
	write_unlock(&mapping->page_lock);
Andrew Morton's avatar
Andrew Morton committed
185
	return ret;
Linus Torvalds's avatar
Linus Torvalds committed
186 187 188
}

/*
189 190 191 192 193 194 195 196 197 198 199 200 201
 * This adds a page to the page cache, starting out as locked, unreferenced,
 * not uptodate and with no errors.
 *
 * This function is used for two things: adding newly allocated pagecache
 * pages and for moving existing anon pages into swapcache.
 *
 * In the case of pagecache pages, the page is new, so we can just run
 * SetPageLocked() against it.  The other page state flags were set by
 * rmqueue()
 *
 * In the case of swapcache, try_to_swap_out() has already locked the page, so
 * SetPageLocked() is ugly-but-OK there too.  The required page state has been
 * set up by swap_out_add_to_swap_cache().
202 203
 *
 * This function does not add the page to the LRU.  The caller must do that.
Linus Torvalds's avatar
Linus Torvalds committed
204
 */
205 206
int add_to_page_cache(struct page *page, struct address_space *mapping,
		pgoff_t offset, int gfp_mask)
Linus Torvalds's avatar
Linus Torvalds committed
207
{
208
	int error = radix_tree_preload(gfp_mask & ~__GFP_HIGHMEM);
209

210 211 212 213 214 215 216 217 218 219 220 221
	if (error == 0) {
		page_cache_get(page);
		write_lock(&mapping->page_lock);
		error = radix_tree_insert(&mapping->page_tree, offset, page);
		if (!error) {
			SetPageLocked(page);
			___add_to_page_cache(page, mapping, offset);
		} else {
			page_cache_release(page);
		}
		write_unlock(&mapping->page_lock);
		radix_tree_preload_end();
222
	}
223
	return error;
Linus Torvalds's avatar
Linus Torvalds committed
224 225
}

226 227
int add_to_page_cache_lru(struct page *page, struct address_space *mapping,
				pgoff_t offset, int gfp_mask)
228
{
229
	int ret = add_to_page_cache(page, mapping, offset, gfp_mask);
230 231 232 233 234
	if (ret == 0)
		lru_cache_add(page);
	return ret;
}

235 236 237 238 239 240 241 242 243 244
/*
 * In order to wait for pages to become available there must be
 * waitqueues associated with pages. By using a hash table of
 * waitqueues where the bucket discipline is to maintain all
 * waiters on the same queue and wake all when any of the pages
 * become available, and for the woken contexts to check to be
 * sure the appropriate page became available, this saves space
 * at a cost of "thundering herd" phenomena during rare hash
 * collisions.
 */
245
static wait_queue_head_t *page_waitqueue(struct page *page)
246
{
247
	const struct zone *zone = page_zone(page);
248

249
	return &zone->wait_table[hash_ptr(page, zone->wait_table_bits)];
250 251
}

252
void wait_on_page_bit(struct page *page, int bit_nr)
Linus Torvalds's avatar
Linus Torvalds committed
253
{
254
	wait_queue_head_t *waitqueue = page_waitqueue(page);
255
	DEFINE_WAIT(wait);
Linus Torvalds's avatar
Linus Torvalds committed
256 257

	do {
258
		prepare_to_wait(waitqueue, &wait, TASK_UNINTERRUPTIBLE);
Linus Torvalds's avatar
Linus Torvalds committed
259
		sync_page(page);
260
		if (test_bit(bit_nr, &page->flags))
261
			io_schedule();
262
	} while (test_bit(bit_nr, &page->flags));
263
	finish_wait(waitqueue, &wait);
Linus Torvalds's avatar
Linus Torvalds committed
264
}
265
EXPORT_SYMBOL(wait_on_page_bit);
266 267 268 269 270 271 272 273 274 275

/**
 * unlock_page() - unlock a locked page
 *
 * @page: the page
 *
 * Unlocks the page and wakes up sleepers in ___wait_on_page_locked().
 * Also wakes sleepers in wait_on_page_writeback() because the wakeup
 * mechananism between PageLocked pages and PageWriteback pages is shared.
 * But that's OK - sleepers in wait_on_page_writeback() just go back to sleep.
Andrew Morton's avatar
Andrew Morton committed
276 277
 *
 * The first mb is necessary to safely close the critical section opened by the
Andrew Morton's avatar
Andrew Morton committed
278
 * TestSetPageLocked(), the second mb is necessary to enforce ordering between
Andrew Morton's avatar
Andrew Morton committed
279
 * the clear_bit and the read of the waitqueue (to avoid SMP races with a
280
 * parallel wait_on_page_locked()).
281
 */
Linus Torvalds's avatar
Linus Torvalds committed
282 283
void unlock_page(struct page *page)
{
284
	wait_queue_head_t *waitqueue = page_waitqueue(page);
Linus Torvalds's avatar
Linus Torvalds committed
285
	smp_mb__before_clear_bit();
Andrew Morton's avatar
Andrew Morton committed
286
	if (!TestClearPageLocked(page))
Linus Torvalds's avatar
Linus Torvalds committed
287 288
		BUG();
	smp_mb__after_clear_bit(); 
289 290
	if (waitqueue_active(waitqueue))
		wake_up_all(waitqueue);
Linus Torvalds's avatar
Linus Torvalds committed
291 292
}

293 294 295 296 297 298
/*
 * End writeback against a page.
 */
void end_page_writeback(struct page *page)
{
	wait_queue_head_t *waitqueue = page_waitqueue(page);
299 300 301 302 303 304 305

	if (!TestClearPageReclaim(page) || rotate_reclaimable_page(page)) {
		smp_mb__before_clear_bit();
		if (!TestClearPageWriteback(page))
			BUG();
		smp_mb__after_clear_bit();
	}
306 307 308 309 310
	if (waitqueue_active(waitqueue))
		wake_up_all(waitqueue);
}
EXPORT_SYMBOL(end_page_writeback);

Linus Torvalds's avatar
Linus Torvalds committed
311
/*
312 313 314 315 316 317
 * Get a lock on the page, assuming we need to sleep to get it.
 *
 * Ugly: running sync_page() in state TASK_UNINTERRUPTIBLE is scary.  If some
 * random driver's requestfn sets TASK_RUNNING, we could busywait.  However
 * chances are that on the second loop, the block layer's plug list is empty,
 * so sync_page() will then return in state TASK_UNINTERRUPTIBLE.
Linus Torvalds's avatar
Linus Torvalds committed
318
 */
319
void __lock_page(struct page *page)
Linus Torvalds's avatar
Linus Torvalds committed
320
{
321 322 323 324 325 326 327
	wait_queue_head_t *wqh = page_waitqueue(page);
	DEFINE_WAIT(wait);

	while (TestSetPageLocked(page)) {
		prepare_to_wait(wqh, &wait, TASK_UNINTERRUPTIBLE);
		sync_page(page);
		if (PageLocked(page))
328
			io_schedule();
Linus Torvalds's avatar
Linus Torvalds committed
329
	}
330
	finish_wait(wqh, &wait);
Linus Torvalds's avatar
Linus Torvalds committed
331
}
332
EXPORT_SYMBOL(__lock_page);
Linus Torvalds's avatar
Linus Torvalds committed
333 334 335

/*
 * a rather lightweight function, finding and getting a reference to a
Linus Torvalds's avatar
Linus Torvalds committed
336
 * hashed page atomically.
Linus Torvalds's avatar
Linus Torvalds committed
337
 */
338
struct page * find_get_page(struct address_space *mapping, unsigned long offset)
Linus Torvalds's avatar
Linus Torvalds committed
339 340 341 342 343 344 345
{
	struct page *page;

	/*
	 * We scan the hash list read-only. Addition to and removal from
	 * the hash-list needs a held write-lock.
	 */
346 347
	read_lock(&mapping->page_lock);
	page = radix_tree_lookup(&mapping->page_tree, offset);
Linus Torvalds's avatar
Linus Torvalds committed
348 349
	if (page)
		page_cache_get(page);
350
	read_unlock(&mapping->page_lock);
Linus Torvalds's avatar
Linus Torvalds committed
351 352 353
	return page;
}

Linus Torvalds's avatar
Linus Torvalds committed
354 355 356 357 358 359 360
/*
 * Same as above, but trylock it instead of incrementing the count.
 */
struct page *find_trylock_page(struct address_space *mapping, unsigned long offset)
{
	struct page *page;

361 362
	read_lock(&mapping->page_lock);
	page = radix_tree_lookup(&mapping->page_tree, offset);
Andrew Morton's avatar
Andrew Morton committed
363
	if (page && TestSetPageLocked(page))
364 365
		page = NULL;
	read_unlock(&mapping->page_lock);
Linus Torvalds's avatar
Linus Torvalds committed
366 367 368
	return page;
}

369 370 371 372 373 374 375 376 377 378
/**
 * find_lock_page - locate, pin and lock a pagecache page
 *
 * @mapping - the address_space to search
 * @offset - the page index
 *
 * Locates the desired pagecache page, locks it, increments its reference
 * count and returns its address.
 *
 * Returns zero if the page was not present. find_lock_page() may sleep.
Linus Torvalds's avatar
Linus Torvalds committed
379
 */
380 381
struct page *find_lock_page(struct address_space *mapping,
				unsigned long offset)
Linus Torvalds's avatar
Linus Torvalds committed
382 383 384
{
	struct page *page;

385
	read_lock(&mapping->page_lock);
Linus Torvalds's avatar
Linus Torvalds committed
386
repeat:
387
	page = radix_tree_lookup(&mapping->page_tree, offset);
Linus Torvalds's avatar
Linus Torvalds committed
388 389
	if (page) {
		page_cache_get(page);
Andrew Morton's avatar
Andrew Morton committed
390
		if (TestSetPageLocked(page)) {
391
			read_unlock(&mapping->page_lock);
Linus Torvalds's avatar
Linus Torvalds committed
392
			lock_page(page);
393
			read_lock(&mapping->page_lock);
Linus Torvalds's avatar
Linus Torvalds committed
394

395
			/* Has the page been truncated while we slept? */
Linus Torvalds's avatar
Linus Torvalds committed
396
			if (page->mapping != mapping || page->index != offset) {
Andrew Morton's avatar
Andrew Morton committed
397
				unlock_page(page);
Linus Torvalds's avatar
Linus Torvalds committed
398 399 400 401 402
				page_cache_release(page);
				goto repeat;
			}
		}
	}
403
	read_unlock(&mapping->page_lock);
Linus Torvalds's avatar
Linus Torvalds committed
404
	return page;
Linus Torvalds's avatar
Linus Torvalds committed
405 406
}

407 408 409 410 411 412 413 414 415 416 417 418 419 420 421 422 423
/**
 * find_or_create_page - locate or add a pagecache page
 *
 * @mapping - the page's address_space
 * @index - the page's index into the mapping
 * @gfp_mask - page allocation mode
 *
 * Locates a page in the pagecache.  If the page is not present, a new page
 * is allocated using @gfp_mask and is added to the pagecache and to the VM's
 * LRU list.  The returned page is locked and has its reference count
 * incremented.
 *
 * find_or_create_page() may sleep, even if @gfp_flags specifies an atomic
 * allocation!
 *
 * find_or_create_page() returns the desired page's address, or zero on
 * memory exhaustion.
Linus Torvalds's avatar
Linus Torvalds committed
424
 */
425
struct page *find_or_create_page(struct address_space *mapping,
426
		unsigned long index, unsigned int gfp_mask)
Linus Torvalds's avatar
Linus Torvalds committed
427
{
428 429 430
	struct page *page, *cached_page = NULL;
	int err;
repeat:
431
	page = find_lock_page(mapping, index);
Linus Torvalds's avatar
Linus Torvalds committed
432
	if (!page) {
433 434 435 436
		if (!cached_page) {
			cached_page = alloc_page(gfp_mask);
			if (!cached_page)
				return NULL;
Linus Torvalds's avatar
Linus Torvalds committed
437
		}
438 439
		err = add_to_page_cache_lru(cached_page, mapping,
					index, gfp_mask);
440 441 442 443 444
		if (!err) {
			page = cached_page;
			cached_page = NULL;
		} else if (err == -EEXIST)
			goto repeat;
Linus Torvalds's avatar
Linus Torvalds committed
445
	}
446 447
	if (cached_page)
		page_cache_release(cached_page);
448
	return page;
Linus Torvalds's avatar
Linus Torvalds committed
449 450
}

451 452 453 454 455 456 457 458 459 460 461 462 463 464 465 466 467 468 469 470 471 472 473 474 475 476 477 478 479 480 481
/**
 * find_get_pages - gang pagecache lookup
 * @mapping:	The address_space to search
 * @start:	The starting page index
 * @nr_pages:	The maximum number of pages
 * @pages:	Where the resulting pages are placed
 *
 * find_get_pages() will search for and return a group of up to
 * @nr_pages pages in the mapping.  The pages are placed at @pages.
 * find_get_pages() takes a reference against the returned pages.
 *
 * The search returns a group of mapping-contiguous pages with ascending
 * indexes.  There may be holes in the indices due to not-present pages.
 *
 * find_get_pages() returns the number of pages which were found.
 */
unsigned int find_get_pages(struct address_space *mapping, pgoff_t start,
			    unsigned int nr_pages, struct page **pages)
{
	unsigned int i;
	unsigned int ret;

	read_lock(&mapping->page_lock);
	ret = radix_tree_gang_lookup(&mapping->page_tree,
				(void **)pages, start, nr_pages);
	for (i = 0; i < ret; i++)
		page_cache_get(pages[i]);
	read_unlock(&mapping->page_lock);
	return ret;
}

Linus Torvalds's avatar
Linus Torvalds committed
482 483 484 485 486
/*
 * Same as grab_cache_page, but do not wait if the page is unavailable.
 * This is intended for speculative data generators, where the data can
 * be regenerated if the page couldn't be grabbed.  This routine should
 * be safe to call while holding the lock for another page.
487 488 489
 *
 * Clear __GFP_FS when allocating the page to avoid recursion into the fs
 * and deadlock against the caller's locked page.
Linus Torvalds's avatar
Linus Torvalds committed
490
 */
491 492
struct page *
grab_cache_page_nowait(struct address_space *mapping, unsigned long index)
Linus Torvalds's avatar
Linus Torvalds committed
493
{
494
	struct page *page = find_get_page(mapping, index);
495
	int gfp_mask;
Linus Torvalds's avatar
Linus Torvalds committed
496

497 498 499
	if (page) {
		if (!TestSetPageLocked(page))
			return page;
Linus Torvalds's avatar
Linus Torvalds committed
500 501 502
		page_cache_release(page);
		return NULL;
	}
503 504 505
	gfp_mask = mapping->gfp_mask & ~__GFP_FS;
	page = alloc_pages(gfp_mask, 0);
	if (page && add_to_page_cache_lru(page, mapping, index, gfp_mask)) {
506 507 508
		page_cache_release(page);
		page = NULL;
	}
Linus Torvalds's avatar
Linus Torvalds committed
509 510 511
	return page;
}

Linus Torvalds's avatar
Linus Torvalds committed
512 513 514 515 516 517 518
/*
 * This is a generic file read routine, and uses the
 * inode->i_op->readpage() function for the actual low-level
 * stuff.
 *
 * This is really ugly. But the goto's actually try to clarify some
 * of the logic when it comes to error handling etc.
519
 * - note the struct file * is only passed for the use of readpage
Linus Torvalds's avatar
Linus Torvalds committed
520
 */
521 522 523 524 525 526
void do_generic_mapping_read(struct address_space *mapping,
			     struct file_ra_state *ra,
			     struct file * filp,
			     loff_t *ppos,
			     read_descriptor_t * desc,
			     read_actor_t actor)
Linus Torvalds's avatar
Linus Torvalds committed
527
{
Linus Torvalds's avatar
Linus Torvalds committed
528
	struct inode *inode = mapping->host;
Linus Torvalds's avatar
Linus Torvalds committed
529 530 531 532 533 534 535 536 537
	unsigned long index, offset;
	struct page *cached_page;
	int error;

	cached_page = NULL;
	index = *ppos >> PAGE_CACHE_SHIFT;
	offset = *ppos & ~PAGE_CACHE_MASK;

	for (;;) {
538
		struct page *page;
Linus Torvalds's avatar
Linus Torvalds committed
539
		unsigned long end_index, nr, ret;
Linus Torvalds's avatar
Linus Torvalds committed
540

Linus Torvalds's avatar
Linus Torvalds committed
541
		end_index = inode->i_size >> PAGE_CACHE_SHIFT;
Linus Torvalds's avatar
Linus Torvalds committed
542
			
Linus Torvalds's avatar
Linus Torvalds committed
543 544 545 546
		if (index > end_index)
			break;
		nr = PAGE_CACHE_SIZE;
		if (index == end_index) {
Linus Torvalds's avatar
Linus Torvalds committed
547
			nr = inode->i_size & ~PAGE_CACHE_MASK;
Linus Torvalds's avatar
Linus Torvalds committed
548 549 550 551
			if (nr <= offset)
				break;
		}

552
		cond_resched();
553
		page_cache_readahead(mapping, ra, filp, index);
Andrew Morton's avatar
Andrew Morton committed
554

Linus Torvalds's avatar
Linus Torvalds committed
555 556 557 558 559
		nr = nr - offset;

		/*
		 * Try to find the data in the page cache..
		 */
560 561
find_page:
		read_lock(&mapping->page_lock);
562
		page = radix_tree_lookup(&mapping->page_tree, index);
Andrew Morton's avatar
Andrew Morton committed
563
		if (!page) {
564
			read_unlock(&mapping->page_lock);
565
			handle_ra_miss(mapping,ra);
Linus Torvalds's avatar
Linus Torvalds committed
566
			goto no_cached_page;
Andrew Morton's avatar
Andrew Morton committed
567
		}
Linus Torvalds's avatar
Linus Torvalds committed
568
		page_cache_get(page);
569
		read_unlock(&mapping->page_lock);
Linus Torvalds's avatar
Linus Torvalds committed
570

Andrew Morton's avatar
Andrew Morton committed
571
		if (!PageUptodate(page))
Linus Torvalds's avatar
Linus Torvalds committed
572 573 574 575 576 577
			goto page_not_up_to_date;
page_ok:
		/* If users can be writing to this page using arbitrary
		 * virtual addresses, take care about potential aliasing
		 * before reading the page on the kernel side.
		 */
578
		if (!list_empty(&mapping->i_mmap_shared))
Linus Torvalds's avatar
Linus Torvalds committed
579 580
			flush_dcache_page(page);

Linus Torvalds's avatar
Linus Torvalds committed
581
		/*
Andrew Morton's avatar
Andrew Morton committed
582
		 * Mark the page accessed if we read the beginning.
Linus Torvalds's avatar
Linus Torvalds committed
583
		 */
Andrew Morton's avatar
Andrew Morton committed
584
		if (!offset)
Linus Torvalds's avatar
Linus Torvalds committed
585 586
			mark_page_accessed(page);

Linus Torvalds's avatar
Linus Torvalds committed
587 588 589 590 591 592 593 594 595 596
		/*
		 * Ok, we have the page, and it's up-to-date, so
		 * now we can copy it to user space...
		 *
		 * The actor routine returns how many bytes were actually used..
		 * NOTE! This may not be the same as how much of a user buffer
		 * we filled up (we may be padding etc), so we can only update
		 * "pos" here (the actor routine has to update the user buffer
		 * pointers and the remaining count).
		 */
Linus Torvalds's avatar
Linus Torvalds committed
597 598
		ret = actor(desc, page, offset, nr);
		offset += ret;
Linus Torvalds's avatar
Linus Torvalds committed
599 600
		index += offset >> PAGE_CACHE_SHIFT;
		offset &= ~PAGE_CACHE_MASK;
Linus Torvalds's avatar
Linus Torvalds committed
601

Linus Torvalds's avatar
Linus Torvalds committed
602
		page_cache_release(page);
Linus Torvalds's avatar
Linus Torvalds committed
603
		if (ret == nr && desc->count)
Linus Torvalds's avatar
Linus Torvalds committed
604 605 606 607
			continue;
		break;

page_not_up_to_date:
Andrew Morton's avatar
Andrew Morton committed
608
		if (PageUptodate(page))
Linus Torvalds's avatar
Linus Torvalds committed
609 610 611 612 613 614 615
			goto page_ok;

		/* Get exclusive access to the page ... */
		lock_page(page);

		/* Did it get unhashed before we got the lock? */
		if (!page->mapping) {
Andrew Morton's avatar
Andrew Morton committed
616
			unlock_page(page);
Linus Torvalds's avatar
Linus Torvalds committed
617 618 619 620 621
			page_cache_release(page);
			continue;
		}

		/* Did somebody else fill it already? */
Andrew Morton's avatar
Andrew Morton committed
622 623
		if (PageUptodate(page)) {
			unlock_page(page);
Linus Torvalds's avatar
Linus Torvalds committed
624 625 626 627 628 629 630 631
			goto page_ok;
		}

readpage:
		/* ... and start the actual read. The read will unlock the page. */
		error = mapping->a_ops->readpage(filp, page);

		if (!error) {
Andrew Morton's avatar
Andrew Morton committed
632
			if (PageUptodate(page))
Linus Torvalds's avatar
Linus Torvalds committed
633
				goto page_ok;
634
			wait_on_page_locked(page);
Andrew Morton's avatar
Andrew Morton committed
635
			if (PageUptodate(page))
Linus Torvalds's avatar
Linus Torvalds committed
636 637 638 639 640 641 642 643 644 645 646 647 648 649 650
				goto page_ok;
			error = -EIO;
		}

		/* UHHUH! A synchronous read error occurred. Report it */
		desc->error = error;
		page_cache_release(page);
		break;

no_cached_page:
		/*
		 * Ok, it wasn't cached, so we need to create a new
		 * page..
		 */
		if (!cached_page) {
651
			cached_page = page_cache_alloc_cold(mapping);
Linus Torvalds's avatar
Linus Torvalds committed
652 653 654 655 656
			if (!cached_page) {
				desc->error = -ENOMEM;
				break;
			}
		}
657 658
		error = add_to_page_cache_lru(cached_page, mapping,
						index, GFP_KERNEL);
659 660 661 662
		if (error) {
			if (error == -EEXIST)
				goto find_page;
			desc->error = error;
663 664
			break;
		}
Linus Torvalds's avatar
Linus Torvalds committed
665 666 667 668 669 670 671
		page = cached_page;
		cached_page = NULL;
		goto readpage;
	}

	*ppos = ((loff_t) index << PAGE_CACHE_SHIFT) + offset;
	if (cached_page)
Linus Torvalds's avatar
Linus Torvalds committed
672
		page_cache_release(cached_page);
Linus Torvalds's avatar
Linus Torvalds committed
673 674 675
	UPDATE_ATIME(inode);
}

676 677 678 679 680 681 682 683 684 685 686 687 688 689 690 691 692 693 694 695 696 697 698 699 700 701 702 703 704
/*
 * Fault a userspace page into pagetables.  Return non-zero on a fault.
 *
 * FIXME: this assumes that two userspace pages are always sufficient.  That's
 * not true if PAGE_CACHE_SIZE > PAGE_SIZE.
 */
static inline int fault_in_pages_writeable(char *uaddr, int size)
{
	int ret;

	/*
	 * Writing zeroes into userspace here is OK, because we know that if
	 * the zero gets there, we'll be overwriting it.
	 */
	ret = __put_user(0, uaddr);
	if (ret == 0) {
		char *end = uaddr + size - 1;

		/*
		 * If the page was already mapped, this will get a cache miss
		 * for sure, so try to avoid doing it.
		 */
		if (((unsigned long)uaddr & PAGE_MASK) !=
				((unsigned long)end & PAGE_MASK))
		 	ret = __put_user(0, end);
	}
	return ret;
}

705
static void fault_in_pages_readable(const char *uaddr, int size)
706 707 708 709 710 711 712 713 714 715 716 717 718 719
{
	volatile char c;
	int ret;

	ret = __get_user(c, (char *)uaddr);
	if (ret == 0) {
		const char *end = uaddr + size - 1;

		if (((unsigned long)uaddr & PAGE_MASK) !=
				((unsigned long)end & PAGE_MASK))
		 	__get_user(c, (char *)end);
	}
}

720 721
int file_read_actor(read_descriptor_t *desc, struct page *page,
			unsigned long offset, unsigned long size)
Linus Torvalds's avatar
Linus Torvalds committed
722 723 724 725 726 727 728
{
	char *kaddr;
	unsigned long left, count = desc->count;

	if (size > count)
		size = count;

729 730 731 732 733 734 735 736 737 738 739 740 741
	/*
	 * Faults on the destination of a read are common, so do it before
	 * taking the kmap.
	 */
	if (!fault_in_pages_writeable(desc->buf, size)) {
		kaddr = kmap_atomic(page, KM_USER0);
		left = __copy_to_user(desc->buf, kaddr + offset, size);
		kunmap_atomic(kaddr, KM_USER0);
		if (left == 0)
			goto success;
	}

	/* Do it the slow way */
Linus Torvalds's avatar
Linus Torvalds committed
742 743 744
	kaddr = kmap(page);
	left = __copy_to_user(desc->buf, kaddr + offset, size);
	kunmap(page);
745

Linus Torvalds's avatar
Linus Torvalds committed
746 747 748 749
	if (left) {
		size -= left;
		desc->error = -EFAULT;
	}
750
success:
Linus Torvalds's avatar
Linus Torvalds committed
751 752 753 754 755 756 757 758 759 760
	desc->count = count - size;
	desc->written += size;
	desc->buf += size;
	return size;
}

/*
 * This is the "read()" routine for all filesystems
 * that can use the page cache directly.
 */
Andrew Morton's avatar
Andrew Morton committed
761
static ssize_t
762
__generic_file_aio_read(struct kiocb *iocb, const struct iovec *iov,
Andrew Morton's avatar
Andrew Morton committed
763
		unsigned long nr_segs, loff_t *ppos)
Linus Torvalds's avatar
Linus Torvalds committed
764
{
765
	struct file *filp = iocb->ki_filp;
Linus Torvalds's avatar
Linus Torvalds committed
766
	ssize_t retval;
Andrew Morton's avatar
Andrew Morton committed
767
	unsigned long seg;
768
	size_t count;
Linus Torvalds's avatar
Linus Torvalds committed
769

770 771 772 773 774 775 776 777 778 779 780 781 782 783 784 785
	count = 0;
	for (seg = 0; seg < nr_segs; seg++) {
		const struct iovec *iv = &iov[seg];

		/*
		 * If any segment has a negative length, or the cumulative
		 * length ever wraps negative then return -EINVAL.
		 */
		count += iv->iov_len;
		if (unlikely((ssize_t)(count|iv->iov_len) < 0))
			return -EINVAL;
		if (access_ok(VERIFY_WRITE, iv->iov_base, iv->iov_len))
			continue;
		if (seg == 0)
			return -EFAULT;
		nr_segs = seg;
786
		count -= iv->iov_len;	/* This segment is no good */
787 788
		break;
	}
Linus Torvalds's avatar
Linus Torvalds committed
789

Andrew Morton's avatar
Andrew Morton committed
790
	/* coalesce the iovecs and go direct-to-BIO for O_DIRECT */
791 792 793 794 795 796 797 798 799 800 801 802
	if (filp->f_flags & O_DIRECT) {
		loff_t pos = *ppos, size;
		struct address_space *mapping;
		struct inode *inode;

		mapping = filp->f_dentry->d_inode->i_mapping;
		inode = mapping->host;
		retval = 0;
		if (!count)
			goto out; /* skip atime */
		size = inode->i_size;
		if (pos < size) {
Andrew Morton's avatar
Andrew Morton committed
803
			if (pos + count > size) {
804
				count = size - pos;
Andrew Morton's avatar
Andrew Morton committed
805 806 807
				nr_segs = iov_shorten((struct iovec *)iov,
							nr_segs, count);
			}
808
			retval = generic_file_direct_IO(READ, iocb,
Andrew Morton's avatar
Andrew Morton committed
809
					iov, pos, nr_segs);
810 811 812 813 814 815
			if (retval > 0)
				*ppos = pos + retval;
		}
		UPDATE_ATIME(filp->f_dentry->d_inode);
		goto out;
	}
Linus Torvalds's avatar
Linus Torvalds committed
816

Andrew Morton's avatar
Andrew Morton committed
817 818 819
	retval = 0;
	if (count) {
		for (seg = 0; seg < nr_segs; seg++) {
Linus Torvalds's avatar
Linus Torvalds committed
820 821 822
			read_descriptor_t desc;

			desc.written = 0;
Andrew Morton's avatar
Andrew Morton committed
823 824 825 826
			desc.buf = iov[seg].iov_base;
			desc.count = iov[seg].iov_len;
			if (desc.count == 0)
				continue;
Linus Torvalds's avatar
Linus Torvalds committed
827
			desc.error = 0;
828
			do_generic_file_read(filp,ppos,&desc,file_read_actor);
Andrew Morton's avatar
Andrew Morton committed
829 830
			retval += desc.written;
			if (!retval) {
Linus Torvalds's avatar
Linus Torvalds committed
831
				retval = desc.error;
Andrew Morton's avatar
Andrew Morton committed
832 833
				break;
			}
Linus Torvalds's avatar
Linus Torvalds committed
834 835
		}
	}
836
out:
Linus Torvalds's avatar
Linus Torvalds committed
837 838 839
	return retval;
}

840
ssize_t
841
generic_file_aio_read(struct kiocb *iocb, char *buf, size_t count, loff_t pos)
842 843 844
{
	struct iovec local_iov = { .iov_base = buf, .iov_len = count };

845 846
	BUG_ON(iocb->ki_pos != pos);
	return __generic_file_aio_read(iocb, &local_iov, 1, &iocb->ki_pos);
847
}
848
EXPORT_SYMBOL(generic_file_aio_read);
849

Andrew Morton's avatar
Andrew Morton committed
850 851 852 853
ssize_t
generic_file_read(struct file *filp, char *buf, size_t count, loff_t *ppos)
{
	struct iovec local_iov = { .iov_base = buf, .iov_len = count };
854 855
	struct kiocb kiocb;
	ssize_t ret;
Andrew Morton's avatar
Andrew Morton committed
856

857 858 859 860 861
	init_sync_kiocb(&kiocb, filp);
	ret = __generic_file_aio_read(&kiocb, &local_iov, 1, ppos);
	if (-EIOCBQUEUED == ret)
		ret = wait_on_sync_kiocb(&kiocb);
	return ret;
Andrew Morton's avatar
Andrew Morton committed
862 863
}

864
int file_send_actor(read_descriptor_t * desc, struct page *page, unsigned long offset, unsigned long size)
Linus Torvalds's avatar
Linus Torvalds committed
865 866 867 868 869 870 871 872
{
	ssize_t written;
	unsigned long count = desc->count;
	struct file *file = (struct file *) desc->buf;

	if (size > count)
		size = count;

873 874
	written = file->f_op->sendpage(file, page, offset,
				       size, &file->f_pos, size<count);
Linus Torvalds's avatar
Linus Torvalds committed
875 876 877 878 879 880 881 882 883
	if (written < 0) {
		desc->error = written;
		written = 0;
	}
	desc->count = count - written;
	desc->written += written;
	return written;
}

884 885
ssize_t generic_file_sendfile(struct file *in_file, loff_t *ppos,
			 size_t count, read_actor_t actor, void *target)
Linus Torvalds's avatar
Linus Torvalds committed
886
{
887
	read_descriptor_t desc;
888

889 890
	if (!count)
		return 0;
891

892 893
	desc.written = 0;
	desc.count = count;
894
	desc.buf = target;
895
	desc.error = 0;
Linus Torvalds's avatar
Linus Torvalds committed
896

897
	do_generic_file_read(in_file, ppos, &desc, actor);
898 899 900
	if (desc.written)
		return desc.written;
	return desc.error;
901 902
}

903
static ssize_t
904 905
do_readahead(struct address_space *mapping, struct file *filp,
	     unsigned long index, unsigned long nr)
Linus Torvalds's avatar
Linus Torvalds committed
906 907 908 909
{
	if (!mapping || !mapping->a_ops || !mapping->a_ops->readpage)
		return -EINVAL;

910
	do_page_cache_readahead(mapping, filp, index, max_sane_readahead(nr));
Linus Torvalds's avatar
Linus Torvalds committed
911 912 913 914 915 916 917 918 919 920 921 922
	return 0;
}

asmlinkage ssize_t sys_readahead(int fd, loff_t offset, size_t count)
{
	ssize_t ret;
	struct file *file;

	ret = -EBADF;
	file = fget(fd);
	if (file) {
		if (file->f_mode & FMODE_READ) {
923
			struct address_space *mapping = file->f_dentry->d_inode->i_mapping;
Linus Torvalds's avatar
Linus Torvalds committed
924
			unsigned long start = offset >> PAGE_CACHE_SHIFT;
Andrew Morton's avatar
Andrew Morton committed
925 926
			unsigned long end = (offset + count - 1) >> PAGE_CACHE_SHIFT;
			unsigned long len = end - start + 1;
927
			ret = do_readahead(mapping, file, start, len);
Linus Torvalds's avatar
Linus Torvalds committed
928 929 930 931 932 933
		}
		fput(file);
	}
	return ret;
}

934 935 936 937 938 939 940 941 942 943 944 945 946 947 948 949
#ifdef CONFIG_MMU
/*
 * This adds the requested page to the page cache if it isn't already there,
 * and schedules an I/O to read in its contents from disk.
 */
static int FASTCALL(page_cache_read(struct file * file, unsigned long offset));
static int page_cache_read(struct file * file, unsigned long offset)
{
	struct address_space *mapping = file->f_dentry->d_inode->i_mapping;
	struct page *page; 
	int error;

	page = page_cache_alloc_cold(mapping);
	if (!page)
		return -ENOMEM;

950
	error = add_to_page_cache_lru(page, mapping, offset, GFP_KERNEL);
951 952 953 954 955 956 957 958 959 960 961 962 963 964 965
	if (!error) {
		error = mapping->a_ops->readpage(file, page);
		page_cache_release(page);
		return error;
	}

	/*
	 * We arrive here in the unlikely event that someone 
	 * raced with us and added our page to the cache first
	 * or we are out of memory for radix-tree nodes.
	 */
	page_cache_release(page);
	return error == -EEXIST ? 0 : error;
}

Linus Torvalds's avatar
Linus Torvalds committed
966 967 968 969 970 971 972 973
/*
 * filemap_nopage() is invoked via the vma operations vector for a
 * mapped memory region to read in file data during a page fault.
 *
 * The goto's are kind of ugly, but this streamlines the normal case of having
 * it in the page cache, and handles the special cases reasonably without
 * having a lot of duplicated code.
 */
Linus Torvalds's avatar
Linus Torvalds committed
974
struct page * filemap_nopage(struct vm_area_struct * area, unsigned long address, int unused)
Linus Torvalds's avatar
Linus Torvalds committed
975 976 977
{
	int error;
	struct file *file = area->vm_file;
Linus Torvalds's avatar
Linus Torvalds committed
978
	struct address_space *mapping = file->f_dentry->d_inode->i_mapping;
979
	struct file_ra_state *ra = &file->f_ra;
Linus Torvalds's avatar
Linus Torvalds committed
980
	struct inode *inode = mapping->host;
981
	struct page *page;
Linus Torvalds's avatar
Linus Torvalds committed
982
	unsigned long size, pgoff, endoff;
983
	int did_readahead;
Linus Torvalds's avatar
Linus Torvalds committed
984 985

	pgoff = ((address - area->vm_start) >> PAGE_CACHE_SHIFT) + area->vm_pgoff;
Linus Torvalds's avatar
Linus Torvalds committed
986
	endoff = ((area->vm_end - area->vm_start) >> PAGE_CACHE_SHIFT) + area->vm_pgoff;
Linus Torvalds's avatar
Linus Torvalds committed
987 988 989 990 991 992

retry_all:
	/*
	 * An external ptracer can access pages that normally aren't
	 * accessible..
	 */
Linus Torvalds's avatar
Linus Torvalds committed
993
	size = (inode->i_size + PAGE_CACHE_SIZE - 1) >> PAGE_CACHE_SHIFT;
Linus Torvalds's avatar
Linus Torvalds committed
994 995 996
	if ((pgoff >= size) && (area->vm_mm == current->mm))
		return NULL;

997 998 999 1000
	/*
	 * The "size" of the file, as far as mmap is concerned, isn't bigger
	 * than the mapping
	 */
Linus Torvalds's avatar
Linus Torvalds committed
1001 1002 1003
	if (size > endoff)
		size = endoff;

1004 1005
	did_readahead = 0;

Andrew Morton's avatar
Andrew Morton committed
1006 1007 1008 1009
	/*
	 * The readahead code wants to be told about each and every page
	 * so it can build and shrink its windows appropriately
	 */
1010 1011
	if (VM_SequentialReadHint(area)) {
		did_readahead = 1;
1012
		page_cache_readahead(mapping, ra, file, pgoff);
1013
	}
Andrew Morton's avatar
Andrew Morton committed
1014 1015 1016 1017 1018

	/*
	 * If the offset is outside the mapping size we're off the end
	 * of a privately mapped file, so we need to map a zero page.
	 */
1019 1020
	if ((pgoff < size) && !VM_RandomReadHint(area)) {
		did_readahead = 1;
1021
		page_cache_readaround(mapping, ra, file, pgoff);
1022
	}
Andrew Morton's avatar
Andrew Morton committed
1023

Linus Torvalds's avatar
Linus Torvalds committed
1024 1025 1026 1027
	/*
	 * Do we have something in the page cache already?
	 */
retry_find:
1028
	page = find_get_page(mapping, pgoff);
1029 1030
	if (!page) {
		if (did_readahead) {
1031
			handle_ra_miss(mapping,ra);
1032 1033
			did_readahead = 0;
		}
Linus Torvalds's avatar
Linus Torvalds committed
1034
		goto no_cached_page;
1035
	}
Linus Torvalds's avatar
Linus Torvalds committed
1036 1037 1038 1039 1040

	/*
	 * Ok, found a page in the page cache, now we need to check
	 * that it's up-to-date.
	 */
Andrew Morton's avatar
Andrew Morton committed
1041
	if (!PageUptodate(page))
Linus Torvalds's avatar
Linus Torvalds committed
1042 1043 1044 1045 1046 1047 1048
		goto page_not_uptodate;

success:
	/*
	 * Found the page and have a reference on it, need to check sharing
	 * and possibly copy it over to another page..
	 */
Linus Torvalds's avatar
Linus Torvalds committed
1049
	mark_page_accessed(page);
Linus Torvalds's avatar
Linus Torvalds committed
1050 1051
	flush_page_to_ram(page);
	return page;
Linus Torvalds's avatar
Linus Torvalds committed
1052 1053 1054

no_cached_page:
	/*
Andrew Morton's avatar
Andrew Morton committed
1055 1056
	 * We're only likely to ever get here if MADV_RANDOM is in
	 * effect.
Linus Torvalds's avatar
Linus Torvalds committed
1057
	 */
Andrew Morton's avatar
Andrew Morton committed
1058
	error = page_cache_read(file, pgoff);
Linus Torvalds's avatar
Linus Torvalds committed
1059 1060 1061 1062 1063 1064 1065 1066 1067 1068 1069 1070 1071 1072 1073 1074 1075 1076 1077

	/*
	 * The page we want has now been added to the page cache.
	 * In the unlikely event that someone removed it in the
	 * meantime, we'll just come back here and read it again.
	 */
	if (error >= 0)
		goto retry_find;

	/*
	 * An error return from page_cache_read can result if the
	 * system is low on memory, or a problem occurs while trying
	 * to schedule I/O.
	 */
	if (error == -ENOMEM)
		return NOPAGE_OOM;
	return NULL;

page_not_uptodate:
1078
	inc_page_state(pgmajfault);
Linus Torvalds's avatar
Linus Torvalds committed
1079 1080 1081 1082
	lock_page(page);

	/* Did it get unhashed while we waited for it? */
	if (!page->mapping) {
Andrew Morton's avatar
Andrew Morton committed
1083
		unlock_page(page);
Linus Torvalds's avatar
Linus Torvalds committed
1084 1085 1086 1087 1088
		page_cache_release(page);
		goto retry_all;
	}

	/* Did somebody else get it up-to-date? */
Andrew Morton's avatar
Andrew Morton committed
1089 1090
	if (PageUptodate(page)) {
		unlock_page(page);
Linus Torvalds's avatar
Linus Torvalds committed
1091 1092 1093 1094
		goto success;
	}

	if (!mapping->a_ops->readpage(file, page)) {
1095
		wait_on_page_locked(page);
Andrew Morton's avatar
Andrew Morton committed
1096
		if (PageUptodate(page))
Linus Torvalds's avatar
Linus Torvalds committed
1097 1098 1099 1100 1101 1102 1103 1104 1105 1106 1107 1108 1109
			goto success;
	}

	/*
	 * Umm, take care of errors if the page isn't up-to-date.
	 * Try to re-read it _once_. We do this synchronously,
	 * because there really aren't any performance issues here
	 * and we need to check for errors.
	 */
	lock_page(page);

	/* Somebody truncated the page on us? */
	if (!page->mapping) {
Andrew Morton's avatar
Andrew Morton committed
1110
		unlock_page(page);
Linus Torvalds's avatar
Linus Torvalds committed
1111 1112 1113 1114 1115
		page_cache_release(page);
		goto retry_all;
	}

	/* Somebody else successfully read it in? */
Andrew Morton's avatar
Andrew Morton committed
1116 1117
	if (PageUptodate(page)) {
		unlock_page(page);
Linus Torvalds's avatar
Linus Torvalds committed
1118 1119 1120 1121
		goto success;
	}
	ClearPageError(page);
	if (!mapping->a_ops->readpage(file, page)) {
1122
		wait_on_page_locked(page);
Andrew Morton's avatar
Andrew Morton committed
1123
		if (PageUptodate(page))
Linus Torvalds's avatar
Linus Torvalds committed
1124 1125 1126 1127 1128 1129 1130 1131 1132 1133 1134
			goto success;
	}

	/*
	 * Things didn't work out. Return zero to tell the
	 * mm layer so, possibly freeing the page cache page first.
	 */
	page_cache_release(page);
	return NULL;
}

Andrew Morton's avatar
Andrew Morton committed
1135 1136 1137 1138 1139 1140 1141 1142 1143 1144 1145 1146 1147 1148 1149 1150 1151 1152 1153 1154 1155 1156 1157 1158 1159 1160 1161 1162 1163 1164 1165 1166 1167 1168 1169 1170 1171 1172 1173 1174 1175 1176 1177 1178 1179 1180 1181 1182 1183 1184 1185 1186 1187 1188 1189 1190 1191 1192 1193 1194 1195 1196 1197 1198 1199 1200 1201 1202 1203 1204 1205 1206 1207 1208 1209 1210 1211 1212 1213 1214 1215 1216 1217 1218 1219 1220 1221 1222 1223 1224 1225 1226 1227 1228 1229 1230 1231 1232 1233 1234 1235 1236 1237 1238 1239 1240 1241 1242 1243 1244 1245 1246 1247 1248 1249 1250 1251 1252 1253 1254 1255 1256 1257 1258 1259 1260 1261 1262 1263 1264 1265 1266 1267 1268 1269 1270 1271 1272 1273 1274 1275 1276 1277 1278 1279 1280 1281 1282 1283 1284
static struct page * filemap_getpage(struct file *file, unsigned long pgoff,
					int nonblock)
{
	struct address_space *mapping = file->f_dentry->d_inode->i_mapping;
	struct page *page;
	int error;

	/*
	 * Do we have something in the page cache already?
	 */
retry_find:
	page = find_get_page(mapping, pgoff);
	if (!page) {
		if (nonblock)
			return NULL;
		goto no_cached_page;
	}

	/*
	 * Ok, found a page in the page cache, now we need to check
	 * that it's up-to-date.
	 */
	if (!PageUptodate(page))
		goto page_not_uptodate;

success:
	/*
	 * Found the page and have a reference on it, need to check sharing
	 * and possibly copy it over to another page..
	 */
	mark_page_accessed(page);
	flush_page_to_ram(page);

	return page;

no_cached_page:
	error = page_cache_read(file, pgoff);

	/*
	 * The page we want has now been added to the page cache.
	 * In the unlikely event that someone removed it in the
	 * meantime, we'll just come back here and read it again.
	 */
	if (error >= 0)
		goto retry_find;

	/*
	 * An error return from page_cache_read can result if the
	 * system is low on memory, or a problem occurs while trying
	 * to schedule I/O.
	 */
	return NULL;

page_not_uptodate:
	lock_page(page);

	/* Did it get unhashed while we waited for it? */
	if (!page->mapping) {
		unlock_page(page);
		goto err;
	}

	/* Did somebody else get it up-to-date? */
	if (PageUptodate(page)) {
		unlock_page(page);
		goto success;
	}

	if (!mapping->a_ops->readpage(file, page)) {
		wait_on_page_locked(page);
		if (PageUptodate(page))
			goto success;
	}

	/*
	 * Umm, take care of errors if the page isn't up-to-date.
	 * Try to re-read it _once_. We do this synchronously,
	 * because there really aren't any performance issues here
	 * and we need to check for errors.
	 */
	lock_page(page);

	/* Somebody truncated the page on us? */
	if (!page->mapping) {
		unlock_page(page);
		goto err;
	}
	/* Somebody else successfully read it in? */
	if (PageUptodate(page)) {
		unlock_page(page);
		goto success;
	}

	ClearPageError(page);
	if (!mapping->a_ops->readpage(file, page)) {
		wait_on_page_locked(page);
		if (PageUptodate(page))
			goto success;
	}

	/*
	 * Things didn't work out. Return zero to tell the
	 * mm layer so, possibly freeing the page cache page first.
	 */
err:
	page_cache_release(page);

	return NULL;
}

static int filemap_populate(struct vm_area_struct *vma,
			unsigned long addr,
			unsigned long len,
			unsigned long prot,
			unsigned long pgoff,
			int nonblock)
{
	struct file *file = vma->vm_file;
	struct address_space *mapping = file->f_dentry->d_inode->i_mapping;
	struct inode *inode = mapping->host;
	unsigned long size;
	struct mm_struct *mm = vma->vm_mm;
	struct page *page;
	int err;

repeat:
	size = (inode->i_size + PAGE_CACHE_SIZE - 1) >> PAGE_CACHE_SHIFT;
	if (pgoff + (len >> PAGE_CACHE_SHIFT) > size)
		return -EINVAL;

	page = filemap_getpage(file, pgoff, nonblock);
	if (!page && !nonblock)
		return -ENOMEM;
	if (page) {
		err = install_page(mm, vma, addr, page, prot);
		if (err) {
			page_cache_release(page);
			return err;
		}
	}

	len -= PAGE_SIZE;
	addr += PAGE_SIZE;
	pgoff++;
	if (len)
		goto repeat;

	return 0;
}

Linus Torvalds's avatar
Linus Torvalds committed
1285
static struct vm_operations_struct generic_file_vm_ops = {
1286
	.nopage		= filemap_nopage,
Andrew Morton's avatar
Andrew Morton committed
1287
	.populate	= filemap_populate,
Linus Torvalds's avatar
Linus Torvalds committed
1288 1289 1290 1291 1292 1293
};

/* This is used for a general mmap of a disk file */

int generic_file_mmap(struct file * file, struct vm_area_struct * vma)
{
Linus Torvalds's avatar
Linus Torvalds committed
1294 1295
	struct address_space *mapping = file->f_dentry->d_inode->i_mapping;
	struct inode *inode = mapping->host;
Linus Torvalds's avatar
Linus Torvalds committed
1296

Linus Torvalds's avatar
Linus Torvalds committed
1297
	if (!mapping->a_ops->readpage)
Linus Torvalds's avatar
Linus Torvalds committed
1298 1299
		return -ENOEXEC;
	UPDATE_ATIME(inode);
Linus Torvalds's avatar
Linus Torvalds committed
1300
	vma->vm_ops = &generic_file_vm_ops;
Linus Torvalds's avatar
Linus Torvalds committed
1301 1302
	return 0;
}
1303 1304 1305 1306 1307 1308 1309 1310

int generic_file_readonly_mmap(struct file *file, struct vm_area_struct *vma)
{
	if ((vma->vm_flags & VM_SHARED) && (vma->vm_flags & VM_WRITE))
		return -EINVAL;
	vma->vm_flags &= ~VM_MAYWRITE;
	return generic_file_mmap(file, vma);
}
1311 1312 1313 1314 1315
#else
int generic_file_mmap(struct file * file, struct vm_area_struct * vma)
{
	return -ENOSYS;
}
1316 1317 1318 1319
int generic_file_readonly_mmap(struct file * file, struct vm_area_struct * vma)
{
	return -ENOSYS;
}
1320
#endif /* CONFIG_MMU */
Linus Torvalds's avatar
Linus Torvalds committed
1321

1322
static inline struct page *__read_cache_page(struct address_space *mapping,
Linus Torvalds's avatar
Linus Torvalds committed
1323 1324 1325 1326 1327 1328 1329
				unsigned long index,
				int (*filler)(void *,struct page*),
				void *data)
{
	struct page *page, *cached_page = NULL;
	int err;
repeat:
1330
	page = find_get_page(mapping, index);
Linus Torvalds's avatar
Linus Torvalds committed
1331 1332
	if (!page) {
		if (!cached_page) {
1333
			cached_page = page_cache_alloc_cold(mapping);
Linus Torvalds's avatar
Linus Torvalds committed
1334 1335 1336
			if (!cached_page)
				return ERR_PTR(-ENOMEM);
		}
1337 1338
		err = add_to_page_cache_lru(cached_page, mapping,
					index, GFP_KERNEL);
1339
		if (err == -EEXIST)
Linus Torvalds's avatar
Linus Torvalds committed
1340
			goto repeat;
1341 1342 1343 1344 1345 1346
		if (err < 0) {
			/* Presumably ENOMEM for radix tree node */
			page_cache_release(cached_page);
			return ERR_PTR(err);
		}
		page = cached_page;
Linus Torvalds's avatar
Linus Torvalds committed
1347 1348 1349 1350 1351 1352 1353 1354
		cached_page = NULL;
		err = filler(data, page);
		if (err < 0) {
			page_cache_release(page);
			page = ERR_PTR(err);
		}
	}
	if (cached_page)
Linus Torvalds's avatar
Linus Torvalds committed
1355
		page_cache_release(cached_page);
Linus Torvalds's avatar
Linus Torvalds committed
1356 1357 1358 1359 1360
	return page;
}

/*
 * Read into the page cache. If a page already exists,
Andrew Morton's avatar
Andrew Morton committed
1361
 * and PageUptodate() is not set, try to fill the page.
Linus Torvalds's avatar
Linus Torvalds committed
1362 1363 1364 1365 1366 1367 1368 1369 1370 1371 1372
 */
struct page *read_cache_page(struct address_space *mapping,
				unsigned long index,
				int (*filler)(void *,struct page*),
				void *data)
{
	struct page *page;
	int err;

retry:
	page = __read_cache_page(mapping, index, filler, data);
Linus Torvalds's avatar
Linus Torvalds committed
1373 1374 1375
	if (IS_ERR(page))
		goto out;
	mark_page_accessed(page);
Andrew Morton's avatar
Andrew Morton committed
1376
	if (PageUptodate(page))
Linus Torvalds's avatar
Linus Torvalds committed
1377 1378 1379 1380
		goto out;

	lock_page(page);
	if (!page->mapping) {
Andrew Morton's avatar
Andrew Morton committed
1381
		unlock_page(page);
Linus Torvalds's avatar
Linus Torvalds committed
1382 1383 1384
		page_cache_release(page);
		goto retry;
	}
Andrew Morton's avatar
Andrew Morton committed
1385 1386
	if (PageUptodate(page)) {
		unlock_page(page);
Linus Torvalds's avatar
Linus Torvalds committed
1387 1388 1389 1390 1391 1392 1393 1394 1395 1396 1397
		goto out;
	}
	err = filler(data, page);
	if (err < 0) {
		page_cache_release(page);
		page = ERR_PTR(err);
	}
 out:
	return page;
}

1398 1399 1400 1401 1402 1403 1404 1405
/*
 * If the page was newly created, increment its refcount and add it to the
 * caller's lru-buffering pagevec.  This function is specifically for
 * generic_file_write().
 */
static inline struct page *
__grab_cache_page(struct address_space *mapping, unsigned long index,
			struct page **cached_page, struct pagevec *lru_pvec)
Linus Torvalds's avatar
Linus Torvalds committed
1406
{
1407 1408
	int err;
	struct page *page;
Linus Torvalds's avatar
Linus Torvalds committed
1409
repeat:
1410
	page = find_lock_page(mapping, index);
Linus Torvalds's avatar
Linus Torvalds committed
1411 1412
	if (!page) {
		if (!*cached_page) {
Linus Torvalds's avatar
Linus Torvalds committed
1413
			*cached_page = page_cache_alloc(mapping);
Linus Torvalds's avatar
Linus Torvalds committed
1414 1415 1416
			if (!*cached_page)
				return NULL;
		}
1417 1418
		err = add_to_page_cache(*cached_page, mapping,
					index, GFP_KERNEL);
1419
		if (err == -EEXIST)
Linus Torvalds's avatar
Linus Torvalds committed
1420
			goto repeat;
1421 1422
		if (err == 0) {
			page = *cached_page;
1423 1424 1425
			page_cache_get(page);
			if (!pagevec_add(lru_pvec, page))
				__pagevec_lru_add(lru_pvec);
1426 1427
			*cached_page = NULL;
		}
Linus Torvalds's avatar
Linus Torvalds committed
1428 1429 1430 1431
	}
	return page;
}

1432
void remove_suid(struct dentry *dentry)
Linus Torvalds's avatar
Linus Torvalds committed
1433
{
1434 1435 1436
	struct iattr newattrs;
	struct inode *inode = dentry->d_inode;
	unsigned int mode = inode->i_mode & (S_ISUID|S_ISGID|S_IXGRP);
Linus Torvalds's avatar
Linus Torvalds committed
1437

1438 1439
	if (!(mode & S_IXGRP))
		mode &= S_ISUID;
Linus Torvalds's avatar
Linus Torvalds committed
1440 1441 1442

	/* was any of the uid bits set? */
	if (mode && !capable(CAP_FSETID)) {
1443 1444
		newattrs.ia_valid = ATTR_KILL_SUID | ATTR_KILL_SGID;
		notify_change(dentry, &newattrs);
Linus Torvalds's avatar
Linus Torvalds committed
1445 1446 1447
	}
}

1448 1449 1450 1451 1452 1453 1454 1455 1456 1457 1458 1459 1460 1461 1462 1463 1464 1465 1466 1467
static inline int
filemap_copy_from_user(struct page *page, unsigned long offset,
			const char *buf, unsigned bytes)
{
	char *kaddr;
	int left;

	kaddr = kmap_atomic(page, KM_USER0);
	left = __copy_from_user(kaddr + offset, buf, bytes);
	kunmap_atomic(kaddr, KM_USER0);

	if (left != 0) {
		/* Do it the slow way */
		kaddr = kmap(page);
		left = __copy_from_user(kaddr + offset, buf, bytes);
		kunmap(page);
	}
	return left;
}

1468
static int
Andrew Morton's avatar
Andrew Morton committed
1469
__filemap_copy_from_user_iovec(char *vaddr, 
1470
			const struct iovec *iov, size_t base, size_t bytes)
Andrew Morton's avatar
Andrew Morton committed
1471 1472 1473 1474 1475 1476 1477 1478 1479 1480 1481 1482 1483 1484 1485 1486 1487 1488
{
	int left = 0;

	while (bytes) {
		char *buf = iov->iov_base + base;
		int copy = min(bytes, iov->iov_len - base);
		base = 0;
		if ((left = __copy_from_user(vaddr, buf, copy)))
			break;
		bytes -= copy;
		vaddr += copy;
		iov++;
	}
	return left;
}

static inline int
filemap_copy_from_user_iovec(struct page *page, unsigned long offset,
1489
			const struct iovec *iov, size_t base, size_t bytes)
Andrew Morton's avatar
Andrew Morton committed
1490 1491 1492 1493 1494 1495 1496 1497 1498 1499 1500 1501 1502 1503 1504 1505
{
	char *kaddr;
	int left;

	kaddr = kmap_atomic(page, KM_USER0);
	left = __filemap_copy_from_user_iovec(kaddr + offset, iov, base, bytes);
	kunmap_atomic(kaddr, KM_USER0);
	if (left != 0) {
		kaddr = kmap(page);
		left = __filemap_copy_from_user_iovec(kaddr + offset, iov, base, bytes);
		kunmap(page);
	}
	return left;
}

static inline void
1506
filemap_set_next_iovec(const struct iovec **iovp, size_t *basep, size_t bytes)
Andrew Morton's avatar
Andrew Morton committed
1507 1508 1509 1510 1511 1512 1513 1514 1515 1516 1517 1518 1519 1520 1521 1522 1523 1524
{
	const struct iovec *iov = *iovp;
	size_t base = *basep;

	while (bytes) {
		int copy = min(bytes, iov->iov_len - base);
		bytes -= copy;
		base += copy;
		if (iov->iov_len == base) {
			iov++;
			base = 0;
		}
	}
	*iovp = iov;
	*basep = base;
}


Linus Torvalds's avatar
Linus Torvalds committed
1525 1526 1527
/*
 * Write to a file through the page cache. 
 *
1528 1529 1530 1531
 * We put everything into the page cache prior to writing it. This is not a
 * problem when writing full pages. With partial pages, however, we first have
 * to read the data into the cache, then dirty the page, and finally schedule
 * it for writing by marking it dirty.
Linus Torvalds's avatar
Linus Torvalds committed
1532 1533
 *							okir@monad.swb.de
 */
Andrew Morton's avatar
Andrew Morton committed
1534
ssize_t
1535
generic_file_aio_write_nolock(struct kiocb *iocb, const struct iovec *iov,
Andrew Morton's avatar
Andrew Morton committed
1536
				unsigned long nr_segs, loff_t *ppos)
Linus Torvalds's avatar
Linus Torvalds committed
1537
{
1538
	struct file *file = iocb->ki_filp;
1539 1540
	struct address_space * mapping = file->f_dentry->d_inode->i_mapping;
	struct address_space_operations *a_ops = mapping->a_ops;
1541 1542
	size_t ocount;		/* original count */
	size_t count;		/* after file limit checks */
1543
	struct inode 	*inode = mapping->host;
Linus Torvalds's avatar
Linus Torvalds committed
1544
	unsigned long	limit = current->rlim[RLIMIT_FSIZE].rlim_cur;
1545
	const int	isblk = S_ISBLK(inode->i_mode);
1546
	long		status = 0;
Linus Torvalds's avatar
Linus Torvalds committed
1547
	loff_t		pos;
1548 1549
	struct page	*page;
	struct page	*cached_page = NULL;
Linus Torvalds's avatar
Linus Torvalds committed
1550
	ssize_t		written;
Linus Torvalds's avatar
Linus Torvalds committed
1551
	int		err;
1552
	size_t		bytes;
1553
	struct pagevec	lru_pvec;
Andrew Morton's avatar
Andrew Morton committed
1554
	const struct iovec *cur_iov = iov; /* current iovec */
1555
	size_t		iov_base = 0;	   /* offset in the current iovec */
Andrew Morton's avatar
Andrew Morton committed
1556 1557
	unsigned long	seg;
	char		*buf;
Linus Torvalds's avatar
Linus Torvalds committed
1558

1559
	ocount = 0;
Andrew Morton's avatar
Andrew Morton committed
1560
	for (seg = 0; seg < nr_segs; seg++) {
1561 1562 1563 1564 1565 1566 1567 1568 1569 1570 1571 1572
		const struct iovec *iv = &iov[seg];

		/*
		 * If any segment has a negative length, or the cumulative
		 * length ever wraps negative then return -EINVAL.
		 */
		ocount += iv->iov_len;
		if (unlikely((ssize_t)(ocount|iv->iov_len) < 0))
			return -EINVAL;
		if (access_ok(VERIFY_READ, iv->iov_base, iv->iov_len))
			continue;
		if (seg == 0)
Andrew Morton's avatar
Andrew Morton committed
1573
			return -EFAULT;
1574
		nr_segs = seg;
1575
		ocount -= iv->iov_len;	/* This segment is no good */
1576
		break;
Andrew Morton's avatar
Andrew Morton committed
1577
	}
1578
	count = ocount;
Linus Torvalds's avatar
Linus Torvalds committed
1579

Linus Torvalds's avatar
Linus Torvalds committed
1580
	pos = *ppos;
1581 1582
	if (unlikely(pos < 0))
		return -EINVAL;
Linus Torvalds's avatar
Linus Torvalds committed
1583

1584 1585 1586
	/* We can write back this queue in page reclaim */
	current->backing_dev_info = mapping->backing_dev_info;

1587
	pagevec_init(&lru_pvec, 0);
1588

1589 1590
	if (unlikely(file->f_error)) {
		err = file->f_error;
Linus Torvalds's avatar
Linus Torvalds committed
1591 1592 1593 1594 1595 1596
		file->f_error = 0;
		goto out;
	}

	written = 0;

1597 1598 1599 1600
	if (!isblk) {
		/* FIXME: this is for backwards compatibility with 2.4 */
		if (file->f_flags & O_APPEND)
			pos = inode->i_size;
Linus Torvalds's avatar
Linus Torvalds committed
1601

1602 1603 1604 1605 1606 1607 1608 1609 1610 1611
		if (limit != RLIM_INFINITY) {
			if (pos >= limit) {
				send_sig(SIGXFSZ, current, 0);
				err = -EFBIG;
				goto out;
			}
			if (pos > 0xFFFFFFFFULL || count > limit - (u32)pos) {
				/* send_sig(SIGXFSZ, current, 0); */
				count = limit - (u32)pos;
			}
Linus Torvalds's avatar
Linus Torvalds committed
1612 1613 1614 1615
		}
	}

	/*
1616
	 * LFS rule
Linus Torvalds's avatar
Linus Torvalds committed
1617
	 */
1618 1619
	if (unlikely(pos + count > MAX_NON_LFS &&
				!(file->f_flags & O_LARGEFILE))) {
Linus Torvalds's avatar
Linus Torvalds committed
1620
		if (pos >= MAX_NON_LFS) {
Linus Torvalds's avatar
Linus Torvalds committed
1621
			send_sig(SIGXFSZ, current, 0);
1622
			err = -EFBIG;
Linus Torvalds's avatar
Linus Torvalds committed
1623 1624 1625 1626 1627
			goto out;
		}
		if (count > MAX_NON_LFS - (u32)pos) {
			/* send_sig(SIGXFSZ, current, 0); */
			count = MAX_NON_LFS - (u32)pos;
Linus Torvalds's avatar
Linus Torvalds committed
1628 1629 1630
		}
	}

Linus Torvalds's avatar
Linus Torvalds committed
1631
	/*
1632
	 * Are we about to exceed the fs block limit ?
Linus Torvalds's avatar
Linus Torvalds committed
1633
	 *
1634 1635 1636
	 * If we have written data it becomes a short write.  If we have
	 * exceeded without writing data we send a signal and return EFBIG.
	 * Linus frestrict idea will clean these up nicely..
Linus Torvalds's avatar
Linus Torvalds committed
1637
	 */
1638
	if (likely(!isblk)) {
1639
		if (unlikely(pos >= inode->i_sb->s_maxbytes)) {
Linus Torvalds's avatar
Linus Torvalds committed
1640 1641 1642 1643 1644 1645 1646 1647
			if (count || pos > inode->i_sb->s_maxbytes) {
				send_sig(SIGXFSZ, current, 0);
				err = -EFBIG;
				goto out;
			}
			/* zero-length writes at ->s_maxbytes are OK */
		}

1648
		if (unlikely(pos + count > inode->i_sb->s_maxbytes))
Linus Torvalds's avatar
Linus Torvalds committed
1649 1650
			count = inode->i_sb->s_maxbytes - pos;
	} else {
Alexander Viro's avatar
Alexander Viro committed
1651
		if (bdev_read_only(inode->i_bdev)) {
Linus Torvalds's avatar
Linus Torvalds committed
1652
			err = -EPERM;
Linus Torvalds's avatar
Linus Torvalds committed
1653 1654
			goto out;
		}
Linus Torvalds's avatar
Linus Torvalds committed
1655 1656
		if (pos >= inode->i_size) {
			if (count || pos > inode->i_size) {
Linus Torvalds's avatar
Linus Torvalds committed
1657 1658 1659 1660
				err = -ENOSPC;
				goto out;
			}
		}
Linus Torvalds's avatar
Linus Torvalds committed
1661

Linus Torvalds's avatar
Linus Torvalds committed
1662 1663
		if (pos + count > inode->i_size)
			count = inode->i_size - pos;
Linus Torvalds's avatar
Linus Torvalds committed
1664
	}
Linus Torvalds's avatar
Linus Torvalds committed
1665

Linus Torvalds's avatar
Linus Torvalds committed
1666 1667
	err = 0;
	if (count == 0)
Linus Torvalds's avatar
Linus Torvalds committed
1668 1669
		goto out;

1670
	remove_suid(file->f_dentry);
1671
	inode_update_time(inode, 1);
Linus Torvalds's avatar
Linus Torvalds committed
1672

Andrew Morton's avatar
Andrew Morton committed
1673
	/* coalesce the iovecs and go direct-to-BIO for O_DIRECT */
1674
	if (unlikely(file->f_flags & O_DIRECT)) {
Andrew Morton's avatar
Andrew Morton committed
1675 1676 1677
		if (count != ocount)
			nr_segs = iov_shorten((struct iovec *)iov,
						nr_segs, count);
1678
		written = generic_file_direct_IO(WRITE, iocb,
Andrew Morton's avatar
Andrew Morton committed
1679
					iov, pos, nr_segs);
1680 1681
		if (written > 0) {
			loff_t end = pos + written;
1682
			if (end > inode->i_size && !isblk) {
1683 1684 1685 1686 1687 1688 1689 1690 1691 1692 1693 1694 1695
				inode->i_size = end;
				mark_inode_dirty(inode);
			}
			*ppos = end;
		}
		/*
		 * Sync the fs metadata but not the minor inode changes and
		 * of course not the data as we did direct DMA for the IO.
		 */
		if (written >= 0 && file->f_flags & O_SYNC)
			status = generic_osync_inode(inode, OSYNC_METADATA);
		goto out_status;
	}
Linus Torvalds's avatar
Linus Torvalds committed
1696

Andrew Morton's avatar
Andrew Morton committed
1697
	buf = iov->iov_base;
Linus Torvalds's avatar
Linus Torvalds committed
1698
	do {
1699 1700
		unsigned long index;
		unsigned long offset;
Linus Torvalds's avatar
Linus Torvalds committed
1701
		long page_fault;
Linus Torvalds's avatar
Linus Torvalds committed
1702 1703 1704 1705

		offset = (pos & (PAGE_CACHE_SIZE -1)); /* Within page */
		index = pos >> PAGE_CACHE_SHIFT;
		bytes = PAGE_CACHE_SIZE - offset;
Linus Torvalds's avatar
Linus Torvalds committed
1706
		if (bytes > count)
Linus Torvalds's avatar
Linus Torvalds committed
1707 1708 1709 1710 1711 1712 1713 1714
			bytes = count;

		/*
		 * Bring in the user page that we will copy from _first_.
		 * Otherwise there's a nasty deadlock on copying from the
		 * same page as we're writing to, without it being marked
		 * up-to-date.
		 */
1715
		fault_in_pages_readable(buf, bytes);
Linus Torvalds's avatar
Linus Torvalds committed
1716

Andrew Morton's avatar
Andrew Morton committed
1717
		page = __grab_cache_page(mapping,index,&cached_page,&lru_pvec);
1718 1719
		if (!page) {
			status = -ENOMEM;
Linus Torvalds's avatar
Linus Torvalds committed
1720 1721 1722
			break;
		}

1723 1724 1725 1726 1727 1728 1729 1730 1731 1732 1733 1734
		status = a_ops->prepare_write(file, page, offset, offset+bytes);
		if (unlikely(status)) {
			/*
			 * prepare_write() may have instantiated a few blocks
			 * outside i_size.  Trim these off again.
			 */
			unlock_page(page);
			page_cache_release(page);
			if (pos + bytes > inode->i_size)
				vmtruncate(inode, inode->i_size);
			break;
		}
Andrew Morton's avatar
Andrew Morton committed
1735 1736 1737 1738 1739 1740
		if (likely(nr_segs == 1))
			page_fault = filemap_copy_from_user(page, offset,
							buf, bytes);
		else
			page_fault = filemap_copy_from_user_iovec(page, offset,
						cur_iov, iov_base, bytes);
1741
		flush_dcache_page(page);
1742 1743 1744 1745 1746 1747 1748 1749 1750 1751 1752 1753
		status = a_ops->commit_write(file, page, offset, offset+bytes);
		if (unlikely(page_fault)) {
			status = -EFAULT;
		} else {
			if (!status)
				status = bytes;

			if (status >= 0) {
				written += status;
				count -= status;
				pos += status;
				buf += status;
Andrew Morton's avatar
Andrew Morton committed
1754 1755 1756
				if (unlikely(nr_segs > 1))
					filemap_set_next_iovec(&cur_iov,
							&iov_base, status);
1757
			}
Linus Torvalds's avatar
Linus Torvalds committed
1758
		}
1759 1760
		if (!PageReferenced(page))
			SetPageReferenced(page);
Andrew Morton's avatar
Andrew Morton committed
1761
		unlock_page(page);
Linus Torvalds's avatar
Linus Torvalds committed
1762 1763 1764
		page_cache_release(page);
		if (status < 0)
			break;
1765
		balance_dirty_pages_ratelimited(mapping);
1766
		cond_resched();
Linus Torvalds's avatar
Linus Torvalds committed
1767
	} while (count);
Linus Torvalds's avatar
Linus Torvalds committed
1768 1769 1770
	*ppos = pos;

	if (cached_page)
Linus Torvalds's avatar
Linus Torvalds committed
1771
		page_cache_release(cached_page);
Linus Torvalds's avatar
Linus Torvalds committed
1772

1773 1774 1775
	/*
	 * For now, when the user asks for O_SYNC, we'll actually give O_DSYNC
	 */
Linus Torvalds's avatar
Linus Torvalds committed
1776 1777
	if (status >= 0) {
		if ((file->f_flags & O_SYNC) || IS_SYNC(inode))
1778 1779
			status = generic_osync_inode(inode,
					OSYNC_METADATA|OSYNC_DATA);
Linus Torvalds's avatar
Linus Torvalds committed
1780
	}
Linus Torvalds's avatar
Linus Torvalds committed
1781
	
Linus Torvalds's avatar
Linus Torvalds committed
1782
out_status:	
Linus Torvalds's avatar
Linus Torvalds committed
1783 1784
	err = written ? written : status;
out:
1785
	pagevec_lru_add(&lru_pvec);
1786
	current->backing_dev_info = 0;
1787 1788 1789
	return err;
}

1790 1791 1792 1793 1794 1795 1796 1797 1798 1799 1800 1801 1802 1803
ssize_t
generic_file_write_nolock(struct file *file, const struct iovec *iov,
				unsigned long nr_segs, loff_t *ppos)
{
	struct kiocb kiocb;
	ssize_t ret;

	init_sync_kiocb(&kiocb, file);
	ret = generic_file_aio_write_nolock(&kiocb, iov, nr_segs, ppos);
	if (-EIOCBQUEUED == ret)
		ret = wait_on_sync_kiocb(&kiocb);
	return ret;
}

1804 1805 1806
ssize_t generic_file_aio_write(struct kiocb *iocb, const char *buf,
			       size_t count, loff_t pos)
{
1807 1808 1809 1810 1811 1812 1813 1814 1815 1816 1817 1818 1819
	struct file *file = iocb->ki_filp;
	struct inode *inode = file->f_dentry->d_inode->i_mapping->host;
	int err;
	struct iovec local_iov = { .iov_base = (void *)buf, .iov_len = count };

	BUG_ON(iocb->ki_pos != pos);

	down(&inode->i_sem);
	err = generic_file_aio_write_nolock(iocb, &local_iov, 1, 
						&iocb->ki_pos);
	up(&inode->i_sem);

	return err;
1820
}
1821
EXPORT_SYMBOL(generic_file_aio_write);
1822
EXPORT_SYMBOL(generic_file_aio_write_nolock);
1823

1824 1825 1826 1827 1828
ssize_t generic_file_write(struct file *file, const char *buf,
			   size_t count, loff_t *ppos)
{
	struct inode	*inode = file->f_dentry->d_inode->i_mapping->host;
	int		err;
Andrew Morton's avatar
Andrew Morton committed
1829
	struct iovec local_iov = { .iov_base = (void *)buf, .iov_len = count };
1830 1831

	down(&inode->i_sem);
Andrew Morton's avatar
Andrew Morton committed
1832
	err = generic_file_write_nolock(file, &local_iov, 1, ppos);
Linus Torvalds's avatar
Linus Torvalds committed
1833
	up(&inode->i_sem);
1834

Linus Torvalds's avatar
Linus Torvalds committed
1835 1836
	return err;
}
Andrew Morton's avatar
Andrew Morton committed
1837 1838 1839 1840

ssize_t generic_file_readv(struct file *filp, const struct iovec *iov,
			unsigned long nr_segs, loff_t *ppos)
{
1841 1842 1843 1844 1845 1846 1847 1848
	struct kiocb kiocb;
	ssize_t ret;

	init_sync_kiocb(&kiocb, filp);
	ret = __generic_file_aio_read(&kiocb, iov, nr_segs, ppos);
	if (-EIOCBQUEUED == ret)
		ret = wait_on_sync_kiocb(&kiocb);
	return ret;
Andrew Morton's avatar
Andrew Morton committed
1849 1850 1851 1852 1853 1854 1855 1856 1857 1858 1859 1860 1861
}

ssize_t generic_file_writev(struct file *file, const struct iovec *iov,
			unsigned long nr_segs, loff_t * ppos) 
{
	struct inode *inode = file->f_dentry->d_inode;
	ssize_t ret;

	down(&inode->i_sem);
	ret = generic_file_write_nolock(file, iov, nr_segs, ppos);
	up(&inode->i_sem);
	return ret;
}
1862 1863 1864 1865 1866 1867 1868 1869 1870 1871 1872 1873 1874 1875 1876 1877 1878 1879 1880 1881 1882 1883 1884

ssize_t
generic_file_direct_IO(int rw, struct kiocb *iocb, const struct iovec *iov,
	loff_t offset, unsigned long nr_segs)
{
	struct file *file = iocb->ki_filp;
	struct address_space *mapping = file->f_dentry->d_inode->i_mapping;
	ssize_t retval;

	if (mapping->nrpages) {
		retval = filemap_fdatawrite(mapping);
		if (retval == 0)
			retval = filemap_fdatawait(mapping);
		if (retval)
			goto out;
	}

	retval = mapping->a_ops->direct_IO(rw, iocb, iov, offset, nr_segs);
	if (rw == WRITE && mapping->nrpages)
		invalidate_inode_pages2(mapping);
out:
	return retval;
}