nommu.c 51.9 KB
Newer Older
Linus Torvalds's avatar
Linus Torvalds committed
1 2 3 4 5 6 7 8
/*
 *  linux/mm/nommu.c
 *
 *  Replacement code for mm functions to support CPU's that don't
 *  have any form of memory management unit (thus no virtual memory).
 *
 *  See Documentation/nommu-mmap.txt
 *
9
 *  Copyright (c) 2004-2008 David Howells <dhowells@redhat.com>
Linus Torvalds's avatar
Linus Torvalds committed
10 11 12
 *  Copyright (c) 2000-2003 David McCullough <davidm@snapgear.com>
 *  Copyright (c) 2000-2001 D Jeff Dionne <jeff@uClinux.org>
 *  Copyright (c) 2002      Greg Ungerer <gerg@snapgear.com>
13
 *  Copyright (c) 2007-2010 Paul Mundt <lethal@linux-sh.org>
Linus Torvalds's avatar
Linus Torvalds committed
14 15
 */

16 17
#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt

18
#include <linux/export.h>
Linus Torvalds's avatar
Linus Torvalds committed
19
#include <linux/mm.h>
Davidlohr Bueso's avatar
Davidlohr Bueso committed
20
#include <linux/vmacache.h>
Linus Torvalds's avatar
Linus Torvalds committed
21 22 23 24 25 26 27 28 29
#include <linux/mman.h>
#include <linux/swap.h>
#include <linux/file.h>
#include <linux/highmem.h>
#include <linux/pagemap.h>
#include <linux/slab.h>
#include <linux/vmalloc.h>
#include <linux/blkdev.h>
#include <linux/backing-dev.h>
30
#include <linux/compiler.h>
Linus Torvalds's avatar
Linus Torvalds committed
31 32 33 34
#include <linux/mount.h>
#include <linux/personality.h>
#include <linux/security.h>
#include <linux/syscalls.h>
Al Viro's avatar
Al Viro committed
35
#include <linux/audit.h>
36
#include <linux/sched/sysctl.h>
37
#include <linux/printk.h>
Linus Torvalds's avatar
Linus Torvalds committed
38 39 40 41

#include <asm/uaccess.h>
#include <asm/tlb.h>
#include <asm/tlbflush.h>
42
#include <asm/mmu_context.h>
43 44
#include "internal.h"

Linus Torvalds's avatar
Linus Torvalds committed
45
void *high_memory;
46
EXPORT_SYMBOL(high_memory);
Linus Torvalds's avatar
Linus Torvalds committed
47 48
struct page *mem_map;
unsigned long max_mapnr;
49
EXPORT_SYMBOL(max_mapnr);
50
unsigned long highest_memmap_pfn;
51
struct percpu_counter vm_committed_as;
Linus Torvalds's avatar
Linus Torvalds committed
52 53
int sysctl_overcommit_memory = OVERCOMMIT_GUESS; /* heuristic overcommit */
int sysctl_overcommit_ratio = 50; /* default is 50% */
54
unsigned long sysctl_overcommit_kbytes __read_mostly;
Linus Torvalds's avatar
Linus Torvalds committed
55
int sysctl_max_map_count = DEFAULT_MAX_MAP_COUNT;
56
int sysctl_nr_trim_pages = CONFIG_NOMMU_INITIAL_TRIM_EXCESS;
57
unsigned long sysctl_user_reserve_kbytes __read_mostly = 1UL << 17; /* 128MB */
58
unsigned long sysctl_admin_reserve_kbytes __read_mostly = 1UL << 13; /* 8MB */
Linus Torvalds's avatar
Linus Torvalds committed
59 60
int heap_stack_gap = 0;

61
atomic_long_t mmap_pages_allocated;
62

63 64 65 66 67 68 69 70 71 72 73 74 75 76 77
/*
 * The global memory commitment made in the system can be a metric
 * that can be used to drive ballooning decisions when Linux is hosted
 * as a guest. On Hyper-V, the host implements a policy engine for dynamically
 * balancing memory across competing virtual machines that are hosted.
 * Several metrics drive this policy engine including the guest reported
 * memory commitment.
 */
unsigned long vm_memory_committed(void)
{
	return percpu_counter_read_positive(&vm_committed_as);
}

EXPORT_SYMBOL_GPL(vm_memory_committed);

Linus Torvalds's avatar
Linus Torvalds committed
78 79
EXPORT_SYMBOL(mem_map);

80 81 82 83
/* list of mapped, potentially shareable regions */
static struct kmem_cache *vm_region_jar;
struct rb_root nommu_region_tree = RB_ROOT;
DECLARE_RWSEM(nommu_region_sem);
Linus Torvalds's avatar
Linus Torvalds committed
84

85
const struct vm_operations_struct generic_file_vm_ops = {
Linus Torvalds's avatar
Linus Torvalds committed
86 87 88 89 90 91 92 93 94 95 96 97
};

/*
 * Return the total memory allocated for this pointer, not
 * just what the caller asked for.
 *
 * Doesn't have to be accurate, i.e. may have races.
 */
unsigned int kobjsize(const void *objp)
{
	struct page *page;

98 99 100 101
	/*
	 * If the object we have should not have ksize performed on it,
	 * return size of 0
	 */
102
	if (!objp || !virt_addr_valid(objp))
103 104 105 106 107 108 109 110
		return 0;

	page = virt_to_head_page(objp);

	/*
	 * If the allocator sets PageSlab, we know the pointer came from
	 * kmalloc().
	 */
Linus Torvalds's avatar
Linus Torvalds committed
111 112 113
	if (PageSlab(page))
		return ksize(objp);

114 115 116 117 118 119 120 121 122 123 124 125 126 127
	/*
	 * If it's not a compound page, see if we have a matching VMA
	 * region. This test is intentionally done in reverse order,
	 * so if there's no VMA, we still fall through and hand back
	 * PAGE_SIZE for 0-order pages.
	 */
	if (!PageCompound(page)) {
		struct vm_area_struct *vma;

		vma = find_vma(current->mm, (unsigned long)objp);
		if (vma)
			return vma->vm_end - vma->vm_start;
	}

128 129
	/*
	 * The ksize() function is only guaranteed to work for pointers
130
	 * returned by kmalloc(). So handle arbitrary pointers here.
131
	 */
132
	return PAGE_SIZE << compound_order(page);
Linus Torvalds's avatar
Linus Torvalds committed
133 134
}

135 136 137 138
long __get_user_pages(struct task_struct *tsk, struct mm_struct *mm,
		      unsigned long start, unsigned long nr_pages,
		      unsigned int foll_flags, struct page **pages,
		      struct vm_area_struct **vmas, int *nonblocking)
Linus Torvalds's avatar
Linus Torvalds committed
139
{
140
	struct vm_area_struct *vma;
141 142 143 144
	unsigned long vm_flags;
	int i;

	/* calculate required read or write permissions.
Hugh Dickins's avatar
Hugh Dickins committed
145
	 * If FOLL_FORCE is set, we only require the "MAY" flags.
146
	 */
Hugh Dickins's avatar
Hugh Dickins committed
147 148 149 150
	vm_flags  = (foll_flags & FOLL_WRITE) ?
			(VM_WRITE | VM_MAYWRITE) : (VM_READ | VM_MAYREAD);
	vm_flags &= (foll_flags & FOLL_FORCE) ?
			(VM_MAYREAD | VM_MAYWRITE) : (VM_READ | VM_WRITE);
Linus Torvalds's avatar
Linus Torvalds committed
151

152
	for (i = 0; i < nr_pages; i++) {
153
		vma = find_vma(mm, start);
154 155 156 157
		if (!vma)
			goto finish_or_fault;

		/* protect what we can, including chardevs */
Hugh Dickins's avatar
Hugh Dickins committed
158 159
		if ((vma->vm_flags & (VM_IO | VM_PFNMAP)) ||
		    !(vm_flags & vma->vm_flags))
160
			goto finish_or_fault;
161

Linus Torvalds's avatar
Linus Torvalds committed
162 163 164 165 166 167
		if (pages) {
			pages[i] = virt_to_page(start);
			if (pages[i])
				page_cache_get(pages[i]);
		}
		if (vmas)
168
			vmas[i] = vma;
169
		start = (start + PAGE_SIZE) & PAGE_MASK;
Linus Torvalds's avatar
Linus Torvalds committed
170
	}
171 172 173 174 175

	return i;

finish_or_fault:
	return i ? : -EFAULT;
Linus Torvalds's avatar
Linus Torvalds committed
176
}
177 178 179 180 181 182 183 184

/*
 * get a list of pages in an address range belonging to the specified process
 * and indicate the VMA that covers each page
 * - this is potentially dodgy as we may end incrementing the page count of a
 *   slab page or a secondary page from a compound page
 * - don't permit access to VMAs that don't support it, such as I/O mappings
 */
185 186
long get_user_pages(struct task_struct *tsk, struct mm_struct *mm,
		    unsigned long start, unsigned long nr_pages,
187
		    unsigned int gup_flags, struct page **pages,
188
		    struct vm_area_struct **vmas)
189
{
190 191
	return __get_user_pages(tsk, mm, start, nr_pages,
				gup_flags, pages, vmas, NULL);
192
}
193 194
EXPORT_SYMBOL(get_user_pages);

195 196
long get_user_pages_locked(struct task_struct *tsk, struct mm_struct *mm,
			   unsigned long start, unsigned long nr_pages,
197
			   unsigned int gup_flags, struct page **pages,
198 199
			   int *locked)
{
200
	return get_user_pages(tsk, mm, start, nr_pages, gup_flags,
201 202 203 204
			      pages, NULL);
}
EXPORT_SYMBOL(get_user_pages_locked);

205 206
long __get_user_pages_unlocked(struct task_struct *tsk, struct mm_struct *mm,
			       unsigned long start, unsigned long nr_pages,
207
			       struct page **pages, unsigned int gup_flags)
208 209 210
{
	long ret;
	down_read(&mm->mmap_sem);
211 212
	ret = __get_user_pages(tsk, mm, start, nr_pages, gup_flags, pages,
			       NULL, NULL);
213 214 215
	up_read(&mm->mmap_sem);
	return ret;
}
216 217 218 219
EXPORT_SYMBOL(__get_user_pages_unlocked);

long get_user_pages_unlocked(struct task_struct *tsk, struct mm_struct *mm,
			     unsigned long start, unsigned long nr_pages,
220
			     struct page **pages, unsigned int gup_flags)
221
{
222
	return __get_user_pages_unlocked(tsk, mm, start, nr_pages,
223
					 pages, gup_flags);
224
}
225 226
EXPORT_SYMBOL(get_user_pages_unlocked);

Paul Mundt's avatar
Paul Mundt committed
227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247
/**
 * follow_pfn - look up PFN at a user virtual address
 * @vma: memory mapping
 * @address: user virtual address
 * @pfn: location to store found PFN
 *
 * Only IO mappings and raw PFN mappings are allowed.
 *
 * Returns zero and the pfn at @pfn on success, -ve otherwise.
 */
int follow_pfn(struct vm_area_struct *vma, unsigned long address,
	unsigned long *pfn)
{
	if (!(vma->vm_flags & (VM_IO | VM_PFNMAP)))
		return -EINVAL;

	*pfn = address >> PAGE_SHIFT;
	return 0;
}
EXPORT_SYMBOL(follow_pfn);

248
LIST_HEAD(vmap_area_list);
Linus Torvalds's avatar
Linus Torvalds committed
249

250
void vfree(const void *addr)
Linus Torvalds's avatar
Linus Torvalds committed
251 252 253
{
	kfree(addr);
}
254
EXPORT_SYMBOL(vfree);
Linus Torvalds's avatar
Linus Torvalds committed
255

256
void *__vmalloc(unsigned long size, gfp_t gfp_mask, pgprot_t prot)
Linus Torvalds's avatar
Linus Torvalds committed
257 258
{
	/*
259 260
	 *  You can't specify __GFP_HIGHMEM with kmalloc() since kmalloc()
	 * returns only a logical address.
Linus Torvalds's avatar
Linus Torvalds committed
261
	 */
262
	return kmalloc(size, (gfp_mask | __GFP_COMP) & ~__GFP_HIGHMEM);
Linus Torvalds's avatar
Linus Torvalds committed
263
}
264
EXPORT_SYMBOL(__vmalloc);
Linus Torvalds's avatar
Linus Torvalds committed
265

266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285
void *vmalloc_user(unsigned long size)
{
	void *ret;

	ret = __vmalloc(size, GFP_KERNEL | __GFP_HIGHMEM | __GFP_ZERO,
			PAGE_KERNEL);
	if (ret) {
		struct vm_area_struct *vma;

		down_write(&current->mm->mmap_sem);
		vma = find_vma(current->mm, (unsigned long)ret);
		if (vma)
			vma->vm_flags |= VM_USERMAP;
		up_write(&current->mm->mmap_sem);
	}

	return ret;
}
EXPORT_SYMBOL(vmalloc_user);

286
struct page *vmalloc_to_page(const void *addr)
Linus Torvalds's avatar
Linus Torvalds committed
287 288 289
{
	return virt_to_page(addr);
}
290
EXPORT_SYMBOL(vmalloc_to_page);
Linus Torvalds's avatar
Linus Torvalds committed
291

292
unsigned long vmalloc_to_pfn(const void *addr)
Linus Torvalds's avatar
Linus Torvalds committed
293 294 295
{
	return page_to_pfn(virt_to_page(addr));
}
296
EXPORT_SYMBOL(vmalloc_to_pfn);
Linus Torvalds's avatar
Linus Torvalds committed
297 298 299

long vread(char *buf, char *addr, unsigned long count)
{
300 301 302 303
	/* Don't allow overflow */
	if ((unsigned long) buf + count < count)
		count = -(unsigned long) buf;

Linus Torvalds's avatar
Linus Torvalds committed
304 305 306 307 308 309 310 311 312 313 314
	memcpy(buf, addr, count);
	return count;
}

long vwrite(char *buf, char *addr, unsigned long count)
{
	/* Don't allow overflow */
	if ((unsigned long) addr + count < count)
		count = -(unsigned long) addr;

	memcpy(addr, buf, count);
315
	return count;
Linus Torvalds's avatar
Linus Torvalds committed
316 317 318
}

/*
319
 *	vmalloc  -  allocate virtually contiguous memory
Linus Torvalds's avatar
Linus Torvalds committed
320 321 322 323
 *
 *	@size:		allocation size
 *
 *	Allocate enough pages to cover @size from the page level
324
 *	allocator and map them into contiguous kernel virtual space.
Linus Torvalds's avatar
Linus Torvalds committed
325
 *
326
 *	For tight control over page level allocator and protection flags
Linus Torvalds's avatar
Linus Torvalds committed
327 328 329 330 331 332
 *	use __vmalloc() instead.
 */
void *vmalloc(unsigned long size)
{
       return __vmalloc(size, GFP_KERNEL | __GFP_HIGHMEM, PAGE_KERNEL);
}
333 334
EXPORT_SYMBOL(vmalloc);

335
/*
336
 *	vzalloc - allocate virtually contiguous memory with zero fill
337 338 339 340
 *
 *	@size:		allocation size
 *
 *	Allocate enough pages to cover @size from the page level
341
 *	allocator and map them into contiguous kernel virtual space.
342 343 344 345 346 347 348 349 350 351 352 353 354 355 356 357 358 359 360 361 362 363 364
 *	The memory allocated is set to zero.
 *
 *	For tight control over page level allocator and protection flags
 *	use __vmalloc() instead.
 */
void *vzalloc(unsigned long size)
{
	return __vmalloc(size, GFP_KERNEL | __GFP_HIGHMEM | __GFP_ZERO,
			PAGE_KERNEL);
}
EXPORT_SYMBOL(vzalloc);

/**
 * vmalloc_node - allocate memory on a specific node
 * @size:	allocation size
 * @node:	numa node
 *
 * Allocate enough pages to cover @size from the page level
 * allocator and map them into contiguous kernel virtual space.
 *
 * For tight control over page level allocator and protection flags
 * use __vmalloc() instead.
 */
365 366 367 368
void *vmalloc_node(unsigned long size, int node)
{
	return vmalloc(size);
}
369
EXPORT_SYMBOL(vmalloc_node);
370 371 372 373 374 375 376 377 378 379 380 381 382 383 384 385 386 387

/**
 * vzalloc_node - allocate memory on a specific node with zero fill
 * @size:	allocation size
 * @node:	numa node
 *
 * Allocate enough pages to cover @size from the page level
 * allocator and map them into contiguous kernel virtual space.
 * The memory allocated is set to zero.
 *
 * For tight control over page level allocator and protection flags
 * use __vmalloc() instead.
 */
void *vzalloc_node(unsigned long size, int node)
{
	return vzalloc(size);
}
EXPORT_SYMBOL(vzalloc_node);
Linus Torvalds's avatar
Linus Torvalds committed
388

Paul Mundt's avatar
Paul Mundt committed
389 390 391 392 393 394 395 396 397 398 399 400 401 402 403 404 405 406 407 408 409
#ifndef PAGE_KERNEL_EXEC
# define PAGE_KERNEL_EXEC PAGE_KERNEL
#endif

/**
 *	vmalloc_exec  -  allocate virtually contiguous, executable memory
 *	@size:		allocation size
 *
 *	Kernel-internal function to allocate enough pages to cover @size
 *	the page level allocator and map them into contiguous and
 *	executable kernel virtual space.
 *
 *	For tight control over page level allocator and protection flags
 *	use __vmalloc() instead.
 */

void *vmalloc_exec(unsigned long size)
{
	return __vmalloc(size, GFP_KERNEL | __GFP_HIGHMEM, PAGE_KERNEL_EXEC);
}

410 411
/**
 * vmalloc_32  -  allocate virtually contiguous memory (32bit addressable)
Linus Torvalds's avatar
Linus Torvalds committed
412 413 414
 *	@size:		allocation size
 *
 *	Allocate enough 32bit PA addressable pages to cover @size from the
415
 *	page level allocator and map them into contiguous kernel virtual space.
Linus Torvalds's avatar
Linus Torvalds committed
416 417 418 419 420
 */
void *vmalloc_32(unsigned long size)
{
	return __vmalloc(size, GFP_KERNEL, PAGE_KERNEL);
}
421 422 423 424 425 426 427 428
EXPORT_SYMBOL(vmalloc_32);

/**
 * vmalloc_32_user - allocate zeroed virtually contiguous 32bit memory
 *	@size:		allocation size
 *
 * The resulting memory area is 32bit addressable and zeroed so it can be
 * mapped to userspace without leaking data.
429 430 431
 *
 * VM_USERMAP is set on the corresponding VMA so that subsequent calls to
 * remap_vmalloc_range() are permissible.
432 433 434
 */
void *vmalloc_32_user(unsigned long size)
{
435 436 437 438 439
	/*
	 * We'll have to sort out the ZONE_DMA bits for 64-bit,
	 * but for now this can simply use vmalloc_user() directly.
	 */
	return vmalloc_user(size);
440 441
}
EXPORT_SYMBOL(vmalloc_32_user);
Linus Torvalds's avatar
Linus Torvalds committed
442 443 444 445 446 447

void *vmap(struct page **pages, unsigned int count, unsigned long flags, pgprot_t prot)
{
	BUG();
	return NULL;
}
448
EXPORT_SYMBOL(vmap);
Linus Torvalds's avatar
Linus Torvalds committed
449

450
void vunmap(const void *addr)
Linus Torvalds's avatar
Linus Torvalds committed
451 452 453
{
	BUG();
}
454
EXPORT_SYMBOL(vunmap);
Linus Torvalds's avatar
Linus Torvalds committed
455

456 457 458 459 460 461 462 463 464 465 466 467 468 469 470 471 472 473
void *vm_map_ram(struct page **pages, unsigned int count, int node, pgprot_t prot)
{
	BUG();
	return NULL;
}
EXPORT_SYMBOL(vm_map_ram);

void vm_unmap_ram(const void *mem, unsigned int count)
{
	BUG();
}
EXPORT_SYMBOL(vm_unmap_ram);

void vm_unmap_aliases(void)
{
}
EXPORT_SYMBOL_GPL(vm_unmap_aliases);

474 475 476 477
/*
 * Implement a stub for vmalloc_sync_all() if the architecture chose not to
 * have one.
 */
478
void __weak vmalloc_sync_all(void)
479 480 481
{
}

482 483 484 485 486 487 488 489 490 491 492 493
/**
 *	alloc_vm_area - allocate a range of kernel address space
 *	@size:		size of the area
 *
 *	Returns:	NULL on failure, vm_struct on success
 *
 *	This function reserves a range of kernel address space, and
 *	allocates pagetables to map that range.  No actual mappings
 *	are created.  If the kernel address space is not shared
 *	between processes, it syncs the pagetable across all
 *	processes.
 */
494
struct vm_struct *alloc_vm_area(size_t size, pte_t **ptes)
495 496 497 498 499 500 501 502 503 504 505 506
{
	BUG();
	return NULL;
}
EXPORT_SYMBOL_GPL(alloc_vm_area);

void free_vm_area(struct vm_struct *area)
{
	BUG();
}
EXPORT_SYMBOL_GPL(free_vm_area);

507 508 509 510 511 512 513
int vm_insert_page(struct vm_area_struct *vma, unsigned long addr,
		   struct page *page)
{
	return -EINVAL;
}
EXPORT_SYMBOL(vm_insert_page);

Linus Torvalds's avatar
Linus Torvalds committed
514 515 516 517 518 519 520
/*
 *  sys_brk() for the most part doesn't need the global kernel
 *  lock, except when an application is doing something nasty
 *  like trying to un-brk an area that has already been mapped
 *  to a regular file.  in this case, the unmapping will need
 *  to invoke file system routines that need the global lock.
 */
521
SYSCALL_DEFINE1(brk, unsigned long, brk)
Linus Torvalds's avatar
Linus Torvalds committed
522 523 524 525 526 527 528 529 530 531 532 533 534 535 536 537 538 539 540 541
{
	struct mm_struct *mm = current->mm;

	if (brk < mm->start_brk || brk > mm->context.end_brk)
		return mm->brk;

	if (mm->brk == brk)
		return mm->brk;

	/*
	 * Always allow shrinking brk
	 */
	if (brk <= mm->brk) {
		mm->brk = brk;
		return brk;
	}

	/*
	 * Ok, looks good - let it rip.
	 */
542
	flush_icache_range(mm->brk, brk);
Linus Torvalds's avatar
Linus Torvalds committed
543 544 545
	return mm->brk = brk;
}

546 547 548 549
/*
 * initialise the VMA and region record slabs
 */
void __init mmap_init(void)
Linus Torvalds's avatar
Linus Torvalds committed
550
{
551 552
	int ret;

553
	ret = percpu_counter_init(&vm_committed_as, 0, GFP_KERNEL);
554
	VM_BUG_ON(ret);
555
	vm_region_jar = KMEM_CACHE(vm_region, SLAB_PANIC);
Linus Torvalds's avatar
Linus Torvalds committed
556 557
}

558
/*
559 560
 * validate the region tree
 * - the caller must hold the region lock
561
 */
562 563
#ifdef CONFIG_DEBUG_NOMMU_REGIONS
static noinline void validate_nommu_regions(void)
564
{
565 566
	struct vm_region *region, *last;
	struct rb_node *p, *lastp;
567

568 569 570 571 572
	lastp = rb_first(&nommu_region_tree);
	if (!lastp)
		return;

	last = rb_entry(lastp, struct vm_region, vm_rb);
573 574
	BUG_ON(last->vm_end <= last->vm_start);
	BUG_ON(last->vm_top < last->vm_end);
575 576 577 578 579

	while ((p = rb_next(lastp))) {
		region = rb_entry(p, struct vm_region, vm_rb);
		last = rb_entry(lastp, struct vm_region, vm_rb);

580 581 582
		BUG_ON(region->vm_end <= region->vm_start);
		BUG_ON(region->vm_top < region->vm_end);
		BUG_ON(region->vm_start < last->vm_top);
583

584 585
		lastp = p;
	}
586
}
587
#else
588 589 590
static void validate_nommu_regions(void)
{
}
591
#endif
592 593

/*
594
 * add a region into the global tree
595
 */
596
static void add_nommu_region(struct vm_region *region)
597
{
598 599
	struct vm_region *pregion;
	struct rb_node **p, *parent;
600

601 602 603 604 605 606 607 608 609 610 611 612 613 614 615
	validate_nommu_regions();

	parent = NULL;
	p = &nommu_region_tree.rb_node;
	while (*p) {
		parent = *p;
		pregion = rb_entry(parent, struct vm_region, vm_rb);
		if (region->vm_start < pregion->vm_start)
			p = &(*p)->rb_left;
		else if (region->vm_start > pregion->vm_start)
			p = &(*p)->rb_right;
		else if (pregion == region)
			return;
		else
			BUG();
616 617
	}

618 619
	rb_link_node(&region->vm_rb, parent, p);
	rb_insert_color(&region->vm_rb, &nommu_region_tree);
620

621
	validate_nommu_regions();
622 623
}

624
/*
625
 * delete a region from the global tree
626
 */
627
static void delete_nommu_region(struct vm_region *region)
628
{
629
	BUG_ON(!nommu_region_tree.rb_node);
630

631 632 633
	validate_nommu_regions();
	rb_erase(&region->vm_rb, &nommu_region_tree);
	validate_nommu_regions();
634 635
}

636
/*
637
 * free a contiguous series of pages
638
 */
639
static void free_page_series(unsigned long from, unsigned long to)
640
{
641 642 643
	for (; from < to; from += PAGE_SIZE) {
		struct page *page = virt_to_page(from);

644
		atomic_long_dec(&mmap_pages_allocated);
645
		put_page(page);
646 647 648
	}
}

649
/*
650
 * release a reference to a region
651
 * - the caller must hold the region semaphore for writing, which this releases
652
 * - the region may not have been added to the tree yet, in which case vm_top
653
 *   will equal vm_start
654
 */
655 656
static void __put_nommu_region(struct vm_region *region)
	__releases(nommu_region_sem)
Linus Torvalds's avatar
Linus Torvalds committed
657
{
658
	BUG_ON(!nommu_region_tree.rb_node);
Linus Torvalds's avatar
Linus Torvalds committed
659

660
	if (--region->vm_usage == 0) {
661
		if (region->vm_top > region->vm_start)
662 663 664 665 666 667 668 669
			delete_nommu_region(region);
		up_write(&nommu_region_sem);

		if (region->vm_file)
			fput(region->vm_file);

		/* IO memory and memory shared directly out of the pagecache
		 * from ramfs/tmpfs mustn't be released here */
670
		if (region->vm_flags & VM_MAPPED_COPY)
671
			free_page_series(region->vm_start, region->vm_top);
672 673 674
		kmem_cache_free(vm_region_jar, region);
	} else {
		up_write(&nommu_region_sem);
Linus Torvalds's avatar
Linus Torvalds committed
675
	}
676
}
Linus Torvalds's avatar
Linus Torvalds committed
677

678 679 680 681 682 683 684
/*
 * release a reference to a region
 */
static void put_nommu_region(struct vm_region *region)
{
	down_write(&nommu_region_sem);
	__put_nommu_region(region);
Linus Torvalds's avatar
Linus Torvalds committed
685 686
}

687 688 689 690 691 692 693 694 695 696 697 698 699 700 701 702
/*
 * update protection on a vma
 */
static void protect_vma(struct vm_area_struct *vma, unsigned long flags)
{
#ifdef CONFIG_MPU
	struct mm_struct *mm = vma->vm_mm;
	long start = vma->vm_start & PAGE_MASK;
	while (start < vma->vm_end) {
		protect_page(mm, start, flags);
		start += PAGE_SIZE;
	}
	update_protections(mm);
#endif
}

703
/*
704 705 706 707
 * add a VMA into a process's mm_struct in the appropriate place in the list
 * and tree and add to the address space's page tree also if not an anonymous
 * page
 * - should be called with mm->mmap_sem held writelocked
708
 */
709
static void add_vma_to_mm(struct mm_struct *mm, struct vm_area_struct *vma)
Linus Torvalds's avatar
Linus Torvalds committed
710
{
711
	struct vm_area_struct *pvma, *prev;
Linus Torvalds's avatar
Linus Torvalds committed
712
	struct address_space *mapping;
713
	struct rb_node **p, *parent, *rb_prev;
714 715 716 717 718

	BUG_ON(!vma->vm_region);

	mm->map_count++;
	vma->vm_mm = mm;
Linus Torvalds's avatar
Linus Torvalds committed
719

720 721
	protect_vma(vma, vma->vm_flags);

Linus Torvalds's avatar
Linus Torvalds committed
722 723 724 725
	/* add the VMA to the mapping */
	if (vma->vm_file) {
		mapping = vma->vm_file->f_mapping;

726
		i_mmap_lock_write(mapping);
Linus Torvalds's avatar
Linus Torvalds committed
727
		flush_dcache_mmap_lock(mapping);
728
		vma_interval_tree_insert(vma, &mapping->i_mmap);
Linus Torvalds's avatar
Linus Torvalds committed
729
		flush_dcache_mmap_unlock(mapping);
730
		i_mmap_unlock_write(mapping);
Linus Torvalds's avatar
Linus Torvalds committed
731 732
	}

733
	/* add the VMA to the tree */
734
	parent = rb_prev = NULL;
735
	p = &mm->mm_rb.rb_node;
Linus Torvalds's avatar
Linus Torvalds committed
736 737 738 739
	while (*p) {
		parent = *p;
		pvma = rb_entry(parent, struct vm_area_struct, vm_rb);

740 741 742
		/* sort by: start addr, end addr, VMA struct addr in that order
		 * (the latter is necessary as we may get identical VMAs) */
		if (vma->vm_start < pvma->vm_start)
Linus Torvalds's avatar
Linus Torvalds committed
743
			p = &(*p)->rb_left;
744 745
		else if (vma->vm_start > pvma->vm_start) {
			rb_prev = parent;
Linus Torvalds's avatar
Linus Torvalds committed
746
			p = &(*p)->rb_right;
747
		} else if (vma->vm_end < pvma->vm_end)
748
			p = &(*p)->rb_left;
749 750
		else if (vma->vm_end > pvma->vm_end) {
			rb_prev = parent;
751
			p = &(*p)->rb_right;
752
		} else if (vma < pvma)
753
			p = &(*p)->rb_left;
754 755
		else if (vma > pvma) {
			rb_prev = parent;
756
			p = &(*p)->rb_right;
757
		} else
758
			BUG();
Linus Torvalds's avatar
Linus Torvalds committed
759 760 761
	}

	rb_link_node(&vma->vm_rb, parent, p);
762 763 764
	rb_insert_color(&vma->vm_rb, &mm->mm_rb);

	/* add VMA to the VMA list also */
765 766 767
	prev = NULL;
	if (rb_prev)
		prev = rb_entry(rb_prev, struct vm_area_struct, vm_rb);
768

769
	__vma_link_list(mm, vma, prev, parent);
Linus Torvalds's avatar
Linus Torvalds committed
770 771
}

772
/*
773
 * delete a VMA from its owning mm_struct and address space
774
 */
775
static void delete_vma_from_mm(struct vm_area_struct *vma)
Linus Torvalds's avatar
Linus Torvalds committed
776
{
Davidlohr Bueso's avatar
Davidlohr Bueso committed
777
	int i;
Linus Torvalds's avatar
Linus Torvalds committed
778
	struct address_space *mapping;
779
	struct mm_struct *mm = vma->vm_mm;
Davidlohr Bueso's avatar
Davidlohr Bueso committed
780
	struct task_struct *curr = current;
781

782 783
	protect_vma(vma, 0);

784
	mm->map_count--;
Davidlohr Bueso's avatar
Davidlohr Bueso committed
785 786 787
	for (i = 0; i < VMACACHE_SIZE; i++) {
		/* if the vma is cached, invalidate the entire cache */
		if (curr->vmacache[i] == vma) {
788
			vmacache_invalidate(mm);
Davidlohr Bueso's avatar
Davidlohr Bueso committed
789 790 791
			break;
		}
	}
Linus Torvalds's avatar
Linus Torvalds committed
792 793 794 795 796

	/* remove the VMA from the mapping */
	if (vma->vm_file) {
		mapping = vma->vm_file->f_mapping;

797
		i_mmap_lock_write(mapping);
Linus Torvalds's avatar
Linus Torvalds committed
798
		flush_dcache_mmap_lock(mapping);
799
		vma_interval_tree_remove(vma, &mapping->i_mmap);
Linus Torvalds's avatar
Linus Torvalds committed
800
		flush_dcache_mmap_unlock(mapping);
801
		i_mmap_unlock_write(mapping);
Linus Torvalds's avatar
Linus Torvalds committed
802 803
	}

804 805
	/* remove from the MM's tree and list */
	rb_erase(&vma->vm_rb, &mm->mm_rb);
806 807 808 809 810 811 812 813

	if (vma->vm_prev)
		vma->vm_prev->vm_next = vma->vm_next;
	else
		mm->mmap = vma->vm_next;

	if (vma->vm_next)
		vma->vm_next->vm_prev = vma->vm_prev;
814 815 816 817 818 819 820 821 822
}

/*
 * destroy a VMA record
 */
static void delete_vma(struct mm_struct *mm, struct vm_area_struct *vma)
{
	if (vma->vm_ops && vma->vm_ops->close)
		vma->vm_ops->close(vma);
823
	if (vma->vm_file)
824 825 826 827 828 829 830 831 832 833 834 835 836 837
		fput(vma->vm_file);
	put_nommu_region(vma->vm_region);
	kmem_cache_free(vm_area_cachep, vma);
}

/*
 * look up the first VMA in which addr resides, NULL if none
 * - should be called with mm->mmap_sem at least held readlocked
 */
struct vm_area_struct *find_vma(struct mm_struct *mm, unsigned long addr)
{
	struct vm_area_struct *vma;

	/* check the cache first */
Davidlohr Bueso's avatar
Davidlohr Bueso committed
838 839
	vma = vmacache_find(mm, addr);
	if (likely(vma))
840 841
		return vma;

842
	/* trawl the list (there may be multiple mappings in which addr
843
	 * resides) */
844
	for (vma = mm->mmap; vma; vma = vma->vm_next) {
845 846 847
		if (vma->vm_start > addr)
			return NULL;
		if (vma->vm_end > addr) {
Davidlohr Bueso's avatar
Davidlohr Bueso committed
848
			vmacache_update(addr, vma);
849 850 851 852 853 854 855 856 857 858 859 860 861 862
			return vma;
		}
	}

	return NULL;
}
EXPORT_SYMBOL(find_vma);

/*
 * find a VMA
 * - we don't extend stack VMAs under NOMMU conditions
 */
struct vm_area_struct *find_extend_vma(struct mm_struct *mm, unsigned long addr)
{
863
	return find_vma(mm, addr);
864 865 866 867 868 869 870 871 872 873 874 875 876 877 878 879 880 881 882 883 884 885 886
}

/*
 * expand a stack to a given address
 * - not supported under NOMMU conditions
 */
int expand_stack(struct vm_area_struct *vma, unsigned long address)
{
	return -ENOMEM;
}

/*
 * look up the first VMA exactly that exactly matches addr
 * - should be called with mm->mmap_sem at least held readlocked
 */
static struct vm_area_struct *find_vma_exact(struct mm_struct *mm,
					     unsigned long addr,
					     unsigned long len)
{
	struct vm_area_struct *vma;
	unsigned long end = addr + len;

	/* check the cache first */
Davidlohr Bueso's avatar
Davidlohr Bueso committed
887 888
	vma = vmacache_find_exact(mm, addr, end);
	if (vma)
889 890
		return vma;

891
	/* trawl the list (there may be multiple mappings in which addr
892
	 * resides) */
893
	for (vma = mm->mmap; vma; vma = vma->vm_next) {
894 895 896 897 898
		if (vma->vm_start < addr)
			continue;
		if (vma->vm_start > addr)
			return NULL;
		if (vma->vm_end == end) {
Davidlohr Bueso's avatar
Davidlohr Bueso committed
899
			vmacache_update(addr, vma);
900 901 902 903 904
			return vma;
		}
	}

	return NULL;
Linus Torvalds's avatar
Linus Torvalds committed
905 906 907 908 909 910 911 912 913 914 915 916 917 918
}

/*
 * determine whether a mapping should be permitted and, if so, what sort of
 * mapping we're capable of supporting
 */
static int validate_mmap_request(struct file *file,
				 unsigned long addr,
				 unsigned long len,
				 unsigned long prot,
				 unsigned long flags,
				 unsigned long pgoff,
				 unsigned long *_capabilities)
{
919
	unsigned long capabilities, rlen;
Linus Torvalds's avatar
Linus Torvalds committed
920 921 922
	int ret;

	/* do the simple checks first */
923
	if (flags & MAP_FIXED)
Linus Torvalds's avatar
Linus Torvalds committed
924 925 926 927 928 929
		return -EINVAL;

	if ((flags & MAP_TYPE) != MAP_PRIVATE &&
	    (flags & MAP_TYPE) != MAP_SHARED)
		return -EINVAL;

930
	if (!len)
Linus Torvalds's avatar
Linus Torvalds committed
931 932
		return -EINVAL;

933
	/* Careful about overflows.. */
934 935
	rlen = PAGE_ALIGN(len);
	if (!rlen || rlen > TASK_SIZE)
936 937
		return -ENOMEM;

Linus Torvalds's avatar
Linus Torvalds committed
938
	/* offset overflow? */
939
	if ((pgoff + (rlen >> PAGE_SHIFT)) < pgoff)
940
		return -EOVERFLOW;
Linus Torvalds's avatar
Linus Torvalds committed
941 942 943

	if (file) {
		/* files must support mmap */
Al Viro's avatar
Al Viro committed
944
		if (!file->f_op->mmap)
Linus Torvalds's avatar
Linus Torvalds committed
945 946 947 948 949 950
			return -ENODEV;

		/* work out if what we've got could possibly be shared
		 * - we support chardevs that provide their own "memory"
		 * - we support files/blockdevs that are memory backed
		 */
951 952 953
		if (file->f_op->mmap_capabilities) {
			capabilities = file->f_op->mmap_capabilities(file);
		} else {
Linus Torvalds's avatar
Linus Torvalds committed
954 955
			/* no explicit capabilities set, so assume some
			 * defaults */
Al Viro's avatar
Al Viro committed
956
			switch (file_inode(file)->i_mode & S_IFMT) {
Linus Torvalds's avatar
Linus Torvalds committed
957 958
			case S_IFREG:
			case S_IFBLK:
959
				capabilities = NOMMU_MAP_COPY;
Linus Torvalds's avatar
Linus Torvalds committed
960 961 962 963
				break;

			case S_IFCHR:
				capabilities =
964 965 966
					NOMMU_MAP_DIRECT |
					NOMMU_MAP_READ |
					NOMMU_MAP_WRITE;
Linus Torvalds's avatar
Linus Torvalds committed
967 968 969 970 971 972 973 974 975 976
				break;

			default:
				return -EINVAL;
			}
		}

		/* eliminate any capabilities that we can't support on this
		 * device */
		if (!file->f_op->get_unmapped_area)
977
			capabilities &= ~NOMMU_MAP_DIRECT;
Al Viro's avatar
Al Viro committed
978
		if (!(file->f_mode & FMODE_CAN_READ))
979
			capabilities &= ~NOMMU_MAP_COPY;
Linus Torvalds's avatar
Linus Torvalds committed
980

981 982 983 984
		/* The file shall have been opened with read permission. */
		if (!(file->f_mode & FMODE_READ))
			return -EACCES;

Linus Torvalds's avatar
Linus Torvalds committed
985 986 987 988 989 990
		if (flags & MAP_SHARED) {
			/* do checks for writing, appending and locking */
			if ((prot & PROT_WRITE) &&
			    !(file->f_mode & FMODE_WRITE))
				return -EACCES;

Al Viro's avatar
Al Viro committed
991
			if (IS_APPEND(file_inode(file)) &&
Linus Torvalds's avatar
Linus Torvalds committed
992 993 994
			    (file->f_mode & FMODE_WRITE))
				return -EACCES;

995
			if (locks_verify_locked(file))
Linus Torvalds's avatar
Linus Torvalds committed
996 997
				return -EAGAIN;

998
			if (!(capabilities & NOMMU_MAP_DIRECT))
Linus Torvalds's avatar
Linus Torvalds committed
999 1000 1001
				return -ENODEV;

			/* we mustn't privatise shared mappings */
1002
			capabilities &= ~NOMMU_MAP_COPY;
1003
		} else {
Linus Torvalds's avatar
Linus Torvalds committed
1004 1005
			/* we're going to read the file into private memory we
			 * allocate */
1006
			if (!(capabilities & NOMMU_MAP_COPY))
Linus Torvalds's avatar
Linus Torvalds committed
1007 1008 1009 1010 1011
				return -ENODEV;

			/* we don't permit a private writable mapping to be
			 * shared with the backing device */
			if (prot & PROT_WRITE)
1012
				capabilities &= ~NOMMU_MAP_DIRECT;
Linus Torvalds's avatar
Linus Torvalds committed
1013 1014
		}

1015 1016 1017 1018
		if (capabilities & NOMMU_MAP_DIRECT) {
			if (((prot & PROT_READ)  && !(capabilities & NOMMU_MAP_READ))  ||
			    ((prot & PROT_WRITE) && !(capabilities & NOMMU_MAP_WRITE)) ||
			    ((prot & PROT_EXEC)  && !(capabilities & NOMMU_MAP_EXEC))
1019
			    ) {
1020
				capabilities &= ~NOMMU_MAP_DIRECT;
1021
				if (flags & MAP_SHARED) {
1022
					pr_warn("MAP_SHARED not completely supported on !MMU\n");
1023 1024 1025 1026 1027
					return -EINVAL;
				}
			}
		}

Linus Torvalds's avatar
Linus Torvalds committed
1028 1029
		/* handle executable mappings and implied executable
		 * mappings */
1030
		if (path_noexec(&file->f_path)) {
Linus Torvalds's avatar
Linus Torvalds committed
1031 1032
			if (prot & PROT_EXEC)
				return -EPERM;
1033
		} else if ((prot & PROT_READ) && !(prot & PROT_EXEC)) {
Linus Torvalds's avatar
Linus Torvalds committed
1034 1035
			/* handle implication of PROT_EXEC by PROT_READ */
			if (current->personality & READ_IMPLIES_EXEC) {
1036
				if (capabilities & NOMMU_MAP_EXEC)
Linus Torvalds's avatar
Linus Torvalds committed
1037 1038
					prot |= PROT_EXEC;
			}
1039
		} else if ((prot & PROT_READ) &&
Linus Torvalds's avatar
Linus Torvalds committed
1040
			 (prot & PROT_EXEC) &&
1041
			 !(capabilities & NOMMU_MAP_EXEC)
Linus Torvalds's avatar
Linus Torvalds committed
1042 1043
			 ) {
			/* backing file is not executable, try to copy */
1044
			capabilities &= ~NOMMU_MAP_DIRECT;
Linus Torvalds's avatar
Linus Torvalds committed
1045
		}
1046
	} else {
Linus Torvalds's avatar
Linus Torvalds committed
1047 1048 1049
		/* anonymous mappings are always memory backed and can be
		 * privately mapped
		 */
1050
		capabilities = NOMMU_MAP_COPY;
Linus Torvalds's avatar
Linus Torvalds committed
1051 1052 1053 1054 1055 1056 1057 1058

		/* handle PROT_EXEC implication by PROT_READ */
		if ((prot & PROT_READ) &&
		    (current->personality & READ_IMPLIES_EXEC))
			prot |= PROT_EXEC;
	}

	/* allow the security API to have its say */
1059
	ret = security_mmap_addr(addr);
Linus Torvalds's avatar
Linus Torvalds committed
1060 1061 1062 1063 1064 1065 1066 1067 1068 1069 1070 1071 1072 1073 1074 1075 1076 1077 1078 1079 1080 1081
	if (ret < 0)
		return ret;

	/* looks okay */
	*_capabilities = capabilities;
	return 0;
}

/*
 * we've determined that we can make the mapping, now translate what we
 * now know into VMA flags
 */
static unsigned long determine_vm_flags(struct file *file,
					unsigned long prot,
					unsigned long flags,
					unsigned long capabilities)
{
	unsigned long vm_flags;

	vm_flags = calc_vm_prot_bits(prot) | calc_vm_flag_bits(flags);
	/* vm_flags |= mm->def_flags; */

1082
	if (!(capabilities & NOMMU_MAP_DIRECT)) {
Linus Torvalds's avatar
Linus Torvalds committed
1083
		/* attempt to share read-only copies of mapped file chunks */
1084
		vm_flags |= VM_MAYREAD | VM_MAYWRITE | VM_MAYEXEC;
Linus Torvalds's avatar
Linus Torvalds committed
1085 1086
		if (file && !(prot & PROT_WRITE))
			vm_flags |= VM_MAYSHARE;
1087
	} else {
Linus Torvalds's avatar
Linus Torvalds committed
1088 1089 1090
		/* overlay a shareable mapping on the backing device or inode
		 * if possible - used for chardevs, ramfs/tmpfs/shmfs and
		 * romfs/cramfs */
1091
		vm_flags |= VM_MAYSHARE | (capabilities & NOMMU_VMFLAGS);
Linus Torvalds's avatar
Linus Torvalds committed
1092
		if (flags & MAP_SHARED)
1093
			vm_flags |= VM_SHARED;
Linus Torvalds's avatar
Linus Torvalds committed
1094 1095 1096 1097 1098 1099
	}

	/* refuse to let anyone share private mappings with this process if
	 * it's being traced - otherwise breakpoints set in it may interfere
	 * with another untraced process
	 */
Tejun Heo's avatar
Tejun Heo committed
1100
	if ((flags & MAP_PRIVATE) && current->ptrace)
Linus Torvalds's avatar
Linus Torvalds committed
1101 1102 1103 1104 1105 1106
		vm_flags &= ~VM_MAYSHARE;

	return vm_flags;
}

/*
1107 1108
 * set up a shared mapping on a file (the driver or filesystem provides and
 * pins the storage)
Linus Torvalds's avatar
Linus Torvalds committed
1109
 */
1110
static int do_mmap_shared_file(struct vm_area_struct *vma)
Linus Torvalds's avatar
Linus Torvalds committed
1111 1112 1113 1114
{
	int ret;

	ret = vma->vm_file->f_op->mmap(vma->vm_file, vma);
1115 1116
	if (ret == 0) {
		vma->vm_region->vm_top = vma->vm_region->vm_end;
1117
		return 0;
1118
	}
Linus Torvalds's avatar
Linus Torvalds committed
1119 1120 1121
	if (ret != -ENOSYS)
		return ret;

1122 1123 1124
	/* getting -ENOSYS indicates that direct mmap isn't possible (as
	 * opposed to tried but failed) so we can only give a suitable error as
	 * it's not possible to make a private copy if MAP_SHARED was given */
Linus Torvalds's avatar
Linus Torvalds committed
1125 1126 1127 1128 1129 1130
	return -ENODEV;
}

/*
 * set up a private mapping or an anonymous shared mapping
 */
1131 1132
static int do_mmap_private(struct vm_area_struct *vma,
			   struct vm_region *region,
1133 1134
			   unsigned long len,
			   unsigned long capabilities)
Linus Torvalds's avatar
Linus Torvalds committed
1135
{
1136
	unsigned long total, point;
Linus Torvalds's avatar
Linus Torvalds committed
1137
	void *base;
1138
	int ret, order;
Linus Torvalds's avatar
Linus Torvalds committed
1139 1140 1141 1142 1143

	/* invoke the file's mapping function so that it can keep track of
	 * shared mappings on devices or memory
	 * - VM_MAYSHARE will be set if it may attempt to share
	 */
1144
	if (capabilities & NOMMU_MAP_DIRECT) {
Linus Torvalds's avatar
Linus Torvalds committed
1145
		ret = vma->vm_file->f_op->mmap(vma->vm_file, vma);
1146
		if (ret == 0) {
Linus Torvalds's avatar
Linus Torvalds committed
1147
			/* shouldn't return success if we're not sharing */
1148 1149
			BUG_ON(!(vma->vm_flags & VM_MAYSHARE));
			vma->vm_region->vm_top = vma->vm_region->vm_end;
1150
			return 0;
Linus Torvalds's avatar
Linus Torvalds committed
1151
		}
1152 1153
		if (ret != -ENOSYS)
			return ret;
Linus Torvalds's avatar
Linus Torvalds committed
1154 1155 1156 1157 1158 1159

		/* getting an ENOSYS error indicates that direct mmap isn't
		 * possible (as opposed to tried but failed) so we'll try to
		 * make a private copy of the data and map that instead */
	}

1160

Linus Torvalds's avatar
Linus Torvalds committed
1161 1162 1163 1164
	/* allocate some memory to hold the mapping
	 * - note that this may not return a page-aligned address if the object
	 *   we're allocating is smaller than a page
	 */
Bob Liu's avatar
Bob Liu committed
1165
	order = get_order(len);
1166
	total = 1 << order;
Bob Liu's avatar
Bob Liu committed
1167
	point = len >> PAGE_SHIFT;
1168

1169
	/* we don't want to allocate a power-of-2 sized page set */
1170
	if (sysctl_nr_trim_pages && total - point >= sysctl_nr_trim_pages)
1171
		total = point;
1172

Joonsoo Kim's avatar
Joonsoo Kim committed
1173
	base = alloc_pages_exact(total << PAGE_SHIFT, GFP_KERNEL);
1174 1175 1176 1177
	if (!base)
		goto enomem;

	atomic_long_add(total, &mmap_pages_allocated);
Linus Torvalds's avatar
Linus Torvalds committed
1178

1179 1180
	region->vm_flags = vma->vm_flags |= VM_MAPPED_COPY;
	region->vm_start = (unsigned long) base;
Bob Liu's avatar
Bob Liu committed
1181
	region->vm_end   = region->vm_start + len;
1182
	region->vm_top   = region->vm_start + (total << PAGE_SHIFT);
1183 1184 1185

	vma->vm_start = region->vm_start;
	vma->vm_end   = region->vm_start + len;
Linus Torvalds's avatar
Linus Torvalds committed
1186 1187 1188 1189 1190 1191 1192 1193 1194 1195 1196

	if (vma->vm_file) {
		/* read the contents of a file into the copy */
		mm_segment_t old_fs;
		loff_t fpos;

		fpos = vma->vm_pgoff;
		fpos <<= PAGE_SHIFT;

		old_fs = get_fs();
		set_fs(KERNEL_DS);
Al Viro's avatar
Al Viro committed
1197
		ret = __vfs_read(vma->vm_file, base, len, &fpos);
Linus Torvalds's avatar
Linus Torvalds committed
1198 1199 1200 1201 1202 1203
		set_fs(old_fs);

		if (ret < 0)
			goto error_free;

		/* clear the last little bit */
Bob Liu's avatar
Bob Liu committed
1204 1205
		if (ret < len)
			memset(base + ret, 0, len - ret);
Linus Torvalds's avatar
Linus Torvalds committed
1206 1207 1208 1209 1210 1211

	}

	return 0;

error_free:
1212
	free_page_series(region->vm_start, region->vm_top);
1213 1214
	region->vm_start = vma->vm_start = 0;
	region->vm_end   = vma->vm_end = 0;
1215
	region->vm_top   = 0;
Linus Torvalds's avatar
Linus Torvalds committed
1216 1217 1218
	return ret;

enomem:
1219
	pr_err("Allocation of length %lu from process %d (%s) failed\n",
1220
	       len, current->pid, current->comm);
1221
	show_free_areas(0);
Linus Torvalds's avatar
Linus Torvalds committed
1222 1223 1224 1225 1226 1227
	return -ENOMEM;
}

/*
 * handle mapping creation for uClinux
 */
1228 1229 1230 1231 1232 1233 1234 1235
unsigned long do_mmap(struct file *file,
			unsigned long addr,
			unsigned long len,
			unsigned long prot,
			unsigned long flags,
			vm_flags_t vm_flags,
			unsigned long pgoff,
			unsigned long *populate)
Linus Torvalds's avatar
Linus Torvalds committed
1236
{
1237 1238
	struct vm_area_struct *vma;
	struct vm_region *region;
Linus Torvalds's avatar
Linus Torvalds committed
1239
	struct rb_node *rb;
1240
	unsigned long capabilities, result;
Linus Torvalds's avatar
Linus Torvalds committed
1241 1242
	int ret;

1243
	*populate = 0;
1244

Linus Torvalds's avatar
Linus Torvalds committed
1245 1246 1247 1248
	/* decide whether we should attempt the mapping, and if so what sort of
	 * mapping */
	ret = validate_mmap_request(file, addr, len, prot, flags, pgoff,
				    &capabilities);
1249
	if (ret < 0)
Linus Torvalds's avatar
Linus Torvalds committed
1250 1251
		return ret;

1252 1253
	/* we ignore the address hint */
	addr = 0;
Bob Liu's avatar
Bob Liu committed
1254
	len = PAGE_ALIGN(len);
1255

Linus Torvalds's avatar
Linus Torvalds committed
1256 1257
	/* we've determined that we can make the mapping, now translate what we
	 * now know into VMA flags */
1258
	vm_flags |= determine_vm_flags(file, prot, flags, capabilities);
Linus Torvalds's avatar
Linus Torvalds committed
1259

1260 1261 1262 1263 1264 1265 1266 1267
	/* we're going to need to record the mapping */
	region = kmem_cache_zalloc(vm_region_jar, GFP_KERNEL);
	if (!region)
		goto error_getting_region;

	vma = kmem_cache_zalloc(vm_area_cachep, GFP_KERNEL);
	if (!vma)
		goto error_getting_vma;
Linus Torvalds's avatar
Linus Torvalds committed
1268

1269
	region->vm_usage = 1;
1270 1271 1272
	region->vm_flags = vm_flags;
	region->vm_pgoff = pgoff;

1273
	INIT_LIST_HEAD(&vma->anon_vma_chain);
1274 1275
	vma->vm_flags = vm_flags;
	vma->vm_pgoff = pgoff;
Linus Torvalds's avatar
Linus Torvalds committed
1276

1277
	if (file) {
1278 1279
		region->vm_file = get_file(file);
		vma->vm_file = get_file(file);
1280 1281 1282 1283 1284
	}

	down_write(&nommu_region_sem);

	/* if we want to share, we need to check for regions created by other
Linus Torvalds's avatar
Linus Torvalds committed
1285
	 * mmap() calls that overlap with our proposed mapping
1286
	 * - we can only share with a superset match on most regular files
Linus Torvalds's avatar
Linus Torvalds committed
1287 1288 1289 1290 1291 1292
	 * - shared mappings on character devices and memory backed files are
	 *   permitted to overlap inexactly as far as we are concerned for in
	 *   these cases, sharing is handled in the driver or filesystem rather
	 *   than here
	 */
	if (vm_flags & VM_MAYSHARE) {
1293 1294
		struct vm_region *pregion;
		unsigned long pglen, rpglen, pgend, rpgend, start;
Linus Torvalds's avatar
Linus Torvalds committed
1295

1296 1297
		pglen = (len + PAGE_SIZE - 1) >> PAGE_SHIFT;
		pgend = pgoff + pglen;
1298

1299 1300
		for (rb = rb_first(&nommu_region_tree); rb; rb = rb_next(rb)) {
			pregion = rb_entry(rb, struct vm_region, vm_rb);
Linus Torvalds's avatar
Linus Torvalds committed
1301

1302
			if (!(pregion->vm_flags & VM_MAYSHARE))
Linus Torvalds's avatar
Linus Torvalds committed
1303 1304 1305
				continue;

			/* search for overlapping mappings on the same file */
Al Viro's avatar
Al Viro committed
1306 1307
			if (file_inode(pregion->vm_file) !=
			    file_inode(file))
Linus Torvalds's avatar
Linus Torvalds committed
1308 1309
				continue;

1310
			if (pregion->vm_pgoff >= pgend)
Linus Torvalds's avatar
Linus Torvalds committed
1311 1312
				continue;

1313 1314 1315 1316
			rpglen = pregion->vm_end - pregion->vm_start;
			rpglen = (rpglen + PAGE_SIZE - 1) >> PAGE_SHIFT;
			rpgend = pregion->vm_pgoff + rpglen;
			if (pgoff >= rpgend)
Linus Torvalds's avatar
Linus Torvalds committed
1317 1318
				continue;

1319 1320 1321 1322 1323
			/* handle inexactly overlapping matches between
			 * mappings */
			if ((pregion->vm_pgoff != pgoff || rpglen != pglen) &&
			    !(pgoff >= pregion->vm_pgoff && pgend <= rpgend)) {
				/* new mapping is not a subset of the region */
1324
				if (!(capabilities & NOMMU_MAP_DIRECT))
Linus Torvalds's avatar
Linus Torvalds committed
1325 1326 1327 1328
					goto sharing_violation;
				continue;
			}

1329
			/* we've found a region we can share */
1330
			pregion->vm_usage++;
1331 1332 1333 1334 1335 1336
			vma->vm_region = pregion;
			start = pregion->vm_start;
			start += (pgoff - pregion->vm_pgoff) << PAGE_SHIFT;
			vma->vm_start = start;
			vma->vm_end = start + len;

1337
			if (pregion->vm_flags & VM_MAPPED_COPY)
1338
				vma->vm_flags |= VM_MAPPED_COPY;
1339
			else {
1340 1341 1342 1343 1344
				ret = do_mmap_shared_file(vma);
				if (ret < 0) {
					vma->vm_region = NULL;
					vma->vm_start = 0;
					vma->vm_end = 0;
1345
					pregion->vm_usage--;
1346 1347 1348 1349 1350 1351 1352 1353 1354
					pregion = NULL;
					goto error_just_free;
				}
			}
			fput(region->vm_file);
			kmem_cache_free(vm_region_jar, region);
			region = pregion;
			result = start;
			goto share;
Linus Torvalds's avatar
Linus Torvalds committed
1355 1356 1357 1358 1359 1360
		}

		/* obtain the address at which to make a shared mapping
		 * - this is the hook for quasi-memory character devices to
		 *   tell us the location of a shared mapping
		 */
1361
		if (capabilities & NOMMU_MAP_DIRECT) {
Linus Torvalds's avatar
Linus Torvalds committed
1362 1363
			addr = file->f_op->get_unmapped_area(file, addr, len,
							     pgoff, flags);
1364
			if (IS_ERR_VALUE(addr)) {
Linus Torvalds's avatar
Linus Torvalds committed
1365
				ret = addr;
1366
				if (ret != -ENOSYS)
1367
					goto error_just_free;
Linus Torvalds's avatar
Linus Torvalds committed
1368 1369 1370 1371

				/* the driver refused to tell us where to site
				 * the mapping so we'll have to attempt to copy
				 * it */
1372
				ret = -ENODEV;
1373
				if (!(capabilities & NOMMU_MAP_COPY))
1374
					goto error_just_free;
Linus Torvalds's avatar
Linus Torvalds committed
1375

1376
				capabilities &= ~NOMMU_MAP_DIRECT;
1377 1378 1379
			} else {
				vma->vm_start = region->vm_start = addr;
				vma->vm_end = region->vm_end = addr + len;
Linus Torvalds's avatar
Linus Torvalds committed
1380 1381 1382 1383
			}
		}
	}

1384
	vma->vm_region = region;
Linus Torvalds's avatar
Linus Torvalds committed
1385

1386
	/* set up the mapping
1387
	 * - the region is filled in if NOMMU_MAP_DIRECT is still set
1388
	 */
Linus Torvalds's avatar
Linus Torvalds committed
1389
	if (file && vma->vm_flags & VM_SHARED)
1390
		ret = do_mmap_shared_file(vma);
Linus Torvalds's avatar
Linus Torvalds committed
1391
	else
1392
		ret = do_mmap_private(vma, region, len, capabilities);
Linus Torvalds's avatar
Linus Torvalds committed
1393
	if (ret < 0)
1394 1395
		goto error_just_free;
	add_nommu_region(region);
1396

1397 1398 1399 1400 1401
	/* clear anonymous mappings that don't ask for uninitialized data */
	if (!vma->vm_file && !(flags & MAP_UNINITIALIZED))
		memset((void *)region->vm_start, 0,
		       region->vm_end - region->vm_start);

Linus Torvalds's avatar
Linus Torvalds committed
1402
	/* okay... we have a mapping; now we have to register it */
1403
	result = vma->vm_start;
Linus Torvalds's avatar
Linus Torvalds committed
1404 1405 1406

	current->mm->total_vm += len >> PAGE_SHIFT;

1407 1408
share:
	add_vma_to_mm(current->mm, vma);
Linus Torvalds's avatar
Linus Torvalds committed
1409

1410 1411 1412 1413 1414 1415
	/* we flush the region from the icache only when the first executable
	 * mapping of it is made  */
	if (vma->vm_flags & VM_EXEC && !region->vm_icache_flushed) {
		flush_icache_range(region->vm_start, region->vm_end);
		region->vm_icache_flushed = true;
	}
Linus Torvalds's avatar
Linus Torvalds committed
1416

1417
	up_write(&nommu_region_sem);
Linus Torvalds's avatar
Linus Torvalds committed
1418

1419
	return result;
Linus Torvalds's avatar
Linus Torvalds committed
1420

1421 1422 1423
error_just_free:
	up_write(&nommu_region_sem);
error:
1424 1425
	if (region->vm_file)
		fput(region->vm_file);
1426
	kmem_cache_free(vm_region_jar, region);
1427 1428
	if (vma->vm_file)
		fput(vma->vm_file);
1429 1430 1431 1432 1433
	kmem_cache_free(vm_area_cachep, vma);
	return ret;

sharing_violation:
	up_write(&nommu_region_sem);
1434
	pr_warn("Attempt to share mismatched mappings\n");
1435 1436
	ret = -EINVAL;
	goto error;
Linus Torvalds's avatar
Linus Torvalds committed
1437

1438 1439
error_getting_vma:
	kmem_cache_free(vm_region_jar, region);
1440 1441
	pr_warn("Allocation of vma for %lu byte allocation from process %d failed\n",
			len, current->pid);
1442
	show_free_areas(0);
Linus Torvalds's avatar
Linus Torvalds committed
1443 1444
	return -ENOMEM;

1445
error_getting_region:
1446 1447
	pr_warn("Allocation of vm region for %lu byte allocation from process %d failed\n",
			len, current->pid);
1448
	show_free_areas(0);
Linus Torvalds's avatar
Linus Torvalds committed
1449 1450
	return -ENOMEM;
}
1451

1452 1453 1454 1455 1456 1457 1458
SYSCALL_DEFINE6(mmap_pgoff, unsigned long, addr, unsigned long, len,
		unsigned long, prot, unsigned long, flags,
		unsigned long, fd, unsigned long, pgoff)
{
	struct file *file = NULL;
	unsigned long retval = -EBADF;

Al Viro's avatar
Al Viro committed
1459
	audit_mmap_fd(fd, flags);
1460 1461 1462 1463 1464 1465 1466 1467
	if (!(flags & MAP_ANONYMOUS)) {
		file = fget(fd);
		if (!file)
			goto out;
	}

	flags &= ~(MAP_EXECUTABLE | MAP_DENYWRITE);

1468
	retval = vm_mmap_pgoff(file, addr, len, prot, flags, pgoff);
1469 1470 1471 1472 1473 1474 1475

	if (file)
		fput(file);
out:
	return retval;
}

1476 1477 1478 1479 1480 1481 1482 1483 1484 1485 1486 1487 1488 1489 1490 1491
#ifdef __ARCH_WANT_SYS_OLD_MMAP
struct mmap_arg_struct {
	unsigned long addr;
	unsigned long len;
	unsigned long prot;
	unsigned long flags;
	unsigned long fd;
	unsigned long offset;
};

SYSCALL_DEFINE1(old_mmap, struct mmap_arg_struct __user *, arg)
{
	struct mmap_arg_struct a;

	if (copy_from_user(&a, arg, sizeof(a)))
		return -EFAULT;
1492
	if (offset_in_page(a.offset))
1493 1494 1495 1496 1497 1498 1499
		return -EINVAL;

	return sys_mmap_pgoff(a.addr, a.len, a.prot, a.flags, a.fd,
			      a.offset >> PAGE_SHIFT);
}
#endif /* __ARCH_WANT_SYS_OLD_MMAP */

Linus Torvalds's avatar
Linus Torvalds committed
1500
/*
1501 1502
 * split a vma into two pieces at address 'addr', a new vma is allocated either
 * for the first part or the tail.
Linus Torvalds's avatar
Linus Torvalds committed
1503
 */
1504 1505
int split_vma(struct mm_struct *mm, struct vm_area_struct *vma,
	      unsigned long addr, int new_below)
Linus Torvalds's avatar
Linus Torvalds committed
1506
{
1507 1508 1509
	struct vm_area_struct *new;
	struct vm_region *region;
	unsigned long npages;
Linus Torvalds's avatar
Linus Torvalds committed
1510

1511 1512 1513
	/* we're only permitted to split anonymous regions (these should have
	 * only a single usage on the region) */
	if (vma->vm_file)
1514
		return -ENOMEM;
Linus Torvalds's avatar
Linus Torvalds committed
1515

1516 1517
	if (mm->map_count >= sysctl_max_map_count)
		return -ENOMEM;
Linus Torvalds's avatar
Linus Torvalds committed
1518

1519 1520 1521
	region = kmem_cache_alloc(vm_region_jar, GFP_KERNEL);
	if (!region)
		return -ENOMEM;
Linus Torvalds's avatar
Linus Torvalds committed
1522

1523 1524 1525 1526 1527 1528 1529 1530 1531 1532 1533 1534 1535 1536
	new = kmem_cache_alloc(vm_area_cachep, GFP_KERNEL);
	if (!new) {
		kmem_cache_free(vm_region_jar, region);
		return -ENOMEM;
	}

	/* most fields are the same, copy all, and then fixup */
	*new = *vma;
	*region = *vma->vm_region;
	new->vm_region = region;

	npages = (addr - vma->vm_start) >> PAGE_SHIFT;

	if (new_below) {
1537
		region->vm_top = region->vm_end = new->vm_end = addr;
1538 1539 1540
	} else {
		region->vm_start = new->vm_start = addr;
		region->vm_pgoff = new->vm_pgoff += npages;
Linus Torvalds's avatar
Linus Torvalds committed
1541
	}
1542 1543 1544 1545 1546 1547 1548 1549 1550 1551 1552 1553

	if (new->vm_ops && new->vm_ops->open)
		new->vm_ops->open(new);

	delete_vma_from_mm(vma);
	down_write(&nommu_region_sem);
	delete_nommu_region(vma->vm_region);
	if (new_below) {
		vma->vm_region->vm_start = vma->vm_start = addr;
		vma->vm_region->vm_pgoff = vma->vm_pgoff += npages;
	} else {
		vma->vm_region->vm_end = vma->vm_end = addr;
1554
		vma->vm_region->vm_top = addr;
1555 1556 1557 1558 1559 1560 1561
	}
	add_nommu_region(vma->vm_region);
	add_nommu_region(new->vm_region);
	up_write(&nommu_region_sem);
	add_vma_to_mm(mm, vma);
	add_vma_to_mm(mm, new);
	return 0;
Linus Torvalds's avatar
Linus Torvalds committed
1562 1563
}

1564
/*
1565 1566
 * shrink a VMA by removing the specified chunk from either the beginning or
 * the end
1567
 */
1568 1569 1570
static int shrink_vma(struct mm_struct *mm,
		      struct vm_area_struct *vma,
		      unsigned long from, unsigned long to)
Linus Torvalds's avatar
Linus Torvalds committed
1571
{
1572
	struct vm_region *region;
Linus Torvalds's avatar
Linus Torvalds committed
1573

1574 1575 1576 1577 1578 1579 1580 1581
	/* adjust the VMA's pointers, which may reposition it in the MM's tree
	 * and list */
	delete_vma_from_mm(vma);
	if (from > vma->vm_start)
		vma->vm_end = from;
	else
		vma->vm_start = to;
	add_vma_to_mm(mm, vma);
Linus Torvalds's avatar
Linus Torvalds committed
1582

1583 1584
	/* cut the backing region down to size */
	region = vma->vm_region;
1585
	BUG_ON(region->vm_usage != 1);
1586 1587 1588

	down_write(&nommu_region_sem);
	delete_nommu_region(region);
1589 1590 1591 1592
	if (from > region->vm_start) {
		to = region->vm_top;
		region->vm_top = region->vm_end = from;
	} else {
1593
		region->vm_start = to;
1594
	}
1595 1596 1597 1598 1599 1600
	add_nommu_region(region);
	up_write(&nommu_region_sem);

	free_page_series(from, to);
	return 0;
}
Linus Torvalds's avatar
Linus Torvalds committed
1601

1602 1603 1604 1605 1606 1607 1608 1609
/*
 * release a mapping
 * - under NOMMU conditions the chunk to be unmapped must be backed by a single
 *   VMA, though it need not cover the whole VMA
 */
int do_munmap(struct mm_struct *mm, unsigned long start, size_t len)
{
	struct vm_area_struct *vma;
Bob Liu's avatar
Bob Liu committed
1610
	unsigned long end;
1611
	int ret;
Linus Torvalds's avatar
Linus Torvalds committed
1612

Bob Liu's avatar
Bob Liu committed
1613
	len = PAGE_ALIGN(len);
1614 1615
	if (len == 0)
		return -EINVAL;
1616

Bob Liu's avatar
Bob Liu committed
1617 1618
	end = start + len;

1619 1620 1621
	/* find the first potentially overlapping VMA */
	vma = find_vma(mm, start);
	if (!vma) {
1622
		static int limit;
1623
		if (limit < 5) {
1624 1625 1626
			pr_warn("munmap of memory not mmapped by process %d (%s): 0x%lx-0x%lx\n",
					current->pid, current->comm,
					start, start + len - 1);
1627 1628
			limit++;
		}
1629 1630
		return -EINVAL;
	}
Linus Torvalds's avatar
Linus Torvalds committed
1631

1632 1633 1634
	/* we're allowed to split an anonymous VMA but not a file-backed one */
	if (vma->vm_file) {
		do {
1635
			if (start > vma->vm_start)
1636 1637 1638
				return -EINVAL;
			if (end == vma->vm_end)
				goto erase_whole_vma;
1639 1640
			vma = vma->vm_next;
		} while (vma);
1641 1642 1643 1644 1645
		return -EINVAL;
	} else {
		/* the chunk must be a subset of the VMA found */
		if (start == vma->vm_start && end == vma->vm_end)
			goto erase_whole_vma;
1646
		if (start < vma->vm_start || end > vma->vm_end)
1647
			return -EINVAL;
1648
		if (offset_in_page(start))
1649
			return -EINVAL;
1650
		if (end != vma->vm_end && offset_in_page(end))
1651 1652 1653
			return -EINVAL;
		if (start != vma->vm_start && end != vma->vm_end) {
			ret = split_vma(mm, vma, start, 1);
1654
			if (ret < 0)
1655 1656 1657 1658
				return ret;
		}
		return shrink_vma(mm, vma, start, end);
	}
Linus Torvalds's avatar
Linus Torvalds committed
1659

1660 1661 1662
erase_whole_vma:
	delete_vma_from_mm(vma);
	delete_vma(mm, vma);
Linus Torvalds's avatar
Linus Torvalds committed
1663 1664
	return 0;
}
1665
EXPORT_SYMBOL(do_munmap);
Linus Torvalds's avatar
Linus Torvalds committed
1666

Al Viro's avatar
Al Viro committed
1667
int vm_munmap(unsigned long addr, size_t len)
1668
{
Al Viro's avatar
Al Viro committed
1669
	struct mm_struct *mm = current->mm;
1670 1671 1672 1673 1674 1675 1676
	int ret;

	down_write(&mm->mmap_sem);
	ret = do_munmap(mm, addr, len);
	up_write(&mm->mmap_sem);
	return ret;
}
1677 1678 1679 1680
EXPORT_SYMBOL(vm_munmap);

SYSCALL_DEFINE2(munmap, unsigned long, addr, size_t, len)
{
Al Viro's avatar
Al Viro committed
1681
	return vm_munmap(addr, len);
1682
}
1683 1684

/*
1685
 * release all the mappings made in a process's VM space
1686
 */
1687
void exit_mmap(struct mm_struct *mm)
Linus Torvalds's avatar
Linus Torvalds committed
1688
{
1689
	struct vm_area_struct *vma;
Linus Torvalds's avatar
Linus Torvalds committed
1690

1691 1692
	if (!mm)
		return;
Linus Torvalds's avatar
Linus Torvalds committed
1693

1694
	mm->total_vm = 0;
Linus Torvalds's avatar
Linus Torvalds committed
1695

1696 1697 1698 1699
	while ((vma = mm->mmap)) {
		mm->mmap = vma->vm_next;
		delete_vma_from_mm(vma);
		delete_vma(mm, vma);
1700
		cond_resched();
Linus Torvalds's avatar
Linus Torvalds committed
1701 1702 1703
	}
}

1704
unsigned long vm_brk(unsigned long addr, unsigned long len)
Linus Torvalds's avatar
Linus Torvalds committed
1705 1706 1707 1708 1709
{
	return -ENOMEM;
}

/*
1710 1711
 * expand (or shrink) an existing mapping, potentially moving it at the same
 * time (controlled by the MREMAP_MAYMOVE flag and available VM space)
Linus Torvalds's avatar
Linus Torvalds committed
1712
 *
1713
 * under NOMMU conditions, we only permit changing a mapping's size, and only
1714 1715
 * as long as it stays within the region allocated by do_mmap_private() and the
 * block is not shareable
Linus Torvalds's avatar
Linus Torvalds committed
1716
 *
1717
 * MREMAP_FIXED is not supported under NOMMU conditions
Linus Torvalds's avatar
Linus Torvalds committed
1718
 */
Al Viro's avatar
Al Viro committed
1719
static unsigned long do_mremap(unsigned long addr,
Linus Torvalds's avatar
Linus Torvalds committed
1720 1721 1722
			unsigned long old_len, unsigned long new_len,
			unsigned long flags, unsigned long new_addr)
{
1723
	struct vm_area_struct *vma;
Linus Torvalds's avatar
Linus Torvalds committed
1724 1725

	/* insanity checks first */
Bob Liu's avatar
Bob Liu committed
1726 1727
	old_len = PAGE_ALIGN(old_len);
	new_len = PAGE_ALIGN(new_len);
1728
	if (old_len == 0 || new_len == 0)
Linus Torvalds's avatar
Linus Torvalds committed
1729 1730
		return (unsigned long) -EINVAL;

1731
	if (offset_in_page(addr))
1732 1733
		return -EINVAL;

Linus Torvalds's avatar
Linus Torvalds committed
1734 1735 1736
	if (flags & MREMAP_FIXED && new_addr != addr)
		return (unsigned long) -EINVAL;

1737
	vma = find_vma_exact(current->mm, addr, old_len);
1738 1739
	if (!vma)
		return (unsigned long) -EINVAL;
Linus Torvalds's avatar
Linus Torvalds committed
1740

1741
	if (vma->vm_end != vma->vm_start + old_len)
Linus Torvalds's avatar
Linus Torvalds committed
1742 1743
		return (unsigned long) -EFAULT;

1744
	if (vma->vm_flags & VM_MAYSHARE)
Linus Torvalds's avatar
Linus Torvalds committed
1745 1746
		return (unsigned long) -EPERM;

1747
	if (new_len > vma->vm_region->vm_end - vma->vm_region->vm_start)
Linus Torvalds's avatar
Linus Torvalds committed
1748 1749 1750
		return (unsigned long) -ENOMEM;

	/* all checks complete - do it */
1751 1752 1753 1754
	vma->vm_end = vma->vm_start + new_len;
	return vma->vm_start;
}

1755 1756 1757
SYSCALL_DEFINE5(mremap, unsigned long, addr, unsigned long, old_len,
		unsigned long, new_len, unsigned long, flags,
		unsigned long, new_addr)
1758 1759 1760 1761 1762 1763 1764
{
	unsigned long ret;

	down_write(&current->mm->mmap_sem);
	ret = do_mremap(addr, old_len, new_len, flags, new_addr);
	up_write(&current->mm->mmap_sem);
	return ret;
Linus Torvalds's avatar
Linus Torvalds committed
1765 1766
}

1767 1768 1769
struct page *follow_page_mask(struct vm_area_struct *vma,
			      unsigned long address, unsigned int flags,
			      unsigned int *page_mask)
Linus Torvalds's avatar
Linus Torvalds committed
1770
{
1771
	*page_mask = 0;
Linus Torvalds's avatar
Linus Torvalds committed
1772 1773 1774
	return NULL;
}

Bob Liu's avatar
Bob Liu committed
1775 1776
int remap_pfn_range(struct vm_area_struct *vma, unsigned long addr,
		unsigned long pfn, unsigned long size, pgprot_t prot)
Linus Torvalds's avatar
Linus Torvalds committed
1777
{
Bob Liu's avatar
Bob Liu committed
1778 1779 1780
	if (addr != (pfn << PAGE_SHIFT))
		return -EINVAL;

1781
	vma->vm_flags |= VM_IO | VM_PFNMAP | VM_DONTEXPAND | VM_DONTDUMP;
1782
	return 0;
Linus Torvalds's avatar
Linus Torvalds committed
1783
}
1784
EXPORT_SYMBOL(remap_pfn_range);
Linus Torvalds's avatar
Linus Torvalds committed
1785

1786 1787 1788 1789 1790 1791 1792 1793 1794 1795
int vm_iomap_memory(struct vm_area_struct *vma, phys_addr_t start, unsigned long len)
{
	unsigned long pfn = start >> PAGE_SHIFT;
	unsigned long vm_len = vma->vm_end - vma->vm_start;

	pfn += vma->vm_pgoff;
	return io_remap_pfn_range(vma, vma->vm_start, pfn, vm_len, vma->vm_page_prot);
}
EXPORT_SYMBOL(vm_iomap_memory);

1796 1797 1798 1799 1800 1801 1802 1803 1804 1805 1806 1807 1808 1809 1810
int remap_vmalloc_range(struct vm_area_struct *vma, void *addr,
			unsigned long pgoff)
{
	unsigned int size = vma->vm_end - vma->vm_start;

	if (!(vma->vm_flags & VM_USERMAP))
		return -EINVAL;

	vma->vm_start = (unsigned long)(addr + (pgoff << PAGE_SHIFT));
	vma->vm_end = vma->vm_start + size;

	return 0;
}
EXPORT_SYMBOL(remap_vmalloc_range);

Linus Torvalds's avatar
Linus Torvalds committed
1811 1812 1813 1814 1815 1816 1817 1818 1819 1820 1821
unsigned long arch_get_unmapped_area(struct file *file, unsigned long addr,
	unsigned long len, unsigned long pgoff, unsigned long flags)
{
	return -ENOMEM;
}

void unmap_mapping_range(struct address_space *mapping,
			 loff_t const holebegin, loff_t const holelen,
			 int even_cows)
{
}
1822
EXPORT_SYMBOL(unmap_mapping_range);
Linus Torvalds's avatar
Linus Torvalds committed
1823 1824 1825 1826 1827 1828 1829 1830 1831 1832 1833 1834 1835 1836 1837 1838 1839

/*
 * Check that a process has enough memory to allocate a new virtual
 * mapping. 0 means there is enough memory for the allocation to
 * succeed and -ENOMEM implies there is not.
 *
 * We currently support three overcommit policies, which are set via the
 * vm.overcommit_memory sysctl.  See Documentation/vm/overcommit-accounting
 *
 * Strict overcommit modes added 2002 Feb 26 by Alan Cox.
 * Additional code 2002 Jul 20 by Robert Love.
 *
 * cap_sys_admin is 1 if the process has admin privileges, 0 otherwise.
 *
 * Note this is a helper function intended to be used by LSMs which
 * wish to use this logic.
 */
1840
int __vm_enough_memory(struct mm_struct *mm, long pages, int cap_sys_admin)
Linus Torvalds's avatar
Linus Torvalds committed
1841
{
1842
	long free, allowed, reserve;
Linus Torvalds's avatar
Linus Torvalds committed
1843 1844 1845 1846 1847 1848 1849 1850 1851 1852

	vm_acct_memory(pages);

	/*
	 * Sometimes we want to use more memory than we have
	 */
	if (sysctl_overcommit_memory == OVERCOMMIT_ALWAYS)
		return 0;

	if (sysctl_overcommit_memory == OVERCOMMIT_GUESS) {
1853 1854 1855 1856 1857 1858 1859 1860 1861 1862
		free = global_page_state(NR_FREE_PAGES);
		free += global_page_state(NR_FILE_PAGES);

		/*
		 * shmem pages shouldn't be counted as free in this
		 * case, they can't be purged, only swapped out, and
		 * that won't affect the overall amount of available
		 * memory in the system.
		 */
		free -= global_page_state(NR_SHMEM);
Linus Torvalds's avatar
Linus Torvalds committed
1863

1864
		free += get_nr_swap_pages();
Linus Torvalds's avatar
Linus Torvalds committed
1865 1866 1867 1868 1869 1870 1871

		/*
		 * Any slabs which are created with the
		 * SLAB_RECLAIM_ACCOUNT flag claim to have contents
		 * which are reclaimable, under pressure.  The dentry
		 * cache and most inode caches should fall into this
		 */
1872
		free += global_page_state(NR_SLAB_RECLAIMABLE);
Linus Torvalds's avatar
Linus Torvalds committed
1873

1874 1875 1876
		/*
		 * Leave reserved pages. The pages are not for anonymous pages.
		 */
1877
		if (free <= totalreserve_pages)
1878 1879
			goto error;
		else
1880
			free -= totalreserve_pages;
1881 1882

		/*
1883
		 * Reserve some for root
1884
		 */
Linus Torvalds's avatar
Linus Torvalds committed
1885
		if (!cap_sys_admin)
1886
			free -= sysctl_admin_reserve_kbytes >> (PAGE_SHIFT - 10);
Linus Torvalds's avatar
Linus Torvalds committed
1887 1888 1889

		if (free > pages)
			return 0;
1890 1891

		goto error;
Linus Torvalds's avatar
Linus Torvalds committed
1892 1893
	}

1894
	allowed = vm_commit_limit();
Linus Torvalds's avatar
Linus Torvalds committed
1895
	/*
1896
	 * Reserve some 3% for root
Linus Torvalds's avatar
Linus Torvalds committed
1897 1898
	 */
	if (!cap_sys_admin)
1899
		allowed -= sysctl_admin_reserve_kbytes >> (PAGE_SHIFT - 10);
Linus Torvalds's avatar
Linus Torvalds committed
1900

1901 1902 1903 1904 1905
	/*
	 * Don't let a single process grow so big a user can't recover
	 */
	if (mm) {
		reserve = sysctl_user_reserve_kbytes >> (PAGE_SHIFT - 10);
1906
		allowed -= min_t(long, mm->total_vm / 32, reserve);
1907
	}
Linus Torvalds's avatar
Linus Torvalds committed
1908

1909
	if (percpu_counter_read_positive(&vm_committed_as) < allowed)
Linus Torvalds's avatar
Linus Torvalds committed
1910
		return 0;
1911

1912
error:
Linus Torvalds's avatar
Linus Torvalds committed
1913 1914 1915 1916 1917
	vm_unacct_memory(pages);

	return -ENOMEM;
}

Nick Piggin's avatar
Nick Piggin committed
1918
int filemap_fault(struct vm_area_struct *vma, struct vm_fault *vmf)
1919 1920
{
	BUG();
Nick Piggin's avatar
Nick Piggin committed
1921
	return 0;
1922
}
1923
EXPORT_SYMBOL(filemap_fault);
1924

1925 1926 1927 1928 1929 1930
void filemap_map_pages(struct vm_area_struct *vma, struct vm_fault *vmf)
{
	BUG();
}
EXPORT_SYMBOL(filemap_map_pages);

1931
static int __access_remote_vm(struct task_struct *tsk, struct mm_struct *mm,
1932
		unsigned long addr, void *buf, int len, unsigned int gup_flags)
1933 1934
{
	struct vm_area_struct *vma;
1935
	int write = gup_flags & FOLL_WRITE;
1936 1937 1938 1939

	down_read(&mm->mmap_sem);

	/* the access must start within one of the target process's mappings */
1940 1941
	vma = find_vma(mm, addr);
	if (vma) {
1942 1943 1944 1945 1946
		/* don't overrun this mapping */
		if (addr + len >= vma->vm_end)
			len = vma->vm_end - addr;

		/* only read or write mappings where it is permitted */
1947
		if (write && vma->vm_flags & VM_MAYWRITE)
1948 1949
			copy_to_user_page(vma, NULL, addr,
					 (void *) addr, buf, len);
1950
		else if (!write && vma->vm_flags & VM_MAYREAD)
1951 1952
			copy_from_user_page(vma, NULL, addr,
					    buf, (void *) addr, len);
1953 1954 1955 1956 1957 1958 1959
		else
			len = 0;
	} else {
		len = 0;
	}

	up_read(&mm->mmap_sem);
1960 1961 1962 1963 1964 1965 1966 1967 1968 1969

	return len;
}

/**
 * @access_remote_vm - access another process' address space
 * @mm:		the mm_struct of the target address space
 * @addr:	start address to access
 * @buf:	source or destination buffer
 * @len:	number of bytes to transfer
1970
 * @gup_flags:	flags modifying lookup behaviour
1971 1972 1973 1974
 *
 * The caller must hold a reference on @mm.
 */
int access_remote_vm(struct mm_struct *mm, unsigned long addr,
1975
		void *buf, int len, unsigned int gup_flags)
1976
{
1977
	return __access_remote_vm(NULL, mm, addr, buf, len, gup_flags);
1978 1979 1980 1981 1982 1983 1984 1985 1986 1987 1988 1989 1990 1991 1992 1993 1994
}

/*
 * Access another process' address space.
 * - source/target buffer must be kernel space
 */
int access_process_vm(struct task_struct *tsk, unsigned long addr, void *buf, int len, int write)
{
	struct mm_struct *mm;

	if (addr + len < addr)
		return 0;

	mm = get_task_mm(tsk);
	if (!mm)
		return 0;

1995 1996
	len = __access_remote_vm(tsk, mm, addr, buf, len,
			write ? FOLL_WRITE : 0);
1997

1998 1999 2000
	mmput(mm);
	return len;
}
2001 2002 2003 2004 2005 2006 2007 2008 2009 2010 2011 2012 2013 2014 2015 2016 2017 2018 2019 2020 2021 2022 2023 2024

/**
 * nommu_shrink_inode_mappings - Shrink the shared mappings on an inode
 * @inode: The inode to check
 * @size: The current filesize of the inode
 * @newsize: The proposed filesize of the inode
 *
 * Check the shared mappings on an inode on behalf of a shrinking truncate to
 * make sure that that any outstanding VMAs aren't broken and then shrink the
 * vm_regions that extend that beyond so that do_mmap_pgoff() doesn't
 * automatically grant mappings that are too large.
 */
int nommu_shrink_inode_mappings(struct inode *inode, size_t size,
				size_t newsize)
{
	struct vm_area_struct *vma;
	struct vm_region *region;
	pgoff_t low, high;
	size_t r_size, r_top;

	low = newsize >> PAGE_SHIFT;
	high = (size + PAGE_SIZE - 1) >> PAGE_SHIFT;

	down_write(&nommu_region_sem);
2025
	i_mmap_lock_read(inode->i_mapping);
2026 2027

	/* search for VMAs that fall within the dead zone */
2028
	vma_interval_tree_foreach(vma, &inode->i_mapping->i_mmap, low, high) {
2029 2030 2031
		/* found one - only interested if it's shared out of the page
		 * cache */
		if (vma->vm_flags & VM_SHARED) {
2032
			i_mmap_unlock_read(inode->i_mapping);
2033 2034 2035 2036 2037 2038 2039 2040 2041 2042 2043
			up_write(&nommu_region_sem);
			return -ETXTBSY; /* not quite true, but near enough */
		}
	}

	/* reduce any regions that overlap the dead zone - if in existence,
	 * these will be pointed to by VMAs that don't overlap the dead zone
	 *
	 * we don't check for any regions that start beyond the EOF as there
	 * shouldn't be any
	 */
2044
	vma_interval_tree_foreach(vma, &inode->i_mapping->i_mmap, 0, ULONG_MAX) {
2045 2046 2047 2048 2049 2050 2051 2052 2053 2054 2055 2056 2057 2058
		if (!(vma->vm_flags & VM_SHARED))
			continue;

		region = vma->vm_region;
		r_size = region->vm_top - region->vm_start;
		r_top = (region->vm_pgoff << PAGE_SHIFT) + r_size;

		if (r_top > newsize) {
			region->vm_top -= r_top - newsize;
			if (region->vm_end > region->vm_top)
				region->vm_end = region->vm_top;
		}
	}

2059
	i_mmap_unlock_read(inode->i_mapping);
2060 2061 2062
	up_write(&nommu_region_sem);
	return 0;
}
2063 2064 2065 2066 2067 2068 2069 2070 2071 2072 2073 2074 2075 2076 2077 2078 2079 2080 2081 2082

/*
 * Initialise sysctl_user_reserve_kbytes.
 *
 * This is intended to prevent a user from starting a single memory hogging
 * process, such that they cannot recover (kill the hog) in OVERCOMMIT_NEVER
 * mode.
 *
 * The default value is min(3% of free memory, 128MB)
 * 128MB is enough to recover with sshd/login, bash, and top/kill.
 */
static int __meminit init_user_reserve(void)
{
	unsigned long free_kbytes;

	free_kbytes = global_page_state(NR_FREE_PAGES) << (PAGE_SHIFT - 10);

	sysctl_user_reserve_kbytes = min(free_kbytes / 32, 1UL << 17);
	return 0;
}
2083
subsys_initcall(init_user_reserve);
2084 2085 2086 2087 2088 2089 2090 2091 2092 2093 2094 2095 2096 2097 2098 2099 2100 2101 2102 2103

/*
 * Initialise sysctl_admin_reserve_kbytes.
 *
 * The purpose of sysctl_admin_reserve_kbytes is to allow the sys admin
 * to log in and kill a memory hogging process.
 *
 * Systems with more than 256MB will reserve 8MB, enough to recover
 * with sshd, bash, and top in OVERCOMMIT_GUESS. Smaller systems will
 * only reserve 3% of free pages by default.
 */
static int __meminit init_admin_reserve(void)
{
	unsigned long free_kbytes;

	free_kbytes = global_page_state(NR_FREE_PAGES) << (PAGE_SHIFT - 10);

	sysctl_admin_reserve_kbytes = min(free_kbytes / 32, 1UL << 13);
	return 0;
}
2104
subsys_initcall(init_admin_reserve);