mmap.c 39.1 KB
Newer Older
Linus Torvalds's avatar
Linus Torvalds committed
1
/*
Andrew Morton's avatar
Andrew Morton committed
2
 * mm/mmap.c
Linus Torvalds's avatar
Linus Torvalds committed
3 4
 *
 * Written by obz.
Andrew Morton's avatar
Andrew Morton committed
5 6
 *
 * Address space accounting code	<alan@redhat.com>
Linus Torvalds's avatar
Linus Torvalds committed
7
 */
Andrew Morton's avatar
Andrew Morton committed
8

Linus Torvalds's avatar
Linus Torvalds committed
9 10 11 12 13
#include <linux/slab.h>
#include <linux/shm.h>
#include <linux/mman.h>
#include <linux/pagemap.h>
#include <linux/swap.h>
Andrew Morton's avatar
Andrew Morton committed
14
#include <linux/syscalls.h>
Linus Torvalds's avatar
Linus Torvalds committed
15 16
#include <linux/init.h>
#include <linux/file.h>
Linus Torvalds's avatar
Linus Torvalds committed
17
#include <linux/fs.h>
Linus Torvalds's avatar
Linus Torvalds committed
18
#include <linux/personality.h>
19
#include <linux/security.h>
20
#include <linux/hugetlb.h>
John Levon's avatar
John Levon committed
21
#include <linux/profile.h>
22
#include <linux/module.h>
23
#include <linux/mount.h>
24
#include <linux/mempolicy.h>
Linus Torvalds's avatar
Linus Torvalds committed
25 26 27

#include <asm/uaccess.h>
#include <asm/pgalloc.h>
28 29
#include <asm/tlb.h>

Linus Torvalds's avatar
Linus Torvalds committed
30 31 32 33 34 35
/*
 * WARNING: the debugging will use recursive algorithms so never enable this
 * unless you know what you are doing.
 */
#undef DEBUG_MM_RB

Linus Torvalds's avatar
Linus Torvalds committed
36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55
/* description of effects of mapping type and prot in current implementation.
 * this is due to the limited x86 page protection hardware.  The expected
 * behavior is in parens:
 *
 * map_type	prot
 *		PROT_NONE	PROT_READ	PROT_WRITE	PROT_EXEC
 * MAP_SHARED	r: (no) no	r: (yes) yes	r: (no) yes	r: (no) yes
 *		w: (no) no	w: (no) no	w: (yes) yes	w: (no) no
 *		x: (no) no	x: (no) yes	x: (no) yes	x: (yes) yes
 *		
 * MAP_PRIVATE	r: (no) no	r: (yes) yes	r: (no) yes	r: (no) yes
 *		w: (no) no	w: (no) no	w: (copy) copy	w: (no) no
 *		x: (no) no	x: (no) yes	x: (no) yes	x: (yes) yes
 *
 */
pgprot_t protection_map[16] = {
	__P000, __P001, __P010, __P011, __P100, __P101, __P110, __P111,
	__S000, __S001, __S010, __S011, __S100, __S101, __S110, __S111
};

Andrew Morton's avatar
Andrew Morton committed
56 57
int sysctl_overcommit_memory = 0;	/* default is heuristic overcommit */
int sysctl_overcommit_ratio = 50;	/* default is 50% */
58
int sysctl_max_map_count = DEFAULT_MAX_MAP_COUNT;
Andrew Morton's avatar
Andrew Morton committed
59 60
atomic_t vm_committed_space = ATOMIC_INIT(0);

61 62
EXPORT_SYMBOL(sysctl_overcommit_memory);
EXPORT_SYMBOL(sysctl_overcommit_ratio);
63
EXPORT_SYMBOL(sysctl_max_map_count);
64
EXPORT_SYMBOL(vm_committed_space);
Andrew Morton's avatar
Andrew Morton committed
65

66
/*
67
 * Requires inode->i_mapping->i_mmap_lock
68 69
 */
static inline void
Andrew Morton's avatar
Andrew Morton committed
70
__remove_shared_vm_struct(struct vm_area_struct *vma, struct file *file)
71
{
Andrew Morton's avatar
Andrew Morton committed
72 73 74
	if (vma->vm_flags & VM_DENYWRITE)
		atomic_inc(&file->f_dentry->d_inode->i_writecount);
	list_del_init(&vma->shared);
75 76
}

77 78 79
/*
 * Remove one vm structure from the inode's i_mapping address space.
 */
80
static void remove_shared_vm_struct(struct vm_area_struct *vma)
Linus Torvalds's avatar
Linus Torvalds committed
81
{
82
	struct file *file = vma->vm_file;
Linus Torvalds's avatar
Linus Torvalds committed
83 84

	if (file) {
85
		struct address_space *mapping = file->f_mapping;
86
		spin_lock(&mapping->i_mmap_lock);
Andrew Morton's avatar
Andrew Morton committed
87
		__remove_shared_vm_struct(vma, file);
88
		spin_unlock(&mapping->i_mmap_lock);
Linus Torvalds's avatar
Linus Torvalds committed
89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104
	}
}

/*
 *  sys_brk() for the most part doesn't need the global kernel
 *  lock, except when an application is doing something nasty
 *  like trying to un-brk an area that has already been mapped
 *  to a regular file.  in this case, the unmapping will need
 *  to invoke file system routines that need the global lock.
 */
asmlinkage unsigned long sys_brk(unsigned long brk)
{
	unsigned long rlim, retval;
	unsigned long newbrk, oldbrk;
	struct mm_struct *mm = current->mm;

Linus Torvalds's avatar
Linus Torvalds committed
105
	down_write(&mm->mmap_sem);
Linus Torvalds's avatar
Linus Torvalds committed
106 107 108 109 110 111 112 113 114 115

	if (brk < mm->end_code)
		goto out;
	newbrk = PAGE_ALIGN(brk);
	oldbrk = PAGE_ALIGN(mm->brk);
	if (oldbrk == newbrk)
		goto set_brk;

	/* Always allow shrinking brk. */
	if (brk <= mm->brk) {
116
		if (!do_munmap(mm, newbrk, oldbrk-newbrk))
Linus Torvalds's avatar
Linus Torvalds committed
117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136
			goto set_brk;
		goto out;
	}

	/* Check against rlimit.. */
	rlim = current->rlim[RLIMIT_DATA].rlim_cur;
	if (rlim < RLIM_INFINITY && brk - mm->start_data > rlim)
		goto out;

	/* Check against existing mmap mappings. */
	if (find_vma_intersection(mm, oldbrk, newbrk+PAGE_SIZE))
		goto out;

	/* Ok, looks good - let it rip. */
	if (do_brk(oldbrk, newbrk-oldbrk) != oldbrk)
		goto out;
set_brk:
	mm->brk = brk;
out:
	retval = mm->brk;
Linus Torvalds's avatar
Linus Torvalds committed
137
	up_write(&mm->mmap_sem);
Linus Torvalds's avatar
Linus Torvalds committed
138 139 140
	return retval;
}

Linus Torvalds's avatar
Linus Torvalds committed
141
#ifdef DEBUG_MM_RB
142 143 144 145 146 147 148 149 150 151 152 153 154 155 156
static int browse_rb(struct rb_root *root) {
	int i, j;
	struct rb_node *nd, *pn = NULL;
	i = 0;
	unsigned long prev = 0, pend = 0;

	for (nd = rb_first(root); nd; nd = rb_next(nd)) {
		struct vm_area_struct *vma;
		vma = rb_entry(nd, struct vm_area_struct, vm_rb);
		if (vma->vm_start < prev)
			printk("vm_start %lx prev %lx\n", vma->vm_start, prev), i = -1;
		if (vma->vm_start < pend)
			printk("vm_start %lx pend %lx\n", vma->vm_start, pend);
		if (vma->vm_start > vma->vm_end)
			printk("vm_end %lx < vm_start %lx\n", vma->vm_end, vma->vm_start);
Linus Torvalds's avatar
Linus Torvalds committed
157
		i++;
158 159 160 161 162
		pn = nd;
	}
	j = 0;
	for (nd = pn; nd; nd = rb_prev(nd)) {
		j++;
Linus Torvalds's avatar
Linus Torvalds committed
163
	}
164 165
	if (i != j)
		printk("backwards %d, forwards %d\n", j, i), i = 0;
Linus Torvalds's avatar
Linus Torvalds committed
166 167 168
	return i;
}

169
void validate_mm(struct mm_struct * mm) {
Linus Torvalds's avatar
Linus Torvalds committed
170 171 172 173 174 175 176 177 178
	int bug = 0;
	int i = 0;
	struct vm_area_struct * tmp = mm->mmap;
	while (tmp) {
		tmp = tmp->vm_next;
		i++;
	}
	if (i != mm->map_count)
		printk("map_count %d vm_next %d\n", mm->map_count, i), bug = 1;
179
	i = browse_rb(&mm->mm_rb);
Linus Torvalds's avatar
Linus Torvalds committed
180 181 182 183 184 185 186 187 188
	if (i != mm->map_count)
		printk("map_count %d rb %d\n", mm->map_count, i), bug = 1;
	if (bug)
		BUG();
}
#else
#define validate_mm(mm) do { } while (0)
#endif

189 190 191 192
static struct vm_area_struct *
find_vma_prepare(struct mm_struct *mm, unsigned long addr,
		struct vm_area_struct **pprev, struct rb_node ***rb_link,
		struct rb_node ** rb_parent)
Linus Torvalds's avatar
Linus Torvalds committed
193 194
{
	struct vm_area_struct * vma;
195
	struct rb_node ** __rb_link, * __rb_parent, * rb_prev;
Linus Torvalds's avatar
Linus Torvalds committed
196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225

	__rb_link = &mm->mm_rb.rb_node;
	rb_prev = __rb_parent = NULL;
	vma = NULL;

	while (*__rb_link) {
		struct vm_area_struct *vma_tmp;

		__rb_parent = *__rb_link;
		vma_tmp = rb_entry(__rb_parent, struct vm_area_struct, vm_rb);

		if (vma_tmp->vm_end > addr) {
			vma = vma_tmp;
			if (vma_tmp->vm_start <= addr)
				return vma;
			__rb_link = &__rb_parent->rb_left;
		} else {
			rb_prev = __rb_parent;
			__rb_link = &__rb_parent->rb_right;
		}
	}

	*pprev = NULL;
	if (rb_prev)
		*pprev = rb_entry(rb_prev, struct vm_area_struct, vm_rb);
	*rb_link = __rb_link;
	*rb_parent = __rb_parent;
	return vma;
}

226 227 228
static inline void
__vma_link_list(struct mm_struct *mm, struct vm_area_struct *vma,
		struct vm_area_struct *prev, struct rb_node *rb_parent)
Linus Torvalds's avatar
Linus Torvalds committed
229 230 231 232 233 234 235
{
	if (prev) {
		vma->vm_next = prev->vm_next;
		prev->vm_next = vma;
	} else {
		mm->mmap = vma;
		if (rb_parent)
236 237
			vma->vm_next = rb_entry(rb_parent,
					struct vm_area_struct, vm_rb);
Linus Torvalds's avatar
Linus Torvalds committed
238 239 240 241 242
		else
			vma->vm_next = NULL;
	}
}

Andrew Morton's avatar
Andrew Morton committed
243 244
void __vma_link_rb(struct mm_struct *mm, struct vm_area_struct *vma,
		struct rb_node **rb_link, struct rb_node *rb_parent)
Linus Torvalds's avatar
Linus Torvalds committed
245 246 247 248 249
{
	rb_link_node(&vma->vm_rb, rb_parent, rb_link);
	rb_insert_color(&vma->vm_rb, &mm->mm_rb);
}

250
static inline void __vma_link_file(struct vm_area_struct *vma)
Linus Torvalds's avatar
Linus Torvalds committed
251 252 253 254 255
{
	struct file * file;

	file = vma->vm_file;
	if (file) {
256
		struct address_space *mapping = file->f_mapping;
Linus Torvalds's avatar
Linus Torvalds committed
257 258

		if (vma->vm_flags & VM_DENYWRITE)
259
			atomic_dec(&file->f_dentry->d_inode->i_writecount);
Linus Torvalds's avatar
Linus Torvalds committed
260 261

		if (vma->vm_flags & VM_SHARED)
262 263 264
			list_add_tail(&vma->shared, &mapping->i_mmap_shared);
		else
			list_add_tail(&vma->shared, &mapping->i_mmap);
Linus Torvalds's avatar
Linus Torvalds committed
265 266 267
	}
}

268 269 270 271
static void
__vma_link(struct mm_struct *mm, struct vm_area_struct *vma,
	struct vm_area_struct *prev, struct rb_node **rb_link,
	struct rb_node *rb_parent)
Linus Torvalds's avatar
Linus Torvalds committed
272 273 274 275 276 277
{
	__vma_link_list(mm, vma, prev, rb_parent);
	__vma_link_rb(mm, vma, rb_link, rb_parent);
	__vma_link_file(vma);
}

278 279 280
static void vma_link(struct mm_struct *mm, struct vm_area_struct *vma,
			struct vm_area_struct *prev, struct rb_node **rb_link,
			struct rb_node *rb_parent)
Linus Torvalds's avatar
Linus Torvalds committed
281
{
282 283 284
	struct address_space *mapping = NULL;

	if (vma->vm_file)
285
		mapping = vma->vm_file->f_mapping;
286 287

	if (mapping)
288
		spin_lock(&mapping->i_mmap_lock);
Linus Torvalds's avatar
Linus Torvalds committed
289 290
	spin_lock(&mm->page_table_lock);
	__vma_link(mm, vma, prev, rb_link, rb_parent);
291
	spin_unlock(&mm->page_table_lock);
292
	if (mapping)
293
		spin_unlock(&mapping->i_mmap_lock);
Linus Torvalds's avatar
Linus Torvalds committed
294

295
	mark_mm_hugetlb(mm, vma);
Linus Torvalds's avatar
Linus Torvalds committed
296 297 298 299
	mm->map_count++;
	validate_mm(mm);
}

300 301 302
/*
 * Insert vm structure into process list sorted by address and into the inode's
 * i_mmap ring. The caller should hold mm->page_table_lock and
303
 * ->f_mappping->i_mmap_lock if vm_file is non-NULL.
304 305 306 307 308 309 310 311 312 313 314 315 316 317 318 319
 */
static void
__insert_vm_struct(struct mm_struct * mm, struct vm_area_struct * vma)
{
	struct vm_area_struct * __vma, * prev;
	struct rb_node ** rb_link, * rb_parent;

	__vma = find_vma_prepare(mm, vma->vm_start,&prev, &rb_link, &rb_parent);
	if (__vma && __vma->vm_start < vma->vm_end)
		BUG();
	__vma_link(mm, vma, prev, rb_link, rb_parent);
	mark_mm_hugetlb(mm, vma);
	mm->map_count++;
	validate_mm(mm);
}

Andrew Morton's avatar
Andrew Morton committed
320 321 322 323 324 325 326 327 328 329 330 331 332 333 334 335 336 337 338 339 340 341 342 343 344 345 346 347 348 349 350 351 352 353 354 355 356 357 358 359 360 361 362 363 364 365 366 367
/*
 * We cannot adjust vm_start, vm_end, vm_pgoff fields of a vma that is
 * already present in an i_mmap{_shared} tree without adjusting the tree.
 * The following helper function should be used when such adjustments
 * are necessary.  The "next" vma (if any) is to be removed or inserted
 * before we drop the necessary locks.
 */
void vma_adjust(struct vm_area_struct *vma, unsigned long start,
	unsigned long end, pgoff_t pgoff, struct vm_area_struct *next)
{
	struct mm_struct *mm = vma->vm_mm;
	struct address_space *mapping = NULL;
	struct file *file = vma->vm_file;

	if (file) {
		mapping = file->f_mapping;
		spin_lock(&mapping->i_mmap_lock);
	}
	spin_lock(&mm->page_table_lock);

	vma->vm_start = start;
	vma->vm_end = end;
	vma->vm_pgoff = pgoff;

	if (next) {
		if (next == vma->vm_next) {
			/*
			 * vma_merge has merged next into vma, and needs
			 * us to remove next before dropping the locks.
			 */
			__vma_unlink(mm, next, vma);
			if (file)
				__remove_shared_vm_struct(next, file);
		} else {
			/*
			 * split_vma has split next from vma, and needs
			 * us to insert next before dropping the locks
			 * (next may either follow vma or precede it).
			 */
			__insert_vm_struct(mm, next);
		}
	}

	spin_unlock(&mm->page_table_lock);
	if (mapping)
		spin_unlock(&mapping->i_mmap_lock);
}

368 369 370 371 372 373 374 375 376 377 378 379 380 381 382 383 384 385
/*
 * If the vma has a ->close operation then the driver probably needs to release
 * per-vma resources, so we don't attempt to merge those.
 */
#define VM_SPECIAL (VM_IO | VM_DONTCOPY | VM_DONTEXPAND | VM_RESERVED)

static inline int is_mergeable_vma(struct vm_area_struct *vma,
			struct file *file, unsigned long vm_flags)
{
	if (vma->vm_ops && vma->vm_ops->close)
		return 0;
	if (vma->vm_file != file)
		return 0;
	if (vma->vm_flags != vm_flags)
		return 0;
	return 1;
}

386 387 388 389 390 391 392 393 394 395 396 397
/*
 * Return true if we can merge this (vm_flags,file,vm_pgoff,size)
 * in front of (at a lower virtual address and file offset than) the vma.
 *
 * We don't check here for the merged mmap wrapping around the end of pagecache
 * indices (16TB on ia32) because do_mmap_pgoff() does not permit mmap's which
 * wrap, nor mmaps which cover the final page at index -1UL.
 */
static int
can_vma_merge_before(struct vm_area_struct *vma, unsigned long vm_flags,
	struct file *file, unsigned long vm_pgoff, unsigned long size)
{
398
	if (is_mergeable_vma(vma, file, vm_flags)) {
399 400 401 402 403 404 405 406 407 408 409 410 411 412 413 414
		if (!file)
			return 1;	/* anon mapping */
		if (vma->vm_pgoff == vm_pgoff + size)
			return 1;
	}
	return 0;
}

/*
 * Return true if we can merge this (vm_flags,file,vm_pgoff)
 * beyond (at a higher virtual address and file offset than) the vma.
 */
static int
can_vma_merge_after(struct vm_area_struct *vma, unsigned long vm_flags,
	struct file *file, unsigned long vm_pgoff)
{
415
	if (is_mergeable_vma(vma, file, vm_flags)) {
416 417 418 419 420 421 422 423 424 425 426 427 428 429 430 431 432
		unsigned long vma_size;

		if (!file)
			return 1;	/* anon mapping */

		vma_size = (vma->vm_end - vma->vm_start) >> PAGE_SHIFT;
		if (vma->vm_pgoff + vma_size == vm_pgoff)
			return 1;
	}
	return 0;
}

/*
 * Given a new mapping request (addr,end,vm_flags,file,pgoff), figure out
 * whether that can be merged with its predecessor or its successor.  Or
 * both (it neatly fills a hole).
 */
433 434
static struct vm_area_struct *vma_merge(struct mm_struct *mm,
			struct vm_area_struct *prev,
435 436
			struct rb_node *rb_parent, unsigned long addr, 
			unsigned long end, unsigned long vm_flags,
437 438
		     	struct file *file, unsigned long pgoff,
		        struct mempolicy *policy)
Linus Torvalds's avatar
Linus Torvalds committed
439
{
Andrew Morton's avatar
Andrew Morton committed
440
	struct vm_area_struct *next;
441

442
	/*
Andrew Morton's avatar
Andrew Morton committed
443 444
	 * We later require that vma->vm_flags == vm_flags,
	 * so this tests vma->vm_flags & VM_SPECIAL, too.
445 446
	 */
	if (vm_flags & VM_SPECIAL)
447
		return NULL;
448

Linus Torvalds's avatar
Linus Torvalds committed
449
	if (!prev) {
Andrew Morton's avatar
Andrew Morton committed
450
		next = rb_entry(rb_parent, struct vm_area_struct, vm_rb);
Linus Torvalds's avatar
Linus Torvalds committed
451 452
		goto merge_next;
	}
Andrew Morton's avatar
Andrew Morton committed
453
	next = prev->vm_next;
Linus Torvalds's avatar
Linus Torvalds committed
454

455 456 457 458
	/*
	 * Can it merge with the predecessor?
	 */
	if (prev->vm_end == addr &&
Andrew Morton's avatar
Andrew Morton committed
459
  			mpol_equal(vma_policy(prev), policy) &&
460 461
			can_vma_merge_after(prev, vm_flags, file, pgoff)) {
		/*
Andrew Morton's avatar
Andrew Morton committed
462
		 * OK, it can.  Can we now merge in the successor as well?
463
		 */
Andrew Morton's avatar
Andrew Morton committed
464 465
		if (next && end == next->vm_start &&
				mpol_equal(policy, vma_policy(next)) &&
466 467
				can_vma_merge_before(next, vm_flags, file,
					pgoff, (end - addr) >> PAGE_SHIFT)) {
Andrew Morton's avatar
Andrew Morton committed
468 469
			vma_adjust(prev, prev->vm_start,
				next->vm_end, prev->vm_pgoff, next);
470 471
			if (file)
				fput(file);
Linus Torvalds's avatar
Linus Torvalds committed
472
			mm->map_count--;
473
			mpol_free(vma_policy(next));
Linus Torvalds's avatar
Linus Torvalds committed
474
			kmem_cache_free(vm_area_cachep, next);
Andrew Morton's avatar
Andrew Morton committed
475 476 477
		} else
			vma_adjust(prev, prev->vm_start,
				end, prev->vm_pgoff, NULL);
478
		return prev;
Linus Torvalds's avatar
Linus Torvalds committed
479 480
	}

481
	/*
Andrew Morton's avatar
Andrew Morton committed
482
	 * Can this new request be merged in front of next?
483
	 */
Andrew Morton's avatar
Andrew Morton committed
484
	if (next) {
Linus Torvalds's avatar
Linus Torvalds committed
485
 merge_next:
Andrew Morton's avatar
Andrew Morton committed
486 487 488 489 490 491 492
		if (end == next->vm_start &&
 				mpol_equal(policy, vma_policy(next)) &&
				can_vma_merge_before(next, vm_flags, file,
					pgoff, (end - addr) >> PAGE_SHIFT)) {
			vma_adjust(next, addr,
				next->vm_end, pgoff, NULL);
			return next;
Linus Torvalds's avatar
Linus Torvalds committed
493 494 495
		}
	}

496
	return NULL;
Linus Torvalds's avatar
Linus Torvalds committed
497 498
}

Andrew Morton's avatar
Andrew Morton committed
499 500 501 502
/*
 * The caller must hold down_write(current->mm->mmap_sem).
 */

Andrew Morton's avatar
Andrew Morton committed
503 504 505
unsigned long do_mmap_pgoff(struct file * file, unsigned long addr,
			unsigned long len, unsigned long prot,
			unsigned long flags, unsigned long pgoff)
Linus Torvalds's avatar
Linus Torvalds committed
506 507
{
	struct mm_struct * mm = current->mm;
Linus Torvalds's avatar
Linus Torvalds committed
508
	struct vm_area_struct * vma, * prev;
509
	struct inode *inode;
Linus Torvalds's avatar
Linus Torvalds committed
510
	unsigned int vm_flags;
Linus Torvalds's avatar
Linus Torvalds committed
511 512
	int correct_wcount = 0;
	int error;
513
	struct rb_node ** rb_link, * rb_parent;
514
	int accountable = 1;
Andrew Morton's avatar
Andrew Morton committed
515
	unsigned long charged = 0;
Linus Torvalds's avatar
Linus Torvalds committed
516

517
	if (file) {
518 519 520
		if (is_file_hugepages(file))
			accountable = 0;

521 522
		if (!file->f_op || !file->f_op->mmap)
			return -ENODEV;
Linus Torvalds's avatar
Linus Torvalds committed
523

524 525 526
		if ((prot & PROT_EXEC) && (file->f_vfsmnt->mnt_flags & MNT_NOEXEC))
			return -EPERM;
	}
527

528
	if (!len)
Linus Torvalds's avatar
Linus Torvalds committed
529 530
		return addr;

531
	/* Careful about overflows.. */
532
	len = PAGE_ALIGN(len);
533 534
	if (!len || len > TASK_SIZE)
		return -EINVAL;
535

Linus Torvalds's avatar
Linus Torvalds committed
536 537 538 539 540
	/* offset overflow? */
	if ((pgoff + (len >> PAGE_SHIFT)) < pgoff)
		return -EINVAL;

	/* Too many mappings? */
541
	if (mm->map_count > sysctl_max_map_count)
Linus Torvalds's avatar
Linus Torvalds committed
542 543
		return -ENOMEM;

Linus Torvalds's avatar
Linus Torvalds committed
544 545 546
	/* Obtain the address to map to. we verify (or select) it and ensure
	 * that it represents a valid section of the address space.
	 */
Linus Torvalds's avatar
Linus Torvalds committed
547 548 549
	addr = get_unmapped_area(file, addr, len, pgoff, flags);
	if (addr & ~PAGE_MASK)
		return addr;
Linus Torvalds's avatar
Linus Torvalds committed
550 551 552 553 554

	/* Do simple checking here so the lower-level routines won't have
	 * to. we assume access permissions have been handled by the open
	 * of the memory object, so we don't do any here.
	 */
555 556
	vm_flags = calc_vm_prot_bits(prot) | calc_vm_flag_bits(flags) |
			mm->def_flags | VM_MAYREAD | VM_MAYWRITE | VM_MAYEXEC;
Linus Torvalds's avatar
Linus Torvalds committed
557

Andrew Morton's avatar
Andrew Morton committed
558 559 560 561 562
	if (flags & MAP_LOCKED) {
		if (!capable(CAP_IPC_LOCK))
			return -EPERM;
		vm_flags |= VM_LOCKED;
	}
Linus Torvalds's avatar
Linus Torvalds committed
563
	/* mlock MCL_FUTURE? */
Linus Torvalds's avatar
Linus Torvalds committed
564
	if (vm_flags & VM_LOCKED) {
Linus Torvalds's avatar
Linus Torvalds committed
565 566 567 568 569 570
		unsigned long locked = mm->locked_vm << PAGE_SHIFT;
		locked += len;
		if (locked > current->rlim[RLIMIT_MEMLOCK].rlim_cur)
			return -EAGAIN;
	}

571 572
	inode = file ? file->f_dentry->d_inode : NULL;

Linus Torvalds's avatar
Linus Torvalds committed
573
	if (file) {
Linus Torvalds's avatar
Linus Torvalds committed
574 575
		switch (flags & MAP_TYPE) {
		case MAP_SHARED:
576
			if ((prot&PROT_WRITE) && !(file->f_mode&FMODE_WRITE))
Linus Torvalds's avatar
Linus Torvalds committed
577 578
				return -EACCES;

579 580 581 582
			/*
			 * Make sure we don't allow writing to an append-only
			 * file..
			 */
583
			if (IS_APPEND(inode) && (file->f_mode & FMODE_WRITE))
Linus Torvalds's avatar
Linus Torvalds committed
584 585
				return -EACCES;

586 587 588
			/*
			 * Make sure there are no mandatory locks on the file.
			 */
589
			if (locks_verify_locked(inode))
Linus Torvalds's avatar
Linus Torvalds committed
590 591
				return -EAGAIN;

Linus Torvalds's avatar
Linus Torvalds committed
592 593 594 595
			vm_flags |= VM_SHARED | VM_MAYSHARE;
			if (!(file->f_mode & FMODE_WRITE))
				vm_flags &= ~(VM_MAYWRITE | VM_SHARED);

Linus Torvalds's avatar
Linus Torvalds committed
596 597 598 599 600 601 602 603 604
			/* fall through */
		case MAP_PRIVATE:
			if (!(file->f_mode & FMODE_READ))
				return -EACCES;
			break;

		default:
			return -EINVAL;
		}
Linus Torvalds's avatar
Linus Torvalds committed
605 606 607 608 609 610 611 612 613 614 615
	} else {
		vm_flags |= VM_SHARED | VM_MAYSHARE;
		switch (flags & MAP_TYPE) {
		default:
			return -EINVAL;
		case MAP_PRIVATE:
			vm_flags &= ~(VM_SHARED | VM_MAYSHARE);
			/* fall through */
		case MAP_SHARED:
			break;
		}
Linus Torvalds's avatar
Linus Torvalds committed
616 617
	}

618 619
	error = security_file_mmap(file, prot, flags);
	if (error)
620 621
		return error;
		
Linus Torvalds's avatar
Linus Torvalds committed
622 623
	/* Clear old maps */
	error = -ENOMEM;
Linus Torvalds's avatar
Linus Torvalds committed
624 625 626
munmap_back:
	vma = find_vma_prepare(mm, addr, &prev, &rb_link, &rb_parent);
	if (vma && vma->vm_start < addr + len) {
627
		if (do_munmap(mm, addr, len))
Linus Torvalds's avatar
Linus Torvalds committed
628 629 630
			return -ENOMEM;
		goto munmap_back;
	}
Linus Torvalds's avatar
Linus Torvalds committed
631 632 633 634 635 636

	/* Check against address space limit. */
	if ((mm->total_vm << PAGE_SHIFT) + len
	    > current->rlim[RLIMIT_AS].rlim_cur)
		return -ENOMEM;

637 638
	if (accountable && (!(flags & MAP_NORESERVE) ||
			sysctl_overcommit_memory > 1)) {
639 640 641 642
		if (vm_flags & VM_SHARED) {
			/* Check memory availability in shmem_file_setup? */
			vm_flags |= VM_ACCOUNT;
		} else if (vm_flags & VM_WRITE) {
643 644 645
			/*
			 * Private writable mapping: check memory availability
			 */
646
			charged = len >> PAGE_SHIFT;
647
			if (security_vm_enough_memory(charged))
648 649 650
				return -ENOMEM;
			vm_flags |= VM_ACCOUNT;
		}
Andrew Morton's avatar
Andrew Morton committed
651
	}
Linus Torvalds's avatar
Linus Torvalds committed
652 653

	/* Can we just expand an old anonymous mapping? */
Linus Torvalds's avatar
Linus Torvalds committed
654
	if (!file && !(vm_flags & VM_SHARED) && rb_parent)
655
		if (vma_merge(mm, prev, rb_parent, addr, addr + len,
656
					vm_flags, NULL, pgoff, NULL))
Linus Torvalds's avatar
Linus Torvalds committed
657
			goto out;
Linus Torvalds's avatar
Linus Torvalds committed
658

659 660
	/*
	 * Determine the object being mapped and call the appropriate
Linus Torvalds's avatar
Linus Torvalds committed
661 662 663 664
	 * specific mapper. the address has already been validated, but
	 * not unmapped, but the maps are removed from the list.
	 */
	vma = kmem_cache_alloc(vm_area_cachep, SLAB_KERNEL);
Andrew Morton's avatar
Andrew Morton committed
665
	error = -ENOMEM;
Linus Torvalds's avatar
Linus Torvalds committed
666
	if (!vma)
Andrew Morton's avatar
Andrew Morton committed
667
		goto unacct_error;
Linus Torvalds's avatar
Linus Torvalds committed
668 669 670 671

	vma->vm_mm = mm;
	vma->vm_start = addr;
	vma->vm_end = addr + len;
Linus Torvalds's avatar
Linus Torvalds committed
672 673
	vma->vm_flags = vm_flags;
	vma->vm_page_prot = protection_map[vm_flags & 0x0f];
Linus Torvalds's avatar
Linus Torvalds committed
674 675 676 677
	vma->vm_ops = NULL;
	vma->vm_pgoff = pgoff;
	vma->vm_file = NULL;
	vma->vm_private_data = NULL;
678
	vma->vm_next = NULL;
679
	mpol_set_vma_default(vma);
680
	INIT_LIST_HEAD(&vma->shared);
Linus Torvalds's avatar
Linus Torvalds committed
681 682

	if (file) {
Linus Torvalds's avatar
Linus Torvalds committed
683 684 685
		error = -EINVAL;
		if (vm_flags & (VM_GROWSDOWN|VM_GROWSUP))
			goto free_vma;
Linus Torvalds's avatar
Linus Torvalds committed
686
		if (vm_flags & VM_DENYWRITE) {
Linus Torvalds's avatar
Linus Torvalds committed
687 688 689 690 691 692 693 694 695 696
			error = deny_write_access(file);
			if (error)
				goto free_vma;
			correct_wcount = 1;
		}
		vma->vm_file = file;
		get_file(file);
		error = file->f_op->mmap(file, vma);
		if (error)
			goto unmap_and_free_vma;
697
	} else if (vm_flags & VM_SHARED) {
Linus Torvalds's avatar
Linus Torvalds committed
698 699 700 701 702
		error = shmem_zero_setup(vma);
		if (error)
			goto free_vma;
	}

703 704 705 706 707 708 709 710
	/* We set VM_ACCOUNT in a shared mapping's vm_flags, to inform
	 * shmem_zero_setup (perhaps called through /dev/zero's ->mmap)
	 * that memory reservation must be checked; but that reservation
	 * belongs to shared memory object, not to vma: so now clear it.
	 */
	if ((vm_flags & (VM_SHARED|VM_ACCOUNT)) == (VM_SHARED|VM_ACCOUNT))
		vma->vm_flags &= ~VM_ACCOUNT;

Linus Torvalds's avatar
Linus Torvalds committed
711 712 713 714 715 716 717
	/* Can addr have changed??
	 *
	 * Answer: Yes, several device drivers can do it in their
	 *         f_op->mmap method. -DaveM
	 */
	addr = vma->vm_start;

718
	if (!file || !rb_parent || !vma_merge(mm, prev, rb_parent, addr,
719 720 721
					      vma->vm_end,
					      vma->vm_flags, file, pgoff,
					      vma_policy(vma))) {
722 723 724 725 726 727 728 729 730
		vma_link(mm, vma, prev, rb_link, rb_parent);
		if (correct_wcount)
			atomic_inc(&inode->i_writecount);
	} else {
		if (file) {
			if (correct_wcount)
				atomic_inc(&inode->i_writecount);
			fput(file);
		}
731
		mpol_free(vma_policy(vma));
732 733
		kmem_cache_free(vm_area_cachep, vma);
	}
Linus Torvalds's avatar
Linus Torvalds committed
734
out:	
Linus Torvalds's avatar
Linus Torvalds committed
735
	mm->total_vm += len >> PAGE_SHIFT;
Linus Torvalds's avatar
Linus Torvalds committed
736
	if (vm_flags & VM_LOCKED) {
Linus Torvalds's avatar
Linus Torvalds committed
737 738 739
		mm->locked_vm += len >> PAGE_SHIFT;
		make_pages_present(addr, addr + len);
	}
Andrew Morton's avatar
Andrew Morton committed
740 741 742 743 744 745
	if (flags & MAP_POPULATE) {
		up_write(&mm->mmap_sem);
		sys_remap_file_pages(addr, len, prot,
					pgoff, flags & MAP_NONBLOCK);
		down_write(&mm->mmap_sem);
	}
Linus Torvalds's avatar
Linus Torvalds committed
746 747 748 749
	return addr;

unmap_and_free_vma:
	if (correct_wcount)
750
		atomic_inc(&inode->i_writecount);
Linus Torvalds's avatar
Linus Torvalds committed
751 752
	vma->vm_file = NULL;
	fput(file);
Linus Torvalds's avatar
Linus Torvalds committed
753

Linus Torvalds's avatar
Linus Torvalds committed
754
	/* Undo any partial mapping done by a device driver. */
755
	zap_page_range(vma, vma->vm_start, vma->vm_end - vma->vm_start, NULL);
Linus Torvalds's avatar
Linus Torvalds committed
756 757
free_vma:
	kmem_cache_free(vm_area_cachep, vma);
Andrew Morton's avatar
Andrew Morton committed
758 759 760
unacct_error:
	if (charged)
		vm_unacct_memory(charged);
Linus Torvalds's avatar
Linus Torvalds committed
761 762 763
	return error;
}

764 765
EXPORT_SYMBOL(do_mmap_pgoff);

Linus Torvalds's avatar
Linus Torvalds committed
766
/* Get an address range which is currently unmapped.
Linus Torvalds's avatar
Linus Torvalds committed
767 768 769 770 771 772 773 774 775
 * For shmat() with addr=0.
 *
 * Ugly calling convention alert:
 * Return value with the low bits set means error value,
 * ie
 *	if (ret & ~PAGE_MASK)
 *		error = ret;
 *
 * This function "knows" that -ENOMEM has the bits set.
Linus Torvalds's avatar
Linus Torvalds committed
776 777
 */
#ifndef HAVE_ARCH_UNMAPPED_AREA
778 779 780
static inline unsigned long
arch_get_unmapped_area(struct file *filp, unsigned long addr,
		unsigned long len, unsigned long pgoff, unsigned long flags)
Linus Torvalds's avatar
Linus Torvalds committed
781
{
782
	struct mm_struct *mm = current->mm;
Linus Torvalds's avatar
Linus Torvalds committed
783
	struct vm_area_struct *vma;
784
	unsigned long start_addr;
Linus Torvalds's avatar
Linus Torvalds committed
785 786

	if (len > TASK_SIZE)
Linus Torvalds's avatar
Linus Torvalds committed
787
		return -ENOMEM;
Linus Torvalds's avatar
Linus Torvalds committed
788 789 790

	if (addr) {
		addr = PAGE_ALIGN(addr);
791
		vma = find_vma(mm, addr);
Linus Torvalds's avatar
Linus Torvalds committed
792 793 794
		if (TASK_SIZE - len >= addr &&
		    (!vma || addr + len <= vma->vm_start))
			return addr;
795 796
	}
	start_addr = addr = mm->free_area_cache;
Linus Torvalds's avatar
Linus Torvalds committed
797

798
full_search:
799
	for (vma = find_vma(mm, addr); ; vma = vma->vm_next) {
Linus Torvalds's avatar
Linus Torvalds committed
800
		/* At this point:  (!vma || addr < vma->vm_end). */
801 802 803 804 805 806 807 808 809
		if (TASK_SIZE - len < addr) {
			/*
			 * Start a new search - just in case we missed
			 * some holes.
			 */
			if (start_addr != TASK_UNMAPPED_BASE) {
				start_addr = addr = TASK_UNMAPPED_BASE;
				goto full_search;
			}
Linus Torvalds's avatar
Linus Torvalds committed
810
			return -ENOMEM;
811
		}
812 813 814 815 816
		if (!vma || addr + len <= vma->vm_start) {
			/*
			 * Remember the place where we stopped the search:
			 */
			mm->free_area_cache = addr + len;
Linus Torvalds's avatar
Linus Torvalds committed
817
			return addr;
818
		}
Linus Torvalds's avatar
Linus Torvalds committed
819
		addr = vma->vm_end;
Linus Torvalds's avatar
Linus Torvalds committed
820 821
	}
}
Linus Torvalds's avatar
Linus Torvalds committed
822
#else
823 824 825
extern unsigned long
arch_get_unmapped_area(struct file *, unsigned long, unsigned long,
			unsigned long, unsigned long);
Linus Torvalds's avatar
Linus Torvalds committed
826 827
#endif	

828 829 830
unsigned long
get_unmapped_area(struct file *file, unsigned long addr, unsigned long len,
		unsigned long pgoff, unsigned long flags)
Linus Torvalds's avatar
Linus Torvalds committed
831 832
{
	if (flags & MAP_FIXED) {
833 834
		unsigned long ret;

Linus Torvalds's avatar
Linus Torvalds committed
835
		if (addr > TASK_SIZE - len)
836
			return -ENOMEM;
Linus Torvalds's avatar
Linus Torvalds committed
837 838
		if (addr & ~PAGE_MASK)
			return -EINVAL;
839 840
		if (file && is_file_hugepages(file))  {
			/*
841 842
			 * Check if the given range is hugepage aligned, and
			 * can be made suitable for hugepages.
843
			 */
844
			ret = prepare_hugepage_range(addr, len);
845 846 847 848 849 850
		} else {
			/*
			 * Ensure that a normal request is not falling in a
			 * reserved hugepage range.  For some archs like IA-64,
			 * there is a separate region for hugepages.
			 */
851
			ret = is_hugepage_only_range(addr, len);
852
		}
853
		if (ret)
854
			return -EINVAL;
Linus Torvalds's avatar
Linus Torvalds committed
855 856 857 858
		return addr;
	}

	if (file && file->f_op && file->f_op->get_unmapped_area)
859 860
		return file->f_op->get_unmapped_area(file, addr, len,
						pgoff, flags);
Linus Torvalds's avatar
Linus Torvalds committed
861 862 863

	return arch_get_unmapped_area(file, addr, len, pgoff, flags);
}
Linus Torvalds's avatar
Linus Torvalds committed
864

865 866
EXPORT_SYMBOL(get_unmapped_area);

Linus Torvalds's avatar
Linus Torvalds committed
867 868 869 870 871 872 873 874 875 876
/* Look up the first VMA which satisfies  addr < vm_end,  NULL if none. */
struct vm_area_struct * find_vma(struct mm_struct * mm, unsigned long addr)
{
	struct vm_area_struct *vma = NULL;

	if (mm) {
		/* Check the cache first. */
		/* (Cache hit rate is typically around 35%.) */
		vma = mm->mmap_cache;
		if (!(vma && vma->vm_end > addr && vma->vm_start <= addr)) {
877
			struct rb_node * rb_node;
Linus Torvalds's avatar
Linus Torvalds committed
878 879 880 881 882 883 884

			rb_node = mm->mm_rb.rb_node;
			vma = NULL;

			while (rb_node) {
				struct vm_area_struct * vma_tmp;

885 886
				vma_tmp = rb_entry(rb_node,
						struct vm_area_struct, vm_rb);
Linus Torvalds's avatar
Linus Torvalds committed
887 888 889 890

				if (vma_tmp->vm_end > addr) {
					vma = vma_tmp;
					if (vma_tmp->vm_start <= addr)
Linus Torvalds's avatar
Linus Torvalds committed
891
						break;
Linus Torvalds's avatar
Linus Torvalds committed
892 893 894
					rb_node = rb_node->rb_left;
				} else
					rb_node = rb_node->rb_right;
Linus Torvalds's avatar
Linus Torvalds committed
895 896 897 898 899 900 901 902
			}
			if (vma)
				mm->mmap_cache = vma;
		}
	}
	return vma;
}

903 904
EXPORT_SYMBOL(find_vma);

Linus Torvalds's avatar
Linus Torvalds committed
905
/* Same as find_vma, but also return a pointer to the previous VMA in *pprev. */
906 907 908
struct vm_area_struct *
find_vma_prev(struct mm_struct *mm, unsigned long addr,
			struct vm_area_struct **pprev)
Linus Torvalds's avatar
Linus Torvalds committed
909
{
910
	struct vm_area_struct *vma = NULL, *prev = NULL;
911
	struct rb_node * rb_node;
912 913 914 915 916 917 918 919 920 921 922 923 924 925 926 927 928 929 930 931
	if (!mm)
		goto out;

	/* Guard against addr being lower than the first VMA */
	vma = mm->mmap;

	/* Go through the RB tree quickly. */
	rb_node = mm->mm_rb.rb_node;

	while (rb_node) {
		struct vm_area_struct *vma_tmp;
		vma_tmp = rb_entry(rb_node, struct vm_area_struct, vm_rb);

		if (addr < vma_tmp->vm_end) {
			rb_node = rb_node->rb_left;
		} else {
			prev = vma_tmp;
			if (!prev->vm_next || (addr < prev->vm_next->vm_end))
				break;
			rb_node = rb_node->rb_right;
Linus Torvalds's avatar
Linus Torvalds committed
932 933
		}
	}
934

935
out:
936 937
	*pprev = prev;
	return prev ? prev->vm_next : vma;
Linus Torvalds's avatar
Linus Torvalds committed
938 939
}

940
#ifdef CONFIG_STACK_GROWSUP
Andrew Morton's avatar
Andrew Morton committed
941
/*
942
 * vma is the first one with address > vma->vm_end.  Have to extend vma.
Andrew Morton's avatar
Andrew Morton committed
943 944 945 946 947
 */
int expand_stack(struct vm_area_struct * vma, unsigned long address)
{
	unsigned long grow;

948 949 950
	if (!(vma->vm_flags & VM_GROWSUP))
		return -EFAULT;

Andrew Morton's avatar
Andrew Morton committed
951 952
	/*
	 * vma->vm_start/vm_end cannot change under us because the caller
953
	 * is required to hold the mmap_sem in read mode. We need to get
Andrew Morton's avatar
Andrew Morton committed
954 955
	 * the spinlock only before relocating the vma range ourself.
	 */
956
	address += 4 + PAGE_SIZE - 1;
Andrew Morton's avatar
Andrew Morton committed
957 958
	address &= PAGE_MASK;
 	spin_lock(&vma->vm_mm->page_table_lock);
959
	grow = (address - vma->vm_end) >> PAGE_SHIFT;
Andrew Morton's avatar
Andrew Morton committed
960 961

	/* Overcommit.. */
962
	if (security_vm_enough_memory(grow)) {
Andrew Morton's avatar
Andrew Morton committed
963 964 965 966
		spin_unlock(&vma->vm_mm->page_table_lock);
		return -ENOMEM;
	}
	
967
	if (address - vma->vm_start > current->rlim[RLIMIT_STACK].rlim_cur ||
Andrew Morton's avatar
Andrew Morton committed
968 969 970 971 972 973
			((vma->vm_mm->total_vm + grow) << PAGE_SHIFT) >
			current->rlim[RLIMIT_AS].rlim_cur) {
		spin_unlock(&vma->vm_mm->page_table_lock);
		vm_unacct_memory(grow);
		return -ENOMEM;
	}
974
	vma->vm_end = address;
Andrew Morton's avatar
Andrew Morton committed
975 976 977 978 979 980 981
	vma->vm_mm->total_vm += grow;
	if (vma->vm_flags & VM_LOCKED)
		vma->vm_mm->locked_vm += grow;
	spin_unlock(&vma->vm_mm->page_table_lock);
	return 0;
}

982 983
struct vm_area_struct *
find_extend_vma(struct mm_struct *mm, unsigned long addr)
984 985 986 987 988 989 990 991 992 993 994 995 996 997 998
{
	struct vm_area_struct *vma, *prev;

	addr &= PAGE_MASK;
	vma = find_vma_prev(mm, addr, &prev);
	if (vma && (vma->vm_start <= addr))
		return vma;
	if (!prev || expand_stack(prev, addr))
		return NULL;
	if (prev->vm_flags & VM_LOCKED) {
		make_pages_present(addr, prev->vm_end);
	}
	return prev;
}
#else
999 1000 1001
/*
 * vma is the first one with address < vma->vm_start.  Have to extend vma.
 */
1002
int expand_stack(struct vm_area_struct *vma, unsigned long address)
1003 1004 1005 1006 1007 1008 1009 1010 1011 1012 1013 1014 1015
{
	unsigned long grow;

	/*
	 * vma->vm_start/vm_end cannot change under us because the caller
	 * is required to hold the mmap_sem in read mode. We need to get
	 * the spinlock only before relocating the vma range ourself.
	 */
	address &= PAGE_MASK;
 	spin_lock(&vma->vm_mm->page_table_lock);
	grow = (vma->vm_start - address) >> PAGE_SHIFT;

	/* Overcommit.. */
1016
	if (security_vm_enough_memory(grow)) {
1017 1018 1019 1020 1021 1022 1023 1024 1025 1026 1027 1028 1029 1030 1031 1032 1033 1034 1035 1036
		spin_unlock(&vma->vm_mm->page_table_lock);
		return -ENOMEM;
	}
	
	if (vma->vm_end - address > current->rlim[RLIMIT_STACK].rlim_cur ||
			((vma->vm_mm->total_vm + grow) << PAGE_SHIFT) >
			current->rlim[RLIMIT_AS].rlim_cur) {
		spin_unlock(&vma->vm_mm->page_table_lock);
		vm_unacct_memory(grow);
		return -ENOMEM;
	}
	vma->vm_start = address;
	vma->vm_pgoff -= grow;
	vma->vm_mm->total_vm += grow;
	if (vma->vm_flags & VM_LOCKED)
		vma->vm_mm->locked_vm += grow;
	spin_unlock(&vma->vm_mm->page_table_lock);
	return 0;
}

1037 1038
struct vm_area_struct *
find_extend_vma(struct mm_struct * mm, unsigned long addr)
Linus Torvalds's avatar
Linus Torvalds committed
1039 1040 1041 1042 1043 1044 1045 1046 1047 1048 1049 1050 1051 1052 1053 1054 1055 1056 1057 1058
{
	struct vm_area_struct * vma;
	unsigned long start;

	addr &= PAGE_MASK;
	vma = find_vma(mm,addr);
	if (!vma)
		return NULL;
	if (vma->vm_start <= addr)
		return vma;
	if (!(vma->vm_flags & VM_GROWSDOWN))
		return NULL;
	start = vma->vm_start;
	if (expand_stack(vma, addr))
		return NULL;
	if (vma->vm_flags & VM_LOCKED) {
		make_pages_present(addr, start);
	}
	return vma;
}
1059
#endif
Linus Torvalds's avatar
Linus Torvalds committed
1060 1061 1062 1063 1064 1065 1066 1067 1068 1069 1070 1071 1072 1073

/*
 * Try to free as many page directory entries as we can,
 * without having to work very hard at actually scanning
 * the page tables themselves.
 *
 * Right now we try to free page tables if we have a nice
 * PGDIR-aligned area that got free'd up. We could be more
 * granular if we want to, but this is fast and simple,
 * and covers the bad cases.
 *
 * "prev", if it exists, points to a vma before the one
 * we just free'd - but there's no telling how much before.
 */
1074
static void free_pgtables(struct mmu_gather *tlb, struct vm_area_struct *prev,
Linus Torvalds's avatar
Linus Torvalds committed
1075 1076 1077 1078 1079
	unsigned long start, unsigned long end)
{
	unsigned long first = start & PGDIR_MASK;
	unsigned long last = end + PGDIR_SIZE - 1;
	unsigned long start_index, end_index;
1080
	struct mm_struct *mm = tlb->mm;
Linus Torvalds's avatar
Linus Torvalds committed
1081 1082 1083 1084 1085 1086 1087 1088 1089 1090 1091 1092 1093 1094 1095 1096 1097 1098 1099 1100 1101 1102 1103 1104 1105 1106 1107

	if (!prev) {
		prev = mm->mmap;
		if (!prev)
			goto no_mmaps;
		if (prev->vm_end > start) {
			if (last > prev->vm_start)
				last = prev->vm_start;
			goto no_mmaps;
		}
	}
	for (;;) {
		struct vm_area_struct *next = prev->vm_next;

		if (next) {
			if (next->vm_start < start) {
				prev = next;
				continue;
			}
			if (last > next->vm_start)
				last = next->vm_start;
		}
		if (prev->vm_end > first)
			first = prev->vm_end + PGDIR_SIZE - 1;
		break;
	}
no_mmaps:
1108
	if (last < first)	/* for arches with discontiguous pgd indices */
1109
		return;
Linus Torvalds's avatar
Linus Torvalds committed
1110 1111 1112 1113 1114
	/*
	 * If the PGD bits are not consecutive in the virtual address, the
	 * old method of shifting the VA >> by PGDIR_SHIFT doesn't work.
	 */
	start_index = pgd_index(first);
Russell King's avatar
Russell King committed
1115 1116
	if (start_index < FIRST_USER_PGD_NR)
		start_index = FIRST_USER_PGD_NR;
Linus Torvalds's avatar
Linus Torvalds committed
1117 1118
	end_index = pgd_index(last);
	if (end_index > start_index) {
1119
		clear_page_tables(tlb, start_index, end_index - start_index);
Linus Torvalds's avatar
Linus Torvalds committed
1120 1121 1122 1123
		flush_tlb_pgtables(mm, first & PGDIR_MASK, last & PGDIR_MASK);
	}
}

1124 1125 1126 1127 1128 1129
/* Normal function to fix up a mapping
 * This function is the default for when an area has no specific
 * function.  This may be used as part of a more specific routine.
 *
 * By the time this function is called, the area struct has been
 * removed from the process mapping list.
Linus Torvalds's avatar
Linus Torvalds committed
1130
 */
1131
static void unmap_vma(struct mm_struct *mm, struct vm_area_struct *area)
1132 1133 1134 1135 1136 1137
{
	size_t len = area->vm_end - area->vm_start;

	area->vm_mm->total_vm -= len >> PAGE_SHIFT;
	if (area->vm_flags & VM_LOCKED)
		area->vm_mm->locked_vm -= len >> PAGE_SHIFT;
1138 1139 1140 1141 1142 1143
	/*
	 * Is this a new hole at the lowest possible address?
	 */
	if (area->vm_start >= TASK_UNMAPPED_BASE &&
				area->vm_start < area->vm_mm->free_area_cache)
	      area->vm_mm->free_area_cache = area->vm_start;
1144 1145 1146

	remove_shared_vm_struct(area);

1147
	mpol_free(vma_policy(area));
1148 1149 1150 1151 1152 1153 1154 1155 1156 1157 1158 1159 1160 1161 1162 1163 1164 1165 1166 1167 1168 1169 1170 1171 1172 1173 1174 1175 1176 1177
	if (area->vm_ops && area->vm_ops->close)
		area->vm_ops->close(area);
	if (area->vm_file)
		fput(area->vm_file);
	kmem_cache_free(vm_area_cachep, area);
}

/*
 * Update the VMA and inode share lists.
 *
 * Ok - we have the memory areas we should free on the 'free' list,
 * so release them, and do the vma updates.
 */
static void unmap_vma_list(struct mm_struct *mm,
	struct vm_area_struct *mpnt)
{
	do {
		struct vm_area_struct *next = mpnt->vm_next;
		unmap_vma(mm, mpnt);
		mpnt = next;
	} while (mpnt != NULL);
	validate_mm(mm);
}

/*
 * Get rid of page table information in the indicated region.
 *
 * Called with the page table lock held.
 */
static void unmap_region(struct mm_struct *mm,
1178
	struct vm_area_struct *vma,
1179 1180
	struct vm_area_struct *prev,
	unsigned long start,
1181
	unsigned long end)
Linus Torvalds's avatar
Linus Torvalds committed
1182
{
1183
	struct mmu_gather *tlb;
1184
	unsigned long nr_accounted = 0;
Linus Torvalds's avatar
Linus Torvalds committed
1185

1186
	lru_add_drain();
1187
	tlb = tlb_gather_mmu(mm, 0);
1188
	unmap_vmas(&tlb, mm, vma, start, end, &nr_accounted, NULL);
1189
	vm_unacct_memory(nr_accounted);
1190 1191 1192 1193 1194

	if (is_hugepage_only_range(start, end - start))
		hugetlb_free_pgtables(tlb, prev, start, end);
	else
		free_pgtables(tlb, prev, start, end);
1195 1196
	tlb_finish_mmu(tlb, start, end);
}
Linus Torvalds's avatar
Linus Torvalds committed
1197

1198
/*
1199 1200
 * Create a list of vma's touched by the unmap, removing them from the mm's
 * vma list as we go..
1201 1202 1203
 *
 * Called with the page_table_lock held.
 */
1204 1205 1206
static void
detach_vmas_to_be_unmapped(struct mm_struct *mm, struct vm_area_struct *vma,
	struct vm_area_struct *prev, unsigned long end)
1207
{
1208 1209
	struct vm_area_struct **insertion_point;
	struct vm_area_struct *tail_vma = NULL;
Linus Torvalds's avatar
Linus Torvalds committed
1210

1211
	insertion_point = (prev ? &prev->vm_next : &mm->mmap);
1212
	do {
1213
		rb_erase(&vma->vm_rb, &mm->mm_rb);
1214
		mm->map_count--;
1215 1216 1217 1218 1219 1220
		tail_vma = vma;
		vma = vma->vm_next;
	} while (vma && vma->vm_start < end);
	*insertion_point = vma;
	tail_vma->vm_next = NULL;
	mm->mmap_cache = NULL;		/* Kill the cache. */
1221
}
1222

1223
/*
1224 1225
 * Split a vma into two pieces at address 'addr', a new vma is allocated
 * either for the first part or the the tail.
1226
 */
1227 1228
int split_vma(struct mm_struct * mm, struct vm_area_struct * vma,
	      unsigned long addr, int new_below)
1229
{
1230
	struct mempolicy *pol;
1231
	struct vm_area_struct *new;
Linus Torvalds's avatar
Linus Torvalds committed
1232

1233
	if (mm->map_count >= sysctl_max_map_count)
1234
		return -ENOMEM;
Linus Torvalds's avatar
Linus Torvalds committed
1235

1236 1237 1238
	new = kmem_cache_alloc(vm_area_cachep, SLAB_KERNEL);
	if (!new)
		return -ENOMEM;
Linus Torvalds's avatar
Linus Torvalds committed
1239

1240
	/* most fields are the same, copy all, and then fixup */
1241 1242
	*new = *vma;

1243 1244
	INIT_LIST_HEAD(&new->shared);

1245
	if (new_below)
1246
		new->vm_end = addr;
1247
	else {
1248 1249 1250
		new->vm_start = addr;
		new->vm_pgoff += ((addr - vma->vm_start) >> PAGE_SHIFT);
	}
Linus Torvalds's avatar
Linus Torvalds committed
1251

1252 1253 1254 1255 1256 1257 1258
	pol = mpol_copy(vma_policy(vma));
	if (IS_ERR(pol)) {
		kmem_cache_free(vm_area_cachep, new);
		return PTR_ERR(pol);
	}
	vma_set_policy(new, pol);

1259 1260 1261 1262 1263
	if (new->vm_file)
		get_file(new->vm_file);

	if (new->vm_ops && new->vm_ops->open)
		new->vm_ops->open(new);
1264

Andrew Morton's avatar
Andrew Morton committed
1265 1266 1267 1268 1269
	if (new_below)
		vma_adjust(vma, addr, vma->vm_end, vma->vm_pgoff +
			((addr - new->vm_start) >> PAGE_SHIFT), new);
	else
		vma_adjust(vma, vma->vm_start, addr, vma->vm_pgoff, new);
1270

1271 1272
	return 0;
}
Linus Torvalds's avatar
Linus Torvalds committed
1273

1274 1275 1276
/* Munmap is split into 2 main parts -- this part which finds
 * what needs doing, and the areas themselves, which do the
 * work.  This now handles partial unmappings.
Andrew Morton's avatar
Andrew Morton committed
1277
 * Jeremy Fitzhardinge <jeremy@goop.org>
1278
 */
1279
int do_munmap(struct mm_struct *mm, unsigned long start, size_t len)
1280 1281 1282 1283 1284 1285 1286 1287 1288 1289 1290 1291 1292 1293 1294 1295
{
	unsigned long end;
	struct vm_area_struct *mpnt, *prev, *last;

	if ((start & ~PAGE_MASK) || start > TASK_SIZE || len > TASK_SIZE-start)
		return -EINVAL;

	if ((len = PAGE_ALIGN(len)) == 0)
		return -EINVAL;

	/* Find the first overlapping VMA */
	mpnt = find_vma_prev(mm, start, &prev);
	if (!mpnt)
		return 0;
	/* we have  start < mpnt->vm_end  */

Andrew Morton's avatar
Andrew Morton committed
1296
	if (is_vm_hugetlb_page(mpnt)) {
1297 1298 1299 1300
		int ret = is_aligned_hugepage_range(start, len);

		if (ret)
			return ret;
Andrew Morton's avatar
Andrew Morton committed
1301 1302
	}

1303 1304 1305 1306 1307
	/* if it doesn't overlap, we have nothing.. */
	end = start + len;
	if (mpnt->vm_start >= end)
		return 0;

John Levon's avatar
John Levon committed
1308 1309 1310 1311
	/* Something will probably happen, so notify. */
	if (mpnt->vm_file && (mpnt->vm_flags & VM_EXEC))
		profile_exec_unmap(mm);
 
1312 1313
	/*
	 * If we need to split any vma, do it now to save pain later.
1314 1315 1316 1317
	 *
	 * Note: mremap's move_vma VM_ACCOUNT handling assumes a partially
	 * unmapped vm_area_struct will remain in use: so lower split_vma
	 * places tmp vma above, and higher split_vma places tmp vma below.
1318 1319
	 */
	if (start > mpnt->vm_start) {
1320
		if (split_vma(mm, mpnt, start, 0))
1321 1322
			return -ENOMEM;
		prev = mpnt;
Linus Torvalds's avatar
Linus Torvalds committed
1323 1324
	}

1325 1326 1327
	/* Does it split the last one? */
	last = find_vma(mm, end);
	if (last && end > last->vm_start) {
1328
		if (split_vma(mm, last, end, 1))
1329 1330
			return -ENOMEM;
	}
1331
	mpnt = prev? prev->vm_next: mm->mmap;
Linus Torvalds's avatar
Linus Torvalds committed
1332

1333 1334 1335 1336
	/*
	 * Remove the vma's, and unmap the actual pages
	 */
	spin_lock(&mm->page_table_lock);
1337
	detach_vmas_to_be_unmapped(mm, mpnt, prev, end);
1338
	unmap_region(mm, mpnt, prev, start, end);
1339
	spin_unlock(&mm->page_table_lock);
Linus Torvalds's avatar
Linus Torvalds committed
1340

1341 1342 1343
	/* Fix up all other VM information */
	unmap_vma_list(mm, mpnt);

Linus Torvalds's avatar
Linus Torvalds committed
1344 1345 1346
	return 0;
}

1347 1348
EXPORT_SYMBOL(do_munmap);

Linus Torvalds's avatar
Linus Torvalds committed
1349 1350 1351 1352 1353
asmlinkage long sys_munmap(unsigned long addr, size_t len)
{
	int ret;
	struct mm_struct *mm = current->mm;

Linus Torvalds's avatar
Linus Torvalds committed
1354
	down_write(&mm->mmap_sem);
1355
	ret = do_munmap(mm, addr, len);
Linus Torvalds's avatar
Linus Torvalds committed
1356
	up_write(&mm->mmap_sem);
Linus Torvalds's avatar
Linus Torvalds committed
1357 1358 1359 1360 1361 1362 1363 1364 1365 1366 1367
	return ret;
}

/*
 *  this is really a simplified "do_mmap".  it only handles
 *  anonymous maps.  eventually we may be able to do some
 *  brk-specific accounting here.
 */
unsigned long do_brk(unsigned long addr, unsigned long len)
{
	struct mm_struct * mm = current->mm;
Linus Torvalds's avatar
Linus Torvalds committed
1368 1369
	struct vm_area_struct * vma, * prev;
	unsigned long flags;
1370
	struct rb_node ** rb_link, * rb_parent;
Linus Torvalds's avatar
Linus Torvalds committed
1371 1372 1373 1374 1375

	len = PAGE_ALIGN(len);
	if (!len)
		return addr;

1376 1377 1378
	if ((addr + len) > TASK_SIZE || (addr + len) < addr)
		return -EINVAL;

Linus Torvalds's avatar
Linus Torvalds committed
1379 1380 1381 1382 1383 1384 1385 1386 1387 1388 1389 1390 1391
	/*
	 * mlock MCL_FUTURE?
	 */
	if (mm->def_flags & VM_LOCKED) {
		unsigned long locked = mm->locked_vm << PAGE_SHIFT;
		locked += len;
		if (locked > current->rlim[RLIMIT_MEMLOCK].rlim_cur)
			return -EAGAIN;
	}

	/*
	 * Clear old maps.  this also does some error checking for us
	 */
Linus Torvalds's avatar
Linus Torvalds committed
1392 1393 1394
 munmap_back:
	vma = find_vma_prepare(mm, addr, &prev, &rb_link, &rb_parent);
	if (vma && vma->vm_start < addr + len) {
1395
		if (do_munmap(mm, addr, len))
Linus Torvalds's avatar
Linus Torvalds committed
1396 1397 1398
			return -ENOMEM;
		goto munmap_back;
	}
Linus Torvalds's avatar
Linus Torvalds committed
1399 1400 1401 1402 1403 1404

	/* Check against address space limits *after* clearing old maps... */
	if ((mm->total_vm << PAGE_SHIFT) + len
	    > current->rlim[RLIMIT_AS].rlim_cur)
		return -ENOMEM;

1405
	if (mm->map_count > sysctl_max_map_count)
Linus Torvalds's avatar
Linus Torvalds committed
1406 1407
		return -ENOMEM;

1408
	if (security_vm_enough_memory(len >> PAGE_SHIFT))
Linus Torvalds's avatar
Linus Torvalds committed
1409 1410
		return -ENOMEM;

Andrew Morton's avatar
Andrew Morton committed
1411
	flags = VM_DATA_DEFAULT_FLAGS | VM_ACCOUNT | mm->def_flags;
Linus Torvalds's avatar
Linus Torvalds committed
1412

Linus Torvalds's avatar
Linus Torvalds committed
1413
	/* Can we just expand an old anonymous mapping? */
1414
	if (rb_parent && vma_merge(mm, prev, rb_parent, addr, addr + len,
1415
					flags, NULL, 0, NULL))
Linus Torvalds's avatar
Linus Torvalds committed
1416
		goto out;
Linus Torvalds's avatar
Linus Torvalds committed
1417 1418 1419 1420 1421

	/*
	 * create a vma struct for an anonymous mapping
	 */
	vma = kmem_cache_alloc(vm_area_cachep, SLAB_KERNEL);
Andrew Morton's avatar
Andrew Morton committed
1422 1423
	if (!vma) {
		vm_unacct_memory(len >> PAGE_SHIFT);
Linus Torvalds's avatar
Linus Torvalds committed
1424
		return -ENOMEM;
Andrew Morton's avatar
Andrew Morton committed
1425
	}
Linus Torvalds's avatar
Linus Torvalds committed
1426 1427 1428 1429 1430 1431 1432 1433 1434 1435

	vma->vm_mm = mm;
	vma->vm_start = addr;
	vma->vm_end = addr + len;
	vma->vm_flags = flags;
	vma->vm_page_prot = protection_map[flags & 0x0f];
	vma->vm_ops = NULL;
	vma->vm_pgoff = 0;
	vma->vm_file = NULL;
	vma->vm_private_data = NULL;
1436
	mpol_set_vma_default(vma);
1437
	INIT_LIST_HEAD(&vma->shared);
Linus Torvalds's avatar
Linus Torvalds committed
1438

Linus Torvalds's avatar
Linus Torvalds committed
1439
	vma_link(mm, vma, prev, rb_link, rb_parent);
Linus Torvalds's avatar
Linus Torvalds committed
1440 1441 1442 1443 1444 1445 1446 1447 1448 1449

out:
	mm->total_vm += len >> PAGE_SHIFT;
	if (flags & VM_LOCKED) {
		mm->locked_vm += len >> PAGE_SHIFT;
		make_pages_present(addr, addr + len);
	}
	return addr;
}

1450 1451
EXPORT_SYMBOL(do_brk);

Linus Torvalds's avatar
Linus Torvalds committed
1452
/* Release all mmaps. */
1453
void exit_mmap(struct mm_struct *mm)
Linus Torvalds's avatar
Linus Torvalds committed
1454
{
1455
	struct mmu_gather *tlb;
1456 1457
	struct vm_area_struct *vma;
	unsigned long nr_accounted = 0;
Linus Torvalds's avatar
Linus Torvalds committed
1458

John Levon's avatar
John Levon committed
1459 1460
	profile_exit_mmap(mm);
 
1461 1462
	lru_add_drain();

Linus Torvalds's avatar
Linus Torvalds committed
1463
	spin_lock(&mm->page_table_lock);
Linus Torvalds's avatar
Linus Torvalds committed
1464

1465
	tlb = tlb_gather_mmu(mm, 1);
Linus Torvalds's avatar
Linus Torvalds committed
1466
	flush_cache_mm(mm);
1467
	/* Use ~0UL here to ensure all VMAs in the mm are unmapped */
1468
	mm->map_count -= unmap_vmas(&tlb, mm, mm->mmap, 0,
1469
					~0UL, &nr_accounted, NULL);
1470 1471
	vm_unacct_memory(nr_accounted);
	BUG_ON(mm->map_count);	/* This is just debugging */
1472
	clear_page_tables(tlb, FIRST_USER_PGD_NR, USER_PTRS_PER_PGD);
1473
	tlb_finish_mmu(tlb, 0, MM_VM_SIZE(mm));
1474

1475
	vma = mm->mmap;
1476 1477 1478 1479 1480 1481
	mm->mmap = mm->mmap_cache = NULL;
	mm->mm_rb = RB_ROOT;
	mm->rss = 0;
	mm->total_vm = 0;
	mm->locked_vm = 0;

1482
	spin_unlock(&mm->page_table_lock);
1483 1484 1485 1486 1487

	/*
	 * Walk the list again, actually closing and freeing it
	 * without holding any MM locks.
	 */
1488 1489 1490 1491 1492 1493
	while (vma) {
		struct vm_area_struct *next = vma->vm_next;
		remove_shared_vm_struct(vma);
		if (vma->vm_ops) {
			if (vma->vm_ops->close)
				vma->vm_ops->close(vma);
1494
		}
1495 1496
		if (vma->vm_file)
			fput(vma->vm_file);
1497
		mpol_free(vma_policy(vma));
1498 1499
		kmem_cache_free(vm_area_cachep, vma);
		vma = next;
1500
	}
Linus Torvalds's avatar
Linus Torvalds committed
1501 1502 1503 1504
}

/* Insert vm structure into process list sorted by address
 * and into the inode's i_mmap ring.  If vm_file is non-NULL
1505
 * then i_mmap_lock is taken here.
Linus Torvalds's avatar
Linus Torvalds committed
1506
 */
Linus Torvalds's avatar
Linus Torvalds committed
1507
void insert_vm_struct(struct mm_struct * mm, struct vm_area_struct * vma)
Linus Torvalds's avatar
Linus Torvalds committed
1508
{
Linus Torvalds's avatar
Linus Torvalds committed
1509
	struct vm_area_struct * __vma, * prev;
1510
	struct rb_node ** rb_link, * rb_parent;
Linus Torvalds's avatar
Linus Torvalds committed
1511

1512
	__vma = find_vma_prepare(mm,vma->vm_start,&prev,&rb_link,&rb_parent);
Linus Torvalds's avatar
Linus Torvalds committed
1513 1514 1515
	if (__vma && __vma->vm_start < vma->vm_end)
		BUG();
	vma_link(mm, vma, prev, rb_link, rb_parent);
1516 1517 1518 1519 1520 1521
}

/*
 * Copy the vma structure to a new location in the same mm,
 * prior to moving page table entries, to effect an mremap move.
 */
Hugh Dickins's avatar
Hugh Dickins committed
1522
struct vm_area_struct *copy_vma(struct vm_area_struct **vmap,
1523 1524
	unsigned long addr, unsigned long len, unsigned long pgoff)
{
Hugh Dickins's avatar
Hugh Dickins committed
1525 1526
	struct vm_area_struct *vma = *vmap;
	unsigned long vma_start = vma->vm_start;
1527 1528 1529
	struct mm_struct *mm = vma->vm_mm;
	struct vm_area_struct *new_vma, *prev;
	struct rb_node **rb_link, *rb_parent;
Andrew Morton's avatar
Andrew Morton committed
1530
	struct mempolicy *pol;
1531 1532 1533

	find_vma_prepare(mm, addr, &prev, &rb_link, &rb_parent);
	new_vma = vma_merge(mm, prev, rb_parent, addr, addr + len,
1534
			vma->vm_flags, vma->vm_file, pgoff, vma_policy(vma));
Hugh Dickins's avatar
Hugh Dickins committed
1535 1536 1537 1538 1539 1540 1541 1542
	if (new_vma) {
		/*
		 * Source vma may have been merged into new_vma
		 */
		if (vma_start >= new_vma->vm_start &&
		    vma_start < new_vma->vm_end)
			*vmap = new_vma;
	} else {
1543 1544 1545
		new_vma = kmem_cache_alloc(vm_area_cachep, SLAB_KERNEL);
		if (new_vma) {
			*new_vma = *vma;
Andrew Morton's avatar
Andrew Morton committed
1546 1547 1548 1549 1550 1551
			pol = mpol_copy(vma_policy(vma));
			if (IS_ERR(pol)) {
				kmem_cache_free(vm_area_cachep, new_vma);
				return NULL;
			}
			vma_set_policy(new_vma, pol);
1552 1553 1554 1555 1556 1557 1558 1559 1560 1561 1562 1563
			INIT_LIST_HEAD(&new_vma->shared);
			new_vma->vm_start = addr;
			new_vma->vm_end = addr + len;
			new_vma->vm_pgoff = pgoff;
			if (new_vma->vm_file)
				get_file(new_vma->vm_file);
			if (new_vma->vm_ops && new_vma->vm_ops->open)
				new_vma->vm_ops->open(new_vma);
			vma_link(mm, new_vma, prev, rb_link, rb_parent);
		}
	}
	return new_vma;
Linus Torvalds's avatar
Linus Torvalds committed
1564
}