Kernel

Kernel Git Source Tree

Root/mm/mmap.c

1	/*
2	* mm/mmap.c
3	*
4	* Written by obz.
5	*
6	* Address space accounting code <alan@lxorguk.ukuu.org.uk>
7	*/
8
9	#include <linux/kernel.h>
10	#include <linux/slab.h>
11	#include <linux/backing-dev.h>
12	#include <linux/mm.h>
13	#include <linux/shm.h>
14	#include <linux/mman.h>
15	#include <linux/pagemap.h>
16	#include <linux/swap.h>
17	#include <linux/syscalls.h>
18	#include <linux/capability.h>
19	#include <linux/init.h>
20	#include <linux/file.h>
21	#include <linux/fs.h>
22	#include <linux/personality.h>
23	#include <linux/security.h>
24	#include <linux/hugetlb.h>
25	#include <linux/profile.h>
26	#include <linux/export.h>
27	#include <linux/mount.h>
28	#include <linux/mempolicy.h>
29	#include <linux/rmap.h>
30	#include <linux/mmu_notifier.h>
31	#include <linux/perf_event.h>
32	#include <linux/audit.h>
33	#include <linux/khugepaged.h>
34	#include <linux/uprobes.h>
35	#include <linux/rbtree_augmented.h>
36	#include <linux/sched/sysctl.h>
37	#include <linux/notifier.h>
38	#include <linux/memory.h>
39
40	#include <asm/uaccess.h>
41	#include <asm/cacheflush.h>
42	#include <asm/tlb.h>
43	#include <asm/mmu_context.h>
44
45	#include "internal.h"
46
47	#ifndef arch_mmap_check
48	#define arch_mmap_check(addr, len, flags) (0)
49	#endif
50
51	#ifndef arch_rebalance_pgtables
52	#define arch_rebalance_pgtables(addr, len) (addr)
53	#endif
54
55	static void unmap_region(struct mm_struct *mm,
56	struct vm_area_struct vma, struct vm_area_struct prev,
57	unsigned long start, unsigned long end);
58
59	/* description of effects of mapping type and prot in current implementation.
60	* this is due to the limited x86 page protection hardware. The expected
61	* behavior is in parens:
62	*
63	* map_type prot
64	* PROT_NONE PROT_READ PROT_WRITE PROT_EXEC
65	* MAP_SHARED r: (no) no r: (yes) yes r: (no) yes r: (no) yes
66	* w: (no) no w: (no) no w: (yes) yes w: (no) no
67	* x: (no) no x: (no) yes x: (no) yes x: (yes) yes
68	*
69	* MAP_PRIVATE r: (no) no r: (yes) yes r: (no) yes r: (no) yes
70	* w: (no) no w: (no) no w: (copy) copy w: (no) no
71	* x: (no) no x: (no) yes x: (no) yes x: (yes) yes
72	*
73	*/
74	pgprot_t protection_map[16] = {
75	__P000, __P001, __P010, __P011, __P100, __P101, __P110, __P111,
76	__S000, __S001, __S010, __S011, __S100, __S101, __S110, __S111
77	};
78
79	pgprot_t vm_get_page_prot(unsigned long vm_flags)
80	{
81	return __pgprot(pgprot_val(protection_map[vm_flags &
82	(VM_READ\|VM_WRITE\|VM_EXEC\|VM_SHARED)]) \|
83	pgprot_val(arch_vm_get_page_prot(vm_flags)));
84	}
85	EXPORT_SYMBOL(vm_get_page_prot);
86
87	int sysctl_overcommit_memory __read_mostly = OVERCOMMIT_GUESS; /* heuristic overcommit */
88	int sysctl_overcommit_ratio __read_mostly = 50; /* default is 50% */
89	int sysctl_max_map_count __read_mostly = DEFAULT_MAX_MAP_COUNT;
90	unsigned long sysctl_user_reserve_kbytes __read_mostly = 1UL << 17; /* 128MB */
91	unsigned long sysctl_admin_reserve_kbytes __read_mostly = 1UL << 13; /* 8MB */
92	/*
93	* Make sure vm_committed_as in one cacheline and not cacheline shared with
94	* other variables. It can be updated by several CPUs frequently.
95	*/
96	struct percpu_counter vm_committed_as ____cacheline_aligned_in_smp;
97
98	/*
99	* The global memory commitment made in the system can be a metric
100	* that can be used to drive ballooning decisions when Linux is hosted
101	* as a guest. On Hyper-V, the host implements a policy engine for dynamically
102	* balancing memory across competing virtual machines that are hosted.
103	* Several metrics drive this policy engine including the guest reported
104	* memory commitment.
105	*/
106	unsigned long vm_memory_committed(void)
107	{
108	return percpu_counter_read_positive(&vm_committed_as);
109	}
110	EXPORT_SYMBOL_GPL(vm_memory_committed);
111
112	/*
113	* Check that a process has enough memory to allocate a new virtual
114	* mapping. 0 means there is enough memory for the allocation to
115	* succeed and -ENOMEM implies there is not.
116	*
117	* We currently support three overcommit policies, which are set via the
118	* vm.overcommit_memory sysctl. See Documentation/vm/overcommit-accounting
119	*
120	* Strict overcommit modes added 2002 Feb 26 by Alan Cox.
121	* Additional code 2002 Jul 20 by Robert Love.
122	*
123	* cap_sys_admin is 1 if the process has admin privileges, 0 otherwise.
124	*
125	* Note this is a helper function intended to be used by LSMs which
126	* wish to use this logic.
127	*/
128	int __vm_enough_memory(struct mm_struct *mm, long pages, int cap_sys_admin)
129	{
130	unsigned long free, allowed, reserve;
131
132	vm_acct_memory(pages);
133
134	/*
135	* Sometimes we want to use more memory than we have
136	*/
137	if (sysctl_overcommit_memory == OVERCOMMIT_ALWAYS)
138	return 0;
139
140	if (sysctl_overcommit_memory == OVERCOMMIT_GUESS) {
141	free = global_page_state(NR_FREE_PAGES);
142	free += global_page_state(NR_FILE_PAGES);
143
144	/*
145	* shmem pages shouldn't be counted as free in this
146	* case, they can't be purged, only swapped out, and
147	* that won't affect the overall amount of available
148	* memory in the system.
149	*/
150	free -= global_page_state(NR_SHMEM);
151
152	free += get_nr_swap_pages();
153
154	/*
155	* Any slabs which are created with the
156	* SLAB_RECLAIM_ACCOUNT flag claim to have contents
157	* which are reclaimable, under pressure. The dentry
158	* cache and most inode caches should fall into this
159	*/
160	free += global_page_state(NR_SLAB_RECLAIMABLE);
161
162	/*
163	* Leave reserved pages. The pages are not for anonymous pages.
164	*/
165	if (free <= totalreserve_pages)
166	goto error;
167	else
168	free -= totalreserve_pages;
169
170	/*
171	* Reserve some for root
172	*/
173	if (!cap_sys_admin)
174	free -= sysctl_admin_reserve_kbytes >> (PAGE_SHIFT - 10);
175
176	if (free > pages)
177	return 0;
178
179	goto error;
180	}
181
182	allowed = (totalram_pages - hugetlb_total_pages())
183	* sysctl_overcommit_ratio / 100;
184	/*
185	* Reserve some for root
186	*/
187	if (!cap_sys_admin)
188	allowed -= sysctl_admin_reserve_kbytes >> (PAGE_SHIFT - 10);
189	allowed += total_swap_pages;
190
191	/*
192	* Don't let a single process grow so big a user can't recover
193	*/
194	if (mm) {
195	reserve = sysctl_user_reserve_kbytes >> (PAGE_SHIFT - 10);
196	allowed -= min(mm->total_vm / 32, reserve);
197	}
198
199	if (percpu_counter_read_positive(&vm_committed_as) < allowed)
200	return 0;
201	error:
202	vm_unacct_memory(pages);
203
204	return -ENOMEM;
205	}
206
207	/*
208	* Requires inode->i_mapping->i_mmap_mutex
209	*/
210	static void __remove_shared_vm_struct(struct vm_area_struct *vma,
211	struct file file, struct address_space mapping)
212	{
213	if (vma->vm_flags & VM_DENYWRITE)
214	atomic_inc(&file_inode(file)->i_writecount);
215	if (vma->vm_flags & VM_SHARED)
216	mapping->i_mmap_writable--;
217
218	flush_dcache_mmap_lock(mapping);
219	if (unlikely(vma->vm_flags & VM_NONLINEAR))
220	list_del_init(&vma->shared.nonlinear);
221	else
222	vma_interval_tree_remove(vma, &mapping->i_mmap);
223	flush_dcache_mmap_unlock(mapping);
224	}
225
226	/*
227	* Unlink a file-based vm structure from its interval tree, to hide
228	* vma from rmap and vmtruncate before freeing its page tables.
229	*/
230	void unlink_file_vma(struct vm_area_struct *vma)
231	{
232	struct file *file = vma->vm_file;
233
234	if (file) {
235	struct address_space *mapping = file->f_mapping;
236	mutex_lock(&mapping->i_mmap_mutex);
237	__remove_shared_vm_struct(vma, file, mapping);
238	mutex_unlock(&mapping->i_mmap_mutex);
239	}
240	}
241
242	/*
243	* Close a vm structure and free it, returning the next.
244	*/
245	static struct vm_area_struct remove_vma(struct vm_area_struct vma)
246	{
247	struct vm_area_struct *next = vma->vm_next;
248
249	might_sleep();
250	if (vma->vm_ops && vma->vm_ops->close)
251	vma->vm_ops->close(vma);
252	if (vma->vm_file)
253	fput(vma->vm_file);
254	mpol_put(vma_policy(vma));
255	kmem_cache_free(vm_area_cachep, vma);
256	return next;
257	}
258
259	static unsigned long do_brk(unsigned long addr, unsigned long len);
260
261	SYSCALL_DEFINE1(brk, unsigned long, brk)
262	{
263	unsigned long rlim, retval;
264	unsigned long newbrk, oldbrk;
265	struct mm_struct *mm = current->mm;
266	unsigned long min_brk;
267	bool populate;
268
269	down_write(&mm->mmap_sem);
270
271	#ifdef CONFIG_COMPAT_BRK
272	/*
273	* CONFIG_COMPAT_BRK can still be overridden by setting
274	* randomize_va_space to 2, which will still cause mm->start_brk
275	* to be arbitrarily shifted
276	*/
277	if (current->brk_randomized)
278	min_brk = mm->start_brk;
279	else
280	min_brk = mm->end_data;
281	#else
282	min_brk = mm->start_brk;
283	#endif
284	if (brk < min_brk)
285	goto out;
286
287	/*
288	* Check against rlimit here. If this check is done later after the test
289	* of oldbrk with newbrk then it can escape the test and let the data
290	* segment grow beyond its set limit the in case where the limit is
291	* not page aligned -Ram Gupta
292	*/
293	rlim = rlimit(RLIMIT_DATA);
294	if (rlim < RLIM_INFINITY && (brk - mm->start_brk) +
295	(mm->end_data - mm->start_data) > rlim)
296	goto out;
297
298	newbrk = PAGE_ALIGN(brk);
299	oldbrk = PAGE_ALIGN(mm->brk);
300	if (oldbrk == newbrk)
301	goto set_brk;
302
303	/* Always allow shrinking brk. */
304	if (brk <= mm->brk) {
305	if (!do_munmap(mm, newbrk, oldbrk-newbrk))
306	goto set_brk;
307	goto out;
308	}
309
310	/* Check against existing mmap mappings. */
311	if (find_vma_intersection(mm, oldbrk, newbrk+PAGE_SIZE))
312	goto out;
313
314	/* Ok, looks good - let it rip. */
315	if (do_brk(oldbrk, newbrk-oldbrk) != oldbrk)
316	goto out;
317
318	set_brk:
319	mm->brk = brk;
320	populate = newbrk > oldbrk && (mm->def_flags & VM_LOCKED) != 0;
321	up_write(&mm->mmap_sem);
322	if (populate)
323	mm_populate(oldbrk, newbrk - oldbrk);
324	return brk;
325
326	out:
327	retval = mm->brk;
328	up_write(&mm->mmap_sem);
329	return retval;
330	}
331
332	static long vma_compute_subtree_gap(struct vm_area_struct *vma)
333	{
334	unsigned long max, subtree_gap;
335	max = vma->vm_start;
336	if (vma->vm_prev)
337	max -= vma->vm_prev->vm_end;
338	if (vma->vm_rb.rb_left) {
339	subtree_gap = rb_entry(vma->vm_rb.rb_left,
340	struct vm_area_struct, vm_rb)->rb_subtree_gap;
341	if (subtree_gap > max)
342	max = subtree_gap;
343	}
344	if (vma->vm_rb.rb_right) {
345	subtree_gap = rb_entry(vma->vm_rb.rb_right,
346	struct vm_area_struct, vm_rb)->rb_subtree_gap;
347	if (subtree_gap > max)
348	max = subtree_gap;
349	}
350	return max;
351	}
352
353	#ifdef CONFIG_DEBUG_VM_RB
354	static int browse_rb(struct rb_root *root)
355	{
356	int i = 0, j, bug = 0;
357	struct rb_node nd, pn = NULL;
358	unsigned long prev = 0, pend = 0;
359
360	for (nd = rb_first(root); nd; nd = rb_next(nd)) {
361	struct vm_area_struct *vma;
362	vma = rb_entry(nd, struct vm_area_struct, vm_rb);
363	if (vma->vm_start < prev) {
364	printk("vm_start %lx prev %lx\n", vma->vm_start, prev);
365	bug = 1;
366	}
367	if (vma->vm_start < pend) {
368	printk("vm_start %lx pend %lx\n", vma->vm_start, pend);
369	bug = 1;
370	}
371	if (vma->vm_start > vma->vm_end) {
372	printk("vm_end %lx < vm_start %lx\n",
373	vma->vm_end, vma->vm_start);
374	bug = 1;
375	}
376	if (vma->rb_subtree_gap != vma_compute_subtree_gap(vma)) {
377	printk("free gap %lx, correct %lx\n",
378	vma->rb_subtree_gap,
379	vma_compute_subtree_gap(vma));
380	bug = 1;
381	}
382	i++;
383	pn = nd;
384	prev = vma->vm_start;
385	pend = vma->vm_end;
386	}
387	j = 0;
388	for (nd = pn; nd; nd = rb_prev(nd))
389	j++;
390	if (i != j) {
391	printk("backwards %d, forwards %d\n", j, i);
392	bug = 1;
393	}
394	return bug ? -1 : i;
395	}
396
397	static void validate_mm_rb(struct rb_root root, struct vm_area_struct ignore)
398	{
399	struct rb_node *nd;
400
401	for (nd = rb_first(root); nd; nd = rb_next(nd)) {
402	struct vm_area_struct *vma;
403	vma = rb_entry(nd, struct vm_area_struct, vm_rb);
404	BUG_ON(vma != ignore &&
405	vma->rb_subtree_gap != vma_compute_subtree_gap(vma));
406	}
407	}
408
409	void validate_mm(struct mm_struct *mm)
410	{
411	int bug = 0;
412	int i = 0;
413	unsigned long highest_address = 0;
414	struct vm_area_struct *vma = mm->mmap;
415	while (vma) {
416	struct anon_vma_chain *avc;
417	vma_lock_anon_vma(vma);
418	list_for_each_entry(avc, &vma->anon_vma_chain, same_vma)
419	anon_vma_interval_tree_verify(avc);
420	vma_unlock_anon_vma(vma);
421	highest_address = vma->vm_end;
422	vma = vma->vm_next;
423	i++;
424	}
425	if (i != mm->map_count) {
426	printk("map_count %d vm_next %d\n", mm->map_count, i);
427	bug = 1;
428	}
429	if (highest_address != mm->highest_vm_end) {
430	printk("mm->highest_vm_end %lx, found %lx\n",
431	mm->highest_vm_end, highest_address);
432	bug = 1;
433	}
434	i = browse_rb(&mm->mm_rb);
435	if (i != mm->map_count) {
436	printk("map_count %d rb %d\n", mm->map_count, i);
437	bug = 1;
438	}
439	BUG_ON(bug);
440	}
441	#else
442	#define validate_mm_rb(root, ignore) do { } while (0)
443	#define validate_mm(mm) do { } while (0)
444	#endif
445
446	RB_DECLARE_CALLBACKS(static, vma_gap_callbacks, struct vm_area_struct, vm_rb,
447	unsigned long, rb_subtree_gap, vma_compute_subtree_gap)
448
449	/*
450	* Update augmented rbtree rb_subtree_gap values after vma->vm_start or
451	* vma->vm_prev->vm_end values changed, without modifying the vma's position
452	* in the rbtree.
453	*/
454	static void vma_gap_update(struct vm_area_struct *vma)
455	{
456	/*
457	* As it turns out, RB_DECLARE_CALLBACKS() already created a callback
458	* function that does exacltly what we want.
459	*/
460	vma_gap_callbacks_propagate(&vma->vm_rb, NULL);
461	}
462
463	static inline void vma_rb_insert(struct vm_area_struct *vma,
464	struct rb_root *root)
465	{
466	/* All rb_subtree_gap values must be consistent prior to insertion */
467	validate_mm_rb(root, NULL);
468
469	rb_insert_augmented(&vma->vm_rb, root, &vma_gap_callbacks);
470	}
471
472	static void vma_rb_erase(struct vm_area_struct vma, struct rb_root root)
473	{
474	/*
475	* All rb_subtree_gap values must be consistent prior to erase,
476	* with the possible exception of the vma being erased.
477	*/
478	validate_mm_rb(root, vma);
479
480	/*
481	* Note rb_erase_augmented is a fairly large inline function,
482	* so make sure we instantiate it only once with our desired
483	* augmented rbtree callbacks.
484	*/
485	rb_erase_augmented(&vma->vm_rb, root, &vma_gap_callbacks);
486	}
487
488	/*
489	* vma has some anon_vma assigned, and is already inserted on that
490	* anon_vma's interval trees.
491	*
492	* Before updating the vma's vm_start / vm_end / vm_pgoff fields, the
493	* vma must be removed from the anon_vma's interval trees using
494	* anon_vma_interval_tree_pre_update_vma().
495	*
496	* After the update, the vma will be reinserted using
497	* anon_vma_interval_tree_post_update_vma().
498	*
499	* The entire update must be protected by exclusive mmap_sem and by
500	* the root anon_vma's mutex.
501	*/
502	static inline void
503	anon_vma_interval_tree_pre_update_vma(struct vm_area_struct *vma)
504	{
505	struct anon_vma_chain *avc;
506
507	list_for_each_entry(avc, &vma->anon_vma_chain, same_vma)
508	anon_vma_interval_tree_remove(avc, &avc->anon_vma->rb_root);
509	}
510
511	static inline void
512	anon_vma_interval_tree_post_update_vma(struct vm_area_struct *vma)
513	{
514	struct anon_vma_chain *avc;
515
516	list_for_each_entry(avc, &vma->anon_vma_chain, same_vma)
517	anon_vma_interval_tree_insert(avc, &avc->anon_vma->rb_root);
518	}
519
520	static int find_vma_links(struct mm_struct *mm, unsigned long addr,
521	unsigned long end, struct vm_area_struct **pprev,
522	struct rb_node *rb_link, struct rb_node rb_parent)
523	{
524	struct rb_node *__rb_link, __rb_parent, *rb_prev;
525
526	__rb_link = &mm->mm_rb.rb_node;
527	rb_prev = __rb_parent = NULL;
528
529	while (*__rb_link) {
530	struct vm_area_struct *vma_tmp;
531
532	__rb_parent = *__rb_link;
533	vma_tmp = rb_entry(__rb_parent, struct vm_area_struct, vm_rb);
534
535	if (vma_tmp->vm_end > addr) {
536	/* Fail if an existing vma overlaps the area */
537	if (vma_tmp->vm_start < end)
538	return -ENOMEM;
539	__rb_link = &__rb_parent->rb_left;
540	} else {
541	rb_prev = __rb_parent;
542	__rb_link = &__rb_parent->rb_right;
543	}
544	}
545
546	*pprev = NULL;
547	if (rb_prev)
548	*pprev = rb_entry(rb_prev, struct vm_area_struct, vm_rb);
549	*rb_link = __rb_link;
550	*rb_parent = __rb_parent;
551	return 0;
552	}
553
554	static unsigned long count_vma_pages_range(struct mm_struct *mm,
555	unsigned long addr, unsigned long end)
556	{
557	unsigned long nr_pages = 0;
558	struct vm_area_struct *vma;
559
560	/* Find first overlaping mapping */
561	vma = find_vma_intersection(mm, addr, end);
562	if (!vma)
563	return 0;
564
565	nr_pages = (min(end, vma->vm_end) -
566	max(addr, vma->vm_start)) >> PAGE_SHIFT;
567
568	/* Iterate over the rest of the overlaps */
569	for (vma = vma->vm_next; vma; vma = vma->vm_next) {
570	unsigned long overlap_len;
571
572	if (vma->vm_start > end)
573	break;
574
575	overlap_len = min(end, vma->vm_end) - vma->vm_start;
576	nr_pages += overlap_len >> PAGE_SHIFT;
577	}
578
579	return nr_pages;
580	}
581
582	void __vma_link_rb(struct mm_struct mm, struct vm_area_struct vma,
583	struct rb_node *rb_link, struct rb_node rb_parent)
584	{
585	/* Update tracking information for the gap following the new vma. */
586	if (vma->vm_next)
587	vma_gap_update(vma->vm_next);
588	else
589	mm->highest_vm_end = vma->vm_end;
590
591	/*
592	* vma->vm_prev wasn't known when we followed the rbtree to find the
593	* correct insertion point for that vma. As a result, we could not
594	* update the vma vm_rb parents rb_subtree_gap values on the way down.
595	* So, we first insert the vma with a zero rb_subtree_gap value
596	* (to be consistent with what we did on the way down), and then
597	* immediately update the gap to the correct value. Finally we
598	* rebalance the rbtree after all augmented values have been set.
599	*/
600	rb_link_node(&vma->vm_rb, rb_parent, rb_link);
601	vma->rb_subtree_gap = 0;
602	vma_gap_update(vma);
603	vma_rb_insert(vma, &mm->mm_rb);
604	}
605
606	static void __vma_link_file(struct vm_area_struct *vma)
607	{
608	struct file *file;
609
610	file = vma->vm_file;
611	if (file) {
612	struct address_space *mapping = file->f_mapping;
613
614	if (vma->vm_flags & VM_DENYWRITE)
615	atomic_dec(&file_inode(file)->i_writecount);
616	if (vma->vm_flags & VM_SHARED)
617	mapping->i_mmap_writable++;
618
619	flush_dcache_mmap_lock(mapping);
620	if (unlikely(vma->vm_flags & VM_NONLINEAR))
621	vma_nonlinear_insert(vma, &mapping->i_mmap_nonlinear);
622	else
623	vma_interval_tree_insert(vma, &mapping->i_mmap);
624	flush_dcache_mmap_unlock(mapping);
625	}
626	}
627
628	static void
629	__vma_link(struct mm_struct mm, struct vm_area_struct vma,
630	struct vm_area_struct prev, struct rb_node *rb_link,
631	struct rb_node *rb_parent)
632	{
633	__vma_link_list(mm, vma, prev, rb_parent);
634	__vma_link_rb(mm, vma, rb_link, rb_parent);
635	}
636
637	static void vma_link(struct mm_struct mm, struct vm_area_struct vma,
638	struct vm_area_struct prev, struct rb_node *rb_link,
639	struct rb_node *rb_parent)
640	{
641	struct address_space *mapping = NULL;
642
643	if (vma->vm_file)
644	mapping = vma->vm_file->f_mapping;
645
646	if (mapping)
647	mutex_lock(&mapping->i_mmap_mutex);
648
649	__vma_link(mm, vma, prev, rb_link, rb_parent);
650	__vma_link_file(vma);
651
652	if (mapping)
653	mutex_unlock(&mapping->i_mmap_mutex);
654
655	mm->map_count++;
656	validate_mm(mm);
657	}
658
659	/*
660	* Helper for vma_adjust() in the split_vma insert case: insert a vma into the
661	* mm's list and rbtree. It has already been inserted into the interval tree.
662	*/
663	static void __insert_vm_struct(struct mm_struct mm, struct vm_area_struct vma)
664	{
665	struct vm_area_struct *prev;
666	struct rb_node *rb_link, rb_parent;
667
668	if (find_vma_links(mm, vma->vm_start, vma->vm_end,
669	&prev, &rb_link, &rb_parent))
670	BUG();
671	__vma_link(mm, vma, prev, rb_link, rb_parent);
672	mm->map_count++;
673	}
674
675	static inline void
676	__vma_unlink(struct mm_struct mm, struct vm_area_struct vma,
677	struct vm_area_struct *prev)
678	{
679	struct vm_area_struct *next;
680
681	vma_rb_erase(vma, &mm->mm_rb);
682	prev->vm_next = next = vma->vm_next;
683	if (next)
684	next->vm_prev = prev;
685	if (mm->mmap_cache == vma)
686	mm->mmap_cache = prev;
687	}
688
689	/*
690	* We cannot adjust vm_start, vm_end, vm_pgoff fields of a vma that
691	* is already present in an i_mmap tree without adjusting the tree.
692	* The following helper function should be used when such adjustments
693	* are necessary. The "insert" vma (if any) is to be inserted
694	* before we drop the necessary locks.
695	*/
696	int vma_adjust(struct vm_area_struct *vma, unsigned long start,
697	unsigned long end, pgoff_t pgoff, struct vm_area_struct *insert)
698	{
699	struct mm_struct *mm = vma->vm_mm;
700	struct vm_area_struct *next = vma->vm_next;
701	struct vm_area_struct *importer = NULL;
702	struct address_space *mapping = NULL;
703	struct rb_root *root = NULL;
704	struct anon_vma *anon_vma = NULL;
705	struct file *file = vma->vm_file;
706	bool start_changed = false, end_changed = false;
707	long adjust_next = 0;
708	int remove_next = 0;
709
710	if (next && !insert) {
711	struct vm_area_struct *exporter = NULL;
712
713	if (end >= next->vm_end) {
714	/*
715	* vma expands, overlapping all the next, and
716	* perhaps the one after too (mprotect case 6).
717	*/
718	again: remove_next = 1 + (end > next->vm_end);
719	end = next->vm_end;
720	exporter = next;
721	importer = vma;
722	} else if (end > next->vm_start) {
723	/*
724	* vma expands, overlapping part of the next:
725	* mprotect case 5 shifting the boundary up.
726	*/
727	adjust_next = (end - next->vm_start) >> PAGE_SHIFT;
728	exporter = next;
729	importer = vma;
730	} else if (end < vma->vm_end) {
731	/*
732	* vma shrinks, and !insert tells it's not
733	* split_vma inserting another: so it must be
734	* mprotect case 4 shifting the boundary down.
735	*/
736	adjust_next = - ((vma->vm_end - end) >> PAGE_SHIFT);
737	exporter = vma;
738	importer = next;
739	}
740
741	/*
742	* Easily overlooked: when mprotect shifts the boundary,
743	* make sure the expanding vma has anon_vma set if the
744	* shrinking vma had, to cover any anon pages imported.
745	*/
746	if (exporter && exporter->anon_vma && !importer->anon_vma) {
747	if (anon_vma_clone(importer, exporter))
748	return -ENOMEM;
749	importer->anon_vma = exporter->anon_vma;
750	}
751	}
752
753	if (file) {
754	mapping = file->f_mapping;
755	if (!(vma->vm_flags & VM_NONLINEAR)) {
756	root = &mapping->i_mmap;
757	uprobe_munmap(vma, vma->vm_start, vma->vm_end);
758
759	if (adjust_next)
760	uprobe_munmap(next, next->vm_start,
761	next->vm_end);
762	}
763
764	mutex_lock(&mapping->i_mmap_mutex);
765	if (insert) {
766	/*
767	* Put into interval tree now, so instantiated pages
768	* are visible to arm/parisc __flush_dcache_page
769	* throughout; but we cannot insert into address
770	* space until vma start or end is updated.
771	*/
772	__vma_link_file(insert);
773	}
774	}
775
776	vma_adjust_trans_huge(vma, start, end, adjust_next);
777
778	anon_vma = vma->anon_vma;
779	if (!anon_vma && adjust_next)
780	anon_vma = next->anon_vma;
781	if (anon_vma) {
782	VM_BUG_ON(adjust_next && next->anon_vma &&
783	anon_vma != next->anon_vma);
784	anon_vma_lock_write(anon_vma);
785	anon_vma_interval_tree_pre_update_vma(vma);
786	if (adjust_next)
787	anon_vma_interval_tree_pre_update_vma(next);
788	}
789
790	if (root) {
791	flush_dcache_mmap_lock(mapping);
792	vma_interval_tree_remove(vma, root);
793	if (adjust_next)
794	vma_interval_tree_remove(next, root);
795	}
796
797	if (start != vma->vm_start) {
798	vma->vm_start = start;
799	start_changed = true;
800	}
801	if (end != vma->vm_end) {
802	vma->vm_end = end;
803	end_changed = true;
804	}
805	vma->vm_pgoff = pgoff;
806	if (adjust_next) {
807	next->vm_start += adjust_next << PAGE_SHIFT;
808	next->vm_pgoff += adjust_next;
809	}
810
811	if (root) {
812	if (adjust_next)
813	vma_interval_tree_insert(next, root);
814	vma_interval_tree_insert(vma, root);
815	flush_dcache_mmap_unlock(mapping);
816	}
817
818	if (remove_next) {
819	/*
820	* vma_merge has merged next into vma, and needs
821	* us to remove next before dropping the locks.
822	*/
823	__vma_unlink(mm, next, vma);
824	if (file)
825	__remove_shared_vm_struct(next, file, mapping);
826	} else if (insert) {
827	/*
828	* split_vma has split insert from vma, and needs
829	* us to insert it before dropping the locks
830	* (it may either follow vma or precede it).
831	*/
832	__insert_vm_struct(mm, insert);
833	} else {
834	if (start_changed)
835	vma_gap_update(vma);
836	if (end_changed) {
837	if (!next)
838	mm->highest_vm_end = end;
839	else if (!adjust_next)
840	vma_gap_update(next);
841	}
842	}
843
844	if (anon_vma) {
845	anon_vma_interval_tree_post_update_vma(vma);
846	if (adjust_next)
847	anon_vma_interval_tree_post_update_vma(next);
848	anon_vma_unlock_write(anon_vma);
849	}
850	if (mapping)
851	mutex_unlock(&mapping->i_mmap_mutex);
852
853	if (root) {
854	uprobe_mmap(vma);
855
856	if (adjust_next)
857	uprobe_mmap(next);
858	}
859
860	if (remove_next) {
861	if (file) {
862	uprobe_munmap(next, next->vm_start, next->vm_end);
863	fput(file);
864	}
865	if (next->anon_vma)
866	anon_vma_merge(vma, next);
867	mm->map_count--;
868	mpol_put(vma_policy(next));
869	kmem_cache_free(vm_area_cachep, next);
870	/*
871	* In mprotect's case 6 (see comments on vma_merge),
872	* we must remove another next too. It would clutter
873	* up the code too much to do both in one go.
874	*/
875	next = vma->vm_next;
876	if (remove_next == 2)
877	goto again;
878	else if (next)
879	vma_gap_update(next);
880	else
881	mm->highest_vm_end = end;
882	}
883	if (insert && file)
884	uprobe_mmap(insert);
885
886	validate_mm(mm);
887
888	return 0;
889	}
890
891	/*
892	* If the vma has a ->close operation then the driver probably needs to release
893	* per-vma resources, so we don't attempt to merge those.
894	*/
895	static inline int is_mergeable_vma(struct vm_area_struct *vma,
896	struct file *file, unsigned long vm_flags)
897	{
898	if (vma->vm_flags ^ vm_flags)
899	return 0;
900	if (vma->vm_file != file)
901	return 0;
902	if (vma->vm_ops && vma->vm_ops->close)
903	return 0;
904	return 1;
905	}
906
907	static inline int is_mergeable_anon_vma(struct anon_vma *anon_vma1,
908	struct anon_vma *anon_vma2,
909	struct vm_area_struct *vma)
910	{
911	/*
912	* The list_is_singular() test is to avoid merging VMA cloned from
913	* parents. This can improve scalability caused by anon_vma lock.
914	*/
915	if ((!anon_vma1 \|\| !anon_vma2) && (!vma \|\|
916	list_is_singular(&vma->anon_vma_chain)))
917	return 1;
918	return anon_vma1 == anon_vma2;
919	}
920
921	/*
922	* Return true if we can merge this (vm_flags,anon_vma,file,vm_pgoff)
923	* in front of (at a lower virtual address and file offset than) the vma.
924	*
925	* We cannot merge two vmas if they have differently assigned (non-NULL)
926	* anon_vmas, nor if same anon_vma is assigned but offsets incompatible.
927	*
928	* We don't check here for the merged mmap wrapping around the end of pagecache
929	* indices (16TB on ia32) because do_mmap_pgoff() does not permit mmap's which
930	* wrap, nor mmaps which cover the final page at index -1UL.
931	*/
932	static int
933	can_vma_merge_before(struct vm_area_struct *vma, unsigned long vm_flags,
934	struct anon_vma anon_vma, struct file file, pgoff_t vm_pgoff)
935	{
936	if (is_mergeable_vma(vma, file, vm_flags) &&
937	is_mergeable_anon_vma(anon_vma, vma->anon_vma, vma)) {
938	if (vma->vm_pgoff == vm_pgoff)
939	return 1;
940	}
941	return 0;
942	}
943
944	/*
945	* Return true if we can merge this (vm_flags,anon_vma,file,vm_pgoff)
946	* beyond (at a higher virtual address and file offset than) the vma.
947	*
948	* We cannot merge two vmas if they have differently assigned (non-NULL)
949	* anon_vmas, nor if same anon_vma is assigned but offsets incompatible.
950	*/
951	static int
952	can_vma_merge_after(struct vm_area_struct *vma, unsigned long vm_flags,
953	struct anon_vma anon_vma, struct file file, pgoff_t vm_pgoff)
954	{
955	if (is_mergeable_vma(vma, file, vm_flags) &&
956	is_mergeable_anon_vma(anon_vma, vma->anon_vma, vma)) {
957	pgoff_t vm_pglen;
958	vm_pglen = vma_pages(vma);
959	if (vma->vm_pgoff + vm_pglen == vm_pgoff)
960	return 1;
961	}
962	return 0;
963	}
964
965	/*
966	* Given a mapping request (addr,end,vm_flags,file,pgoff), figure out
967	* whether that can be merged with its predecessor or its successor.
968	* Or both (it neatly fills a hole).
969	*
970	* In most cases - when called for mmap, brk or mremap - [addr,end) is
971	* certain not to be mapped by the time vma_merge is called; but when
972	* called for mprotect, it is certain to be already mapped (either at
973	* an offset within prev, or at the start of next), and the flags of
974	* this area are about to be changed to vm_flags - and the no-change
975	* case has already been eliminated.
976	*
977	* The following mprotect cases have to be considered, where AAAA is
978	* the area passed down from mprotect_fixup, never extending beyond one
979	* vma, PPPPPP is the prev vma specified, and NNNNNN the next vma after:
980	*
981	* AAAA AAAA AAAA AAAA
982	* PPPPPPNNNNNN PPPPPPNNNNNN PPPPPPNNNNNN PPPPNNNNXXXX
983	* cannot merge might become might become might become
984	* PPNNNNNNNNNN PPPPPPPPPPNN PPPPPPPPPPPP 6 or
985	* mmap, brk or case 4 below case 5 below PPPPPPPPXXXX 7 or
986	* mremap move: PPPPNNNNNNNN 8
987	* AAAA
988	* PPPP NNNN PPPPPPPPPPPP PPPPPPPPNNNN PPPPNNNNNNNN
989	* might become case 1 below case 2 below case 3 below
990	*
991	* Odd one out? Case 8, because it extends NNNN but needs flags of XXXX:
992	* mprotect_fixup updates vm_flags & vm_page_prot on successful return.
993	*/
994	struct vm_area_struct vma_merge(struct mm_struct mm,
995	struct vm_area_struct *prev, unsigned long addr,
996	unsigned long end, unsigned long vm_flags,
997	struct anon_vma anon_vma, struct file file,
998	pgoff_t pgoff, struct mempolicy *policy)
999	{
1000	pgoff_t pglen = (end - addr) >> PAGE_SHIFT;
1001	struct vm_area_struct area, next;
1002	int err;
1003
1004	/*
1005	* We later require that vma->vm_flags == vm_flags,
1006	* so this tests vma->vm_flags & VM_SPECIAL, too.
1007	*/
1008	if (vm_flags & VM_SPECIAL)
1009	return NULL;
1010
1011	if (prev)
1012	next = prev->vm_next;
1013	else
1014	next = mm->mmap;
1015	area = next;
1016	if (next && next->vm_end == end) /* cases 6, 7, 8 */
1017	next = next->vm_next;
1018
1019	/*
1020	* Can it merge with the predecessor?
1021	*/
1022	if (prev && prev->vm_end == addr &&
1023	mpol_equal(vma_policy(prev), policy) &&
1024	can_vma_merge_after(prev, vm_flags,
1025	anon_vma, file, pgoff)) {
1026	/*
1027	* OK, it can. Can we now merge in the successor as well?
1028	*/
1029	if (next && end == next->vm_start &&
1030	mpol_equal(policy, vma_policy(next)) &&
1031	can_vma_merge_before(next, vm_flags,
1032	anon_vma, file, pgoff+pglen) &&
1033	is_mergeable_anon_vma(prev->anon_vma,
1034	next->anon_vma, NULL)) {
1035	/* cases 1, 6 */
1036	err = vma_adjust(prev, prev->vm_start,
1037	next->vm_end, prev->vm_pgoff, NULL);
1038	} else /* cases 2, 5, 7 */
1039	err = vma_adjust(prev, prev->vm_start,
1040	end, prev->vm_pgoff, NULL);
1041	if (err)
1042	return NULL;
1043	khugepaged_enter_vma_merge(prev);
1044	return prev;
1045	}
1046
1047	/*
1048	* Can this new request be merged in front of next?
1049	*/
1050	if (next && end == next->vm_start &&
1051	mpol_equal(policy, vma_policy(next)) &&
1052	can_vma_merge_before(next, vm_flags,
1053	anon_vma, file, pgoff+pglen)) {
1054	if (prev && addr < prev->vm_end) /* case 4 */
1055	err = vma_adjust(prev, prev->vm_start,
1056	addr, prev->vm_pgoff, NULL);
1057	else /* cases 3, 8 */
1058	err = vma_adjust(area, addr, next->vm_end,
1059	next->vm_pgoff - pglen, NULL);
1060	if (err)
1061	return NULL;
1062	khugepaged_enter_vma_merge(area);
1063	return area;
1064	}
1065
1066	return NULL;
1067	}
1068
1069	/*
1070	* Rough compatbility check to quickly see if it's even worth looking
1071	* at sharing an anon_vma.
1072	*
1073	* They need to have the same vm_file, and the flags can only differ
1074	* in things that mprotect may change.
1075	*
1076	* NOTE! The fact that we share an anon_vma doesn't _have_ to mean that
1077	* we can merge the two vma's. For example, we refuse to merge a vma if
1078	* there is a vm_ops->close() function, because that indicates that the
1079	* driver is doing some kind of reference counting. But that doesn't
1080	* really matter for the anon_vma sharing case.
1081	*/
1082	static int anon_vma_compatible(struct vm_area_struct a, struct vm_area_struct b)
1083	{
1084	return a->vm_end == b->vm_start &&
1085	mpol_equal(vma_policy(a), vma_policy(b)) &&
1086	a->vm_file == b->vm_file &&
1087	!((a->vm_flags ^ b->vm_flags) & ~(VM_READ\|VM_WRITE\|VM_EXEC)) &&
1088	b->vm_pgoff == a->vm_pgoff + ((b->vm_start - a->vm_start) >> PAGE_SHIFT);
1089	}
1090
1091	/*
1092	* Do some basic sanity checking to see if we can re-use the anon_vma
1093	* from 'old'. The 'a'/'b' vma's are in VM order - one of them will be
1094	* the same as 'old', the other will be the new one that is trying
1095	* to share the anon_vma.
1096	*
1097	* NOTE! This runs with mm_sem held for reading, so it is possible that
1098	* the anon_vma of 'old' is concurrently in the process of being set up
1099	* by another page fault trying to merge _that_. But that's ok: if it
1100	* is being set up, that automatically means that it will be a singleton
1101	* acceptable for merging, so we can do all of this optimistically. But
1102	* we do that ACCESS_ONCE() to make sure that we never re-load the pointer.
1103	*
1104	* IOW: that the "list_is_singular()" test on the anon_vma_chain only
1105	* matters for the 'stable anon_vma' case (ie the thing we want to avoid
1106	* is to return an anon_vma that is "complex" due to having gone through
1107	* a fork).
1108	*
1109	* We also make sure that the two vma's are compatible (adjacent,
1110	* and with the same memory policies). That's all stable, even with just
1111	* a read lock on the mm_sem.
1112	*/
1113	static struct anon_vma reusable_anon_vma(struct vm_area_struct old, struct vm_area_struct a, struct vm_area_struct b)
1114	{
1115	if (anon_vma_compatible(a, b)) {
1116	struct anon_vma *anon_vma = ACCESS_ONCE(old->anon_vma);
1117
1118	if (anon_vma && list_is_singular(&old->anon_vma_chain))
1119	return anon_vma;
1120	}
1121	return NULL;
1122	}
1123
1124	/*
1125	* find_mergeable_anon_vma is used by anon_vma_prepare, to check
1126	* neighbouring vmas for a suitable anon_vma, before it goes off
1127	* to allocate a new anon_vma. It checks because a repetitive
1128	* sequence of mprotects and faults may otherwise lead to distinct
1129	* anon_vmas being allocated, preventing vma merge in subsequent
1130	* mprotect.
1131	*/
1132	struct anon_vma find_mergeable_anon_vma(struct vm_area_struct vma)
1133	{
1134	struct anon_vma *anon_vma;
1135	struct vm_area_struct *near;
1136
1137	near = vma->vm_next;
1138	if (!near)
1139	goto try_prev;
1140
1141	anon_vma = reusable_anon_vma(near, vma, near);
1142	if (anon_vma)
1143	return anon_vma;
1144	try_prev:
1145	near = vma->vm_prev;
1146	if (!near)
1147	goto none;
1148
1149	anon_vma = reusable_anon_vma(near, near, vma);
1150	if (anon_vma)
1151	return anon_vma;
1152	none:
1153	/*
1154	* There's no absolute need to look only at touching neighbours:
1155	* we could search further afield for "compatible" anon_vmas.
1156	* But it would probably just be a waste of time searching,
1157	* or lead to too many vmas hanging off the same anon_vma.
1158	* We're trying to allow mprotect remerging later on,
1159	* not trying to minimize memory used for anon_vmas.
1160	*/
1161	return NULL;
1162	}
1163
1164	#ifdef CONFIG_PROC_FS
1165	void vm_stat_account(struct mm_struct *mm, unsigned long flags,
1166	struct file *file, long pages)
1167	{
1168	const unsigned long stack_flags
1169	= VM_STACK_FLAGS & (VM_GROWSUP\|VM_GROWSDOWN);
1170
1171	mm->total_vm += pages;
1172
1173	if (file) {
1174	mm->shared_vm += pages;
1175	if ((flags & (VM_EXEC\|VM_WRITE)) == VM_EXEC)
1176	mm->exec_vm += pages;
1177	} else if (flags & stack_flags)
1178	mm->stack_vm += pages;
1179	}
1180	#endif /* CONFIG_PROC_FS */
1181
1182	/*
1183	* If a hint addr is less than mmap_min_addr change hint to be as
1184	* low as possible but still greater than mmap_min_addr
1185	*/
1186	static inline unsigned long round_hint_to_min(unsigned long hint)
1187	{
1188	hint &= PAGE_MASK;
1189	if (((void *)hint != NULL) &&
1190	(hint < mmap_min_addr))
1191	return PAGE_ALIGN(mmap_min_addr);
1192	return hint;
1193	}
1194
1195	/*
1196	* The caller must hold down_write(&current->mm->mmap_sem).
1197	*/
1198
1199	unsigned long do_mmap_pgoff(struct file *file, unsigned long addr,
1200	unsigned long len, unsigned long prot,
1201	unsigned long flags, unsigned long pgoff,
1202	unsigned long *populate)
1203	{
1204	struct mm_struct * mm = current->mm;
1205	vm_flags_t vm_flags;
1206
1207	*populate = 0;
1208
1209	/*
1210	* Does the application expect PROT_READ to imply PROT_EXEC?
1211	*
1212	* (the exception is when the underlying filesystem is noexec
1213	* mounted, in which case we dont add PROT_EXEC.)
1214	*/
1215	if ((prot & PROT_READ) && (current->personality & READ_IMPLIES_EXEC))
1216	if (!(file && (file->f_path.mnt->mnt_flags & MNT_NOEXEC)))
1217	prot \|= PROT_EXEC;
1218
1219	if (!len)
1220	return -EINVAL;
1221
1222	if (!(flags & MAP_FIXED))
1223	addr = round_hint_to_min(addr);
1224
1225	/* Careful about overflows.. */
1226	len = PAGE_ALIGN(len);
1227	if (!len)
1228	return -ENOMEM;
1229
1230	/* offset overflow? */
1231	if ((pgoff + (len >> PAGE_SHIFT)) < pgoff)
1232	return -EOVERFLOW;
1233
1234	/* Too many mappings? */
1235	if (mm->map_count > sysctl_max_map_count)
1236	return -ENOMEM;
1237
1238	/* Obtain the address to map to. we verify (or select) it and ensure
1239	* that it represents a valid section of the address space.
1240	*/
1241	addr = get_unmapped_area(file, addr, len, pgoff, flags);
1242	if (addr & ~PAGE_MASK)
1243	return addr;
1244
1245	/* Do simple checking here so the lower-level routines won't have
1246	* to. we assume access permissions have been handled by the open
1247	* of the memory object, so we don't do any here.
1248	*/
1249	vm_flags = calc_vm_prot_bits(prot) \| calc_vm_flag_bits(flags) \|
1250	mm->def_flags \| VM_MAYREAD \| VM_MAYWRITE \| VM_MAYEXEC;
1251
1252	if (flags & MAP_LOCKED)
1253	if (!can_do_mlock())
1254	return -EPERM;
1255
1256	/* mlock MCL_FUTURE? */
1257	if (vm_flags & VM_LOCKED) {
1258	unsigned long locked, lock_limit;
1259	locked = len >> PAGE_SHIFT;
1260	locked += mm->locked_vm;
1261	lock_limit = rlimit(RLIMIT_MEMLOCK);
1262	lock_limit >>= PAGE_SHIFT;
1263	if (locked > lock_limit && !capable(CAP_IPC_LOCK))
1264	return -EAGAIN;
1265	}
1266
1267	if (file) {
1268	struct inode *inode = file_inode(file);
1269
1270	switch (flags & MAP_TYPE) {
1271	case MAP_SHARED:
1272	if ((prot&PROT_WRITE) && !(file->f_mode&FMODE_WRITE))
1273	return -EACCES;
1274
1275	/*
1276	* Make sure we don't allow writing to an append-only
1277	* file..
1278	*/
1279	if (IS_APPEND(inode) && (file->f_mode & FMODE_WRITE))
1280	return -EACCES;
1281
1282	/*
1283	* Make sure there are no mandatory locks on the file.
1284	*/
1285	if (locks_verify_locked(inode))
1286	return -EAGAIN;
1287
1288	vm_flags \|= VM_SHARED \| VM_MAYSHARE;
1289	if (!(file->f_mode & FMODE_WRITE))
1290	vm_flags &= ~(VM_MAYWRITE \| VM_SHARED);
1291
1292	/* fall through */
1293	case MAP_PRIVATE:
1294	if (!(file->f_mode & FMODE_READ))
1295	return -EACCES;
1296	if (file->f_path.mnt->mnt_flags & MNT_NOEXEC) {
1297	if (vm_flags & VM_EXEC)
1298	return -EPERM;
1299	vm_flags &= ~VM_MAYEXEC;
1300	}
1301
1302	if (!file->f_op \|\| !file->f_op->mmap)
1303	return -ENODEV;
1304	if (vm_flags & (VM_GROWSDOWN\|VM_GROWSUP))
1305	return -EINVAL;
1306	break;
1307
1308	default:
1309	return -EINVAL;
1310	}
1311	} else {
1312	switch (flags & MAP_TYPE) {
1313	case MAP_SHARED:
1314	if (vm_flags & (VM_GROWSDOWN\|VM_GROWSUP))
1315	return -EINVAL;
1316	/*
1317	* Ignore pgoff.
1318	*/
1319	pgoff = 0;
1320	vm_flags \|= VM_SHARED \| VM_MAYSHARE;
1321	break;
1322	case MAP_PRIVATE:
1323	/*
1324	* Set pgoff according to addr for anon_vma.
1325	*/
1326	pgoff = addr >> PAGE_SHIFT;
1327	break;
1328	default:
1329	return -EINVAL;
1330	}
1331	}
1332
1333	/*
1334	* Set 'VM_NORESERVE' if we should not account for the
1335	* memory use of this mapping.
1336	*/
1337	if (flags & MAP_NORESERVE) {
1338	/* We honor MAP_NORESERVE if allowed to overcommit */
1339	if (sysctl_overcommit_memory != OVERCOMMIT_NEVER)
1340	vm_flags \|= VM_NORESERVE;
1341
1342	/* hugetlb applies strict overcommit unless MAP_NORESERVE */
1343	if (file && is_file_hugepages(file))
1344	vm_flags \|= VM_NORESERVE;
1345	}
1346
1347	addr = mmap_region(file, addr, len, vm_flags, pgoff);
1348	if (!IS_ERR_VALUE(addr) &&
1349	((vm_flags & VM_LOCKED) \|\|
1350	(flags & (MAP_POPULATE \| MAP_NONBLOCK)) == MAP_POPULATE))
1351	*populate = len;
1352	return addr;
1353	}
1354
1355	SYSCALL_DEFINE6(mmap_pgoff, unsigned long, addr, unsigned long, len,
1356	unsigned long, prot, unsigned long, flags,
1357	unsigned long, fd, unsigned long, pgoff)
1358	{
1359	struct file *file = NULL;
1360	unsigned long retval = -EBADF;
1361
1362	if (!(flags & MAP_ANONYMOUS)) {
1363	audit_mmap_fd(fd, flags);
1364	file = fget(fd);
1365	if (!file)
1366	goto out;
1367	if (is_file_hugepages(file))
1368	len = ALIGN(len, huge_page_size(hstate_file(file)));
1369	retval = -EINVAL;
1370	if (unlikely(flags & MAP_HUGETLB && !is_file_hugepages(file)))
1371	goto out_fput;
1372	} else if (flags & MAP_HUGETLB) {
1373	struct user_struct *user = NULL;
1374	struct hstate *hs;
1375
1376	hs = hstate_sizelog((flags >> MAP_HUGE_SHIFT) & SHM_HUGE_MASK);
1377	if (!hs)
1378	return -EINVAL;
1379
1380	len = ALIGN(len, huge_page_size(hs));
1381	/*
1382	* VM_NORESERVE is used because the reservations will be
1383	* taken when vm_ops->mmap() is called
1384	* A dummy user value is used because we are not locking
1385	* memory so no accounting is necessary
1386	*/
1387	file = hugetlb_file_setup(HUGETLB_ANON_FILE, len,
1388	VM_NORESERVE,
1389	&user, HUGETLB_ANONHUGE_INODE,
1390	(flags >> MAP_HUGE_SHIFT) & MAP_HUGE_MASK);
1391	if (IS_ERR(file))
1392	return PTR_ERR(file);
1393	}
1394
1395	flags &= ~(MAP_EXECUTABLE \| MAP_DENYWRITE);
1396
1397	retval = vm_mmap_pgoff(file, addr, len, prot, flags, pgoff);
1398	out_fput:
1399	if (file)
1400	fput(file);
1401	out:
1402	return retval;
1403	}
1404
1405	#ifdef __ARCH_WANT_SYS_OLD_MMAP
1406	struct mmap_arg_struct {
1407	unsigned long addr;
1408	unsigned long len;
1409	unsigned long prot;
1410	unsigned long flags;
1411	unsigned long fd;
1412	unsigned long offset;
1413	};
1414
1415	SYSCALL_DEFINE1(old_mmap, struct mmap_arg_struct __user *, arg)
1416	{
1417	struct mmap_arg_struct a;
1418
1419	if (copy_from_user(&a, arg, sizeof(a)))
1420	return -EFAULT;
1421	if (a.offset & ~PAGE_MASK)
1422	return -EINVAL;
1423
1424	return sys_mmap_pgoff(a.addr, a.len, a.prot, a.flags, a.fd,
1425	a.offset >> PAGE_SHIFT);
1426	}
1427	#endif /* __ARCH_WANT_SYS_OLD_MMAP */
1428
1429	/*
1430	* Some shared mappigns will want the pages marked read-only
1431	* to track write events. If so, we'll downgrade vm_page_prot
1432	* to the private version (using protection_map[] without the
1433	* VM_SHARED bit).
1434	*/
1435	int vma_wants_writenotify(struct vm_area_struct *vma)
1436	{
1437	vm_flags_t vm_flags = vma->vm_flags;
1438
1439	/* If it was private or non-writable, the write bit is already clear */
1440	if ((vm_flags & (VM_WRITE\|VM_SHARED)) != ((VM_WRITE\|VM_SHARED)))
1441	return 0;
1442
1443	/* The backer wishes to know when pages are first written to? */
1444	if (vma->vm_ops && vma->vm_ops->page_mkwrite)
1445	return 1;
1446
1447	/* The open routine did something to the protections already? */
1448	if (pgprot_val(vma->vm_page_prot) !=
1449	pgprot_val(vm_get_page_prot(vm_flags)))
1450	return 0;
1451
1452	/* Specialty mapping? */
1453	if (vm_flags & VM_PFNMAP)
1454	return 0;
1455
1456	/* Can the mapping track the dirty pages? */
1457	return vma->vm_file && vma->vm_file->f_mapping &&
1458	mapping_cap_account_dirty(vma->vm_file->f_mapping);
1459	}
1460
1461	/*
1462	* We account for memory if it's a private writeable mapping,
1463	* not hugepages and VM_NORESERVE wasn't set.
1464	*/
1465	static inline int accountable_mapping(struct file *file, vm_flags_t vm_flags)
1466	{
1467	/*
1468	* hugetlb has its own accounting separate from the core VM
1469	* VM_HUGETLB may not be set yet so we cannot check for that flag.
1470	*/
1471	if (file && is_file_hugepages(file))
1472	return 0;
1473
1474	return (vm_flags & (VM_NORESERVE \| VM_SHARED \| VM_WRITE)) == VM_WRITE;
1475	}
1476
1477	unsigned long mmap_region(struct file *file, unsigned long addr,
1478	unsigned long len, vm_flags_t vm_flags, unsigned long pgoff)
1479	{
1480	struct mm_struct *mm = current->mm;
1481	struct vm_area_struct vma, prev;
1482	int error;
1483	struct rb_node *rb_link, rb_parent;
1484	unsigned long charged = 0;
1485
1486	/* Check against address space limit. */
1487	if (!may_expand_vm(mm, len >> PAGE_SHIFT)) {
1488	unsigned long nr_pages;
1489
1490	/*
1491	* MAP_FIXED may remove pages of mappings that intersects with
1492	* requested mapping. Account for the pages it would unmap.
1493	*/
1494	if (!(vm_flags & MAP_FIXED))
1495	return -ENOMEM;
1496
1497	nr_pages = count_vma_pages_range(mm, addr, addr + len);
1498
1499	if (!may_expand_vm(mm, (len >> PAGE_SHIFT) - nr_pages))
1500	return -ENOMEM;
1501	}
1502
1503	/* Clear old maps */
1504	error = -ENOMEM;
1505	munmap_back:
1506	if (find_vma_links(mm, addr, addr + len, &prev, &rb_link, &rb_parent)) {
1507	if (do_munmap(mm, addr, len))
1508	return -ENOMEM;
1509	goto munmap_back;
1510	}
1511
1512	/*
1513	* Private writable mapping: check memory availability
1514	*/
1515	if (accountable_mapping(file, vm_flags)) {
1516	charged = len >> PAGE_SHIFT;
1517	if (security_vm_enough_memory_mm(mm, charged))
1518	return -ENOMEM;
1519	vm_flags \|= VM_ACCOUNT;
1520	}
1521
1522	/*
1523	* Can we just expand an old mapping?
1524	*/
1525	vma = vma_merge(mm, prev, addr, addr + len, vm_flags, NULL, file, pgoff, NULL);
1526	if (vma)
1527	goto out;
1528
1529	/*
1530	* Determine the object being mapped and call the appropriate
1531	* specific mapper. the address has already been validated, but
1532	* not unmapped, but the maps are removed from the list.
1533	*/
1534	vma = kmem_cache_zalloc(vm_area_cachep, GFP_KERNEL);
1535	if (!vma) {
1536	error = -ENOMEM;
1537	goto unacct_error;
1538	}
1539
1540	vma->vm_mm = mm;
1541	vma->vm_start = addr;
1542	vma->vm_end = addr + len;
1543	vma->vm_flags = vm_flags;
1544	vma->vm_page_prot = vm_get_page_prot(vm_flags);
1545	vma->vm_pgoff = pgoff;
1546	INIT_LIST_HEAD(&vma->anon_vma_chain);
1547
1548	if (file) {
1549	if (vm_flags & VM_DENYWRITE) {
1550	error = deny_write_access(file);
1551	if (error)
1552	goto free_vma;
1553	}
1554	vma->vm_file = get_file(file);
1555	error = file->f_op->mmap(file, vma);
1556	if (error)
1557	goto unmap_and_free_vma;
1558
1559	/* Can addr have changed??
1560	*
1561	* Answer: Yes, several device drivers can do it in their
1562	* f_op->mmap method. -DaveM
1563	* Bug: If addr is changed, prev, rb_link, rb_parent should
1564	* be updated for vma_link()
1565	*/
1566	WARN_ON_ONCE(addr != vma->vm_start);
1567
1568	addr = vma->vm_start;
1569	vm_flags = vma->vm_flags;
1570	} else if (vm_flags & VM_SHARED) {
1571	error = shmem_zero_setup(vma);
1572	if (error)
1573	goto free_vma;
1574	}
1575
1576	if (vma_wants_writenotify(vma)) {
1577	pgprot_t pprot = vma->vm_page_prot;
1578
1579	/* Can vma->vm_page_prot have changed??
1580	*
1581	* Answer: Yes, drivers may have changed it in their
1582	* f_op->mmap method.
1583	*
1584	* Ensures that vmas marked as uncached stay that way.
1585	*/
1586	vma->vm_page_prot = vm_get_page_prot(vm_flags & ~VM_SHARED);
1587	if (pgprot_val(pprot) == pgprot_val(pgprot_noncached(pprot)))
1588	vma->vm_page_prot = pgprot_noncached(vma->vm_page_prot);
1589	}
1590
1591	vma_link(mm, vma, prev, rb_link, rb_parent);
1592	/* Once vma denies write, undo our temporary denial count */
1593	if (vm_flags & VM_DENYWRITE)
1594	allow_write_access(file);
1595	file = vma->vm_file;
1596	out:
1597	perf_event_mmap(vma);
1598
1599	vm_stat_account(mm, vm_flags, file, len >> PAGE_SHIFT);
1600	if (vm_flags & VM_LOCKED) {
1601	if (!((vm_flags & VM_SPECIAL) \|\| is_vm_hugetlb_page(vma) \|\|
1602	vma == get_gate_vma(current->mm)))
1603	mm->locked_vm += (len >> PAGE_SHIFT);
1604	else
1605	vma->vm_flags &= ~VM_LOCKED;
1606	}
1607
1608	if (file)
1609	uprobe_mmap(vma);
1610
1611	/*
1612	* New (or expanded) vma always get soft dirty status.
1613	* Otherwise user-space soft-dirty page tracker won't
1614	* be able to distinguish situation when vma area unmapped,
1615	* then new mapped in-place (which must be aimed as
1616	* a completely new data area).
1617	*/
1618	vma->vm_flags \|= VM_SOFTDIRTY;
1619
1620	return addr;
1621
1622	unmap_and_free_vma:
1623	if (vm_flags & VM_DENYWRITE)
1624	allow_write_access(file);
1625	vma->vm_file = NULL;
1626	fput(file);
1627
1628	/* Undo any partial mapping done by a device driver. */
1629	unmap_region(mm, vma, prev, vma->vm_start, vma->vm_end);
1630	charged = 0;
1631	free_vma:
1632	kmem_cache_free(vm_area_cachep, vma);
1633	unacct_error:
1634	if (charged)
1635	vm_unacct_memory(charged);
1636	return error;
1637	}
1638
1639	unsigned long unmapped_area(struct vm_unmapped_area_info *info)
1640	{
1641	/*
1642	* We implement the search by looking for an rbtree node that
1643	* immediately follows a suitable gap. That is,
1644	* - gap_start = vma->vm_prev->vm_end <= info->high_limit - length;
1645	* - gap_end = vma->vm_start >= info->low_limit + length;
1646	* - gap_end - gap_start >= length
1647	*/
1648
1649	struct mm_struct *mm = current->mm;
1650	struct vm_area_struct *vma;
1651	unsigned long length, low_limit, high_limit, gap_start, gap_end;
1652
1653	/* Adjust search length to account for worst case alignment overhead */
1654	length = info->length + info->align_mask;
1655	if (length < info->length)
1656	return -ENOMEM;
1657
1658	/* Adjust search limits by the desired length */
1659	if (info->high_limit < length)
1660	return -ENOMEM;
1661	high_limit = info->high_limit - length;
1662
1663	if (info->low_limit > high_limit)
1664	return -ENOMEM;
1665	low_limit = info->low_limit + length;
1666
1667	/* Check if rbtree root looks promising */
1668	if (RB_EMPTY_ROOT(&mm->mm_rb))
1669	goto check_highest;
1670	vma = rb_entry(mm->mm_rb.rb_node, struct vm_area_struct, vm_rb);
1671	if (vma->rb_subtree_gap < length)
1672	goto check_highest;
1673
1674	while (true) {
1675	/* Visit left subtree if it looks promising */
1676	gap_end = vma->vm_start;
1677	if (gap_end >= low_limit && vma->vm_rb.rb_left) {
1678	struct vm_area_struct *left =
1679	rb_entry(vma->vm_rb.rb_left,
1680	struct vm_area_struct, vm_rb);
1681	if (left->rb_subtree_gap >= length) {
1682	vma = left;
1683	continue;
1684	}
1685	}
1686
1687	gap_start = vma->vm_prev ? vma->vm_prev->vm_end : 0;
1688	check_current:
1689	/* Check if current node has a suitable gap */
1690	if (gap_start > high_limit)
1691	return -ENOMEM;
1692	if (gap_end >= low_limit && gap_end - gap_start >= length)
1693	goto found;
1694
1695	/* Visit right subtree if it looks promising */
1696	if (vma->vm_rb.rb_right) {
1697	struct vm_area_struct *right =
1698	rb_entry(vma->vm_rb.rb_right,
1699	struct vm_area_struct, vm_rb);
1700	if (right->rb_subtree_gap >= length) {
1701	vma = right;
1702	continue;
1703	}
1704	}
1705
1706	/* Go back up the rbtree to find next candidate node */
1707	while (true) {
1708	struct rb_node *prev = &vma->vm_rb;
1709	if (!rb_parent(prev))
1710	goto check_highest;
1711	vma = rb_entry(rb_parent(prev),
1712	struct vm_area_struct, vm_rb);
1713	if (prev == vma->vm_rb.rb_left) {
1714	gap_start = vma->vm_prev->vm_end;
1715	gap_end = vma->vm_start;
1716	goto check_current;
1717	}
1718	}
1719	}
1720
1721	check_highest:
1722	/* Check highest gap, which does not precede any rbtree node */
1723	gap_start = mm->highest_vm_end;
1724	gap_end = ULONG_MAX; /* Only for VM_BUG_ON below */
1725	if (gap_start > high_limit)
1726	return -ENOMEM;
1727
1728	found:
1729	/* We found a suitable gap. Clip it with the original low_limit. */
1730	if (gap_start < info->low_limit)
1731	gap_start = info->low_limit;
1732
1733	/* Adjust gap address to the desired alignment */
1734	gap_start += (info->align_offset - gap_start) & info->align_mask;
1735
1736	VM_BUG_ON(gap_start + info->length > info->high_limit);
1737	VM_BUG_ON(gap_start + info->length > gap_end);
1738	return gap_start;
1739	}
1740
1741	unsigned long unmapped_area_topdown(struct vm_unmapped_area_info *info)
1742	{
1743	struct mm_struct *mm = current->mm;
1744	struct vm_area_struct *vma;
1745	unsigned long length, low_limit, high_limit, gap_start, gap_end;
1746
1747	/* Adjust search length to account for worst case alignment overhead */
1748	length = info->length + info->align_mask;
1749	if (length < info->length)
1750	return -ENOMEM;
1751
1752	/*
1753	* Adjust search limits by the desired length.
1754	* See implementation comment at top of unmapped_area().
1755	*/
1756	gap_end = info->high_limit;
1757	if (gap_end < length)
1758	return -ENOMEM;
1759	high_limit = gap_end - length;
1760
1761	if (info->low_limit > high_limit)
1762	return -ENOMEM;
1763	low_limit = info->low_limit + length;
1764
1765	/* Check highest gap, which does not precede any rbtree node */
1766	gap_start = mm->highest_vm_end;
1767	if (gap_start <= high_limit)
1768	goto found_highest;
1769
1770	/* Check if rbtree root looks promising */
1771	if (RB_EMPTY_ROOT(&mm->mm_rb))
1772	return -ENOMEM;
1773	vma = rb_entry(mm->mm_rb.rb_node, struct vm_area_struct, vm_rb);
1774	if (vma->rb_subtree_gap < length)
1775	return -ENOMEM;
1776
1777	while (true) {
1778	/* Visit right subtree if it looks promising */
1779	gap_start = vma->vm_prev ? vma->vm_prev->vm_end : 0;
1780	if (gap_start <= high_limit && vma->vm_rb.rb_right) {
1781	struct vm_area_struct *right =
1782	rb_entry(vma->vm_rb.rb_right,
1783	struct vm_area_struct, vm_rb);
1784	if (right->rb_subtree_gap >= length) {
1785	vma = right;
1786	continue;
1787	}
1788	}
1789
1790	check_current:
1791	/* Check if current node has a suitable gap */
1792	gap_end = vma->vm_start;
1793	if (gap_end < low_limit)
1794	return -ENOMEM;
1795	if (gap_start <= high_limit && gap_end - gap_start >= length)
1796	goto found;
1797
1798	/* Visit left subtree if it looks promising */
1799	if (vma->vm_rb.rb_left) {
1800	struct vm_area_struct *left =
1801	rb_entry(vma->vm_rb.rb_left,
1802	struct vm_area_struct, vm_rb);
1803	if (left->rb_subtree_gap >= length) {
1804	vma = left;
1805	continue;
1806	}
1807	}
1808
1809	/* Go back up the rbtree to find next candidate node */
1810	while (true) {
1811	struct rb_node *prev = &vma->vm_rb;
1812	if (!rb_parent(prev))
1813	return -ENOMEM;
1814	vma = rb_entry(rb_parent(prev),
1815	struct vm_area_struct, vm_rb);
1816	if (prev == vma->vm_rb.rb_right) {
1817	gap_start = vma->vm_prev ?
1818	vma->vm_prev->vm_end : 0;
1819	goto check_current;
1820	}
1821	}
1822	}
1823
1824	found:
1825	/* We found a suitable gap. Clip it with the original high_limit. */
1826	if (gap_end > info->high_limit)
1827	gap_end = info->high_limit;
1828
1829	found_highest:
1830	/* Compute highest gap address at the desired alignment */
1831	gap_end -= info->length;
1832	gap_end -= (gap_end - info->align_offset) & info->align_mask;
1833
1834	VM_BUG_ON(gap_end < info->low_limit);
1835	VM_BUG_ON(gap_end < gap_start);
1836	return gap_end;
1837	}
1838
1839	/* Get an address range which is currently unmapped.
1840	* For shmat() with addr=0.
1841	*
1842	* Ugly calling convention alert:
1843	* Return value with the low bits set means error value,
1844	* ie
1845	* if (ret & ~PAGE_MASK)
1846	* error = ret;
1847	*
1848	* This function "knows" that -ENOMEM has the bits set.
1849	*/
1850	#ifndef HAVE_ARCH_UNMAPPED_AREA
1851	unsigned long
1852	arch_get_unmapped_area(struct file *filp, unsigned long addr,
1853	unsigned long len, unsigned long pgoff, unsigned long flags)
1854	{
1855	struct mm_struct *mm = current->mm;
1856	struct vm_area_struct *vma;
1857	struct vm_unmapped_area_info info;
1858
1859	if (len > TASK_SIZE)
1860	return -ENOMEM;
1861
1862	if (flags & MAP_FIXED)
1863	return addr;
1864
1865	if (addr) {
1866	addr = PAGE_ALIGN(addr);
1867	vma = find_vma(mm, addr);
1868	if (TASK_SIZE - len >= addr &&
1869	(!vma \|\| addr + len <= vma->vm_start))
1870	return addr;
1871	}
1872
1873	info.flags = 0;
1874	info.length = len;
1875	info.low_limit = TASK_UNMAPPED_BASE;
1876	info.high_limit = TASK_SIZE;
1877	info.align_mask = 0;
1878	return vm_unmapped_area(&info);
1879	}
1880	#endif
1881
1882	/*
1883	* This mmap-allocator allocates new areas top-down from below the
1884	* stack's low limit (the base):
1885	*/
1886	#ifndef HAVE_ARCH_UNMAPPED_AREA_TOPDOWN
1887	unsigned long
1888	arch_get_unmapped_area_topdown(struct file *filp, const unsigned long addr0,
1889	const unsigned long len, const unsigned long pgoff,
1890	const unsigned long flags)
1891	{
1892	struct vm_area_struct *vma;
1893	struct mm_struct *mm = current->mm;
1894	unsigned long addr = addr0;
1895	struct vm_unmapped_area_info info;
1896
1897	/* requested length too big for entire address space */
1898	if (len > TASK_SIZE)
1899	return -ENOMEM;
1900
1901	if (flags & MAP_FIXED)
1902	return addr;
1903
1904	/* requesting a specific address */
1905	if (addr) {
1906	addr = PAGE_ALIGN(addr);
1907	vma = find_vma(mm, addr);
1908	if (TASK_SIZE - len >= addr &&
1909	(!vma \|\| addr + len <= vma->vm_start))
1910	return addr;
1911	}
1912
1913	info.flags = VM_UNMAPPED_AREA_TOPDOWN;
1914	info.length = len;
1915	info.low_limit = PAGE_SIZE;
1916	info.high_limit = mm->mmap_base;
1917	info.align_mask = 0;
1918	addr = vm_unmapped_area(&info);
1919
1920	/*
1921	* A failed mmap() very likely causes application failure,
1922	* so fall back to the bottom-up function here. This scenario
1923	* can happen with large stack limits and large mmap()
1924	* allocations.
1925	*/
1926	if (addr & ~PAGE_MASK) {
1927	VM_BUG_ON(addr != -ENOMEM);
1928	info.flags = 0;
1929	info.low_limit = TASK_UNMAPPED_BASE;
1930	info.high_limit = TASK_SIZE;
1931	addr = vm_unmapped_area(&info);
1932	}
1933
1934	return addr;
1935	}
1936	#endif
1937
1938	unsigned long
1939	get_unmapped_area(struct file *file, unsigned long addr, unsigned long len,
1940	unsigned long pgoff, unsigned long flags)
1941	{
1942	unsigned long (get_area)(struct file , unsigned long,
1943	unsigned long, unsigned long, unsigned long);
1944
1945	unsigned long error = arch_mmap_check(addr, len, flags);
1946	if (error)
1947	return error;
1948
1949	/* Careful about overflows.. */
1950	if (len > TASK_SIZE)
1951	return -ENOMEM;
1952
1953	get_area = current->mm->get_unmapped_area;
1954	if (file && file->f_op && file->f_op->get_unmapped_area)
1955	get_area = file->f_op->get_unmapped_area;
1956	addr = get_area(file, addr, len, pgoff, flags);
1957	if (IS_ERR_VALUE(addr))
1958	return addr;
1959
1960	if (addr > TASK_SIZE - len)
1961	return -ENOMEM;
1962	if (addr & ~PAGE_MASK)
1963	return -EINVAL;
1964
1965	addr = arch_rebalance_pgtables(addr, len);
1966	error = security_mmap_addr(addr);
1967	return error ? error : addr;
1968	}
1969
1970	EXPORT_SYMBOL(get_unmapped_area);
1971
1972	/* Look up the first VMA which satisfies addr < vm_end, NULL if none. */
1973	struct vm_area_struct find_vma(struct mm_struct mm, unsigned long addr)
1974	{
1975	struct vm_area_struct *vma = NULL;
1976
1977	/* Check the cache first. */
1978	/* (Cache hit rate is typically around 35%.) */
1979	vma = ACCESS_ONCE(mm->mmap_cache);
1980	if (!(vma && vma->vm_end > addr && vma->vm_start <= addr)) {
1981	struct rb_node *rb_node;
1982
1983	rb_node = mm->mm_rb.rb_node;
1984	vma = NULL;
1985
1986	while (rb_node) {
1987	struct vm_area_struct *vma_tmp;
1988
1989	vma_tmp = rb_entry(rb_node,
1990	struct vm_area_struct, vm_rb);
1991
1992	if (vma_tmp->vm_end > addr) {
1993	vma = vma_tmp;
1994	if (vma_tmp->vm_start <= addr)
1995	break;
1996	rb_node = rb_node->rb_left;
1997	} else
1998	rb_node = rb_node->rb_right;
1999	}
2000	if (vma)
2001	mm->mmap_cache = vma;
2002	}
2003	return vma;
2004	}
2005
2006	EXPORT_SYMBOL(find_vma);
2007
2008	/*
2009	* Same as find_vma, but also return a pointer to the previous VMA in *pprev.
2010	*/
2011	struct vm_area_struct *
2012	find_vma_prev(struct mm_struct *mm, unsigned long addr,
2013	struct vm_area_struct **pprev)
2014	{
2015	struct vm_area_struct *vma;
2016
2017	vma = find_vma(mm, addr);
2018	if (vma) {
2019	*pprev = vma->vm_prev;
2020	} else {
2021	struct rb_node *rb_node = mm->mm_rb.rb_node;
2022	*pprev = NULL;
2023	while (rb_node) {
2024	*pprev = rb_entry(rb_node, struct vm_area_struct, vm_rb);
2025	rb_node = rb_node->rb_right;
2026	}
2027	}
2028	return vma;
2029	}
2030
2031	/*
2032	* Verify that the stack growth is acceptable and
2033	* update accounting. This is shared with both the
2034	* grow-up and grow-down cases.
2035	*/
2036	static int acct_stack_growth(struct vm_area_struct *vma, unsigned long size, unsigned long grow)
2037	{
2038	struct mm_struct *mm = vma->vm_mm;
2039	struct rlimit *rlim = current->signal->rlim;
2040	unsigned long new_start;
2041
2042	/* address space limit tests */
2043	if (!may_expand_vm(mm, grow))
2044	return -ENOMEM;
2045
2046	/* Stack limit test */
2047	if (size > ACCESS_ONCE(rlim[RLIMIT_STACK].rlim_cur))
2048	return -ENOMEM;
2049
2050	/* mlock limit tests */
2051	if (vma->vm_flags & VM_LOCKED) {
2052	unsigned long locked;
2053	unsigned long limit;
2054	locked = mm->locked_vm + grow;
2055	limit = ACCESS_ONCE(rlim[RLIMIT_MEMLOCK].rlim_cur);
2056	limit >>= PAGE_SHIFT;
2057	if (locked > limit && !capable(CAP_IPC_LOCK))
2058	return -ENOMEM;
2059	}
2060
2061	/* Check to ensure the stack will not grow into a hugetlb-only region */
2062	new_start = (vma->vm_flags & VM_GROWSUP) ? vma->vm_start :
2063	vma->vm_end - size;
2064	if (is_hugepage_only_range(vma->vm_mm, new_start, size))
2065	return -EFAULT;
2066
2067	/*
2068	* Overcommit.. This must be the final test, as it will
2069	* update security statistics.
2070	*/
2071	if (security_vm_enough_memory_mm(mm, grow))
2072	return -ENOMEM;
2073
2074	/* Ok, everything looks good - let it rip */
2075	if (vma->vm_flags & VM_LOCKED)
2076	mm->locked_vm += grow;
2077	vm_stat_account(mm, vma->vm_flags, vma->vm_file, grow);
2078	return 0;
2079	}
2080
2081	#if defined(CONFIG_STACK_GROWSUP) \|\| defined(CONFIG_IA64)
2082	/*
2083	* PA-RISC uses this for its stack; IA64 for its Register Backing Store.
2084	* vma is the last one with address > vma->vm_end. Have to extend vma.
2085	*/
2086	int expand_upwards(struct vm_area_struct *vma, unsigned long address)
2087	{
2088	int error;
2089
2090	if (!(vma->vm_flags & VM_GROWSUP))
2091	return -EFAULT;
2092
2093	/*
2094	* We must make sure the anon_vma is allocated
2095	* so that the anon_vma locking is not a noop.
2096	*/
2097	if (unlikely(anon_vma_prepare(vma)))
2098	return -ENOMEM;
2099	vma_lock_anon_vma(vma);
2100
2101	/*
2102	* vma->vm_start/vm_end cannot change under us because the caller
2103	* is required to hold the mmap_sem in read mode. We need the
2104	* anon_vma lock to serialize against concurrent expand_stacks.
2105	* Also guard against wrapping around to address 0.
2106	*/
2107	if (address < PAGE_ALIGN(address+4))
2108	address = PAGE_ALIGN(address+4);
2109	else {
2110	vma_unlock_anon_vma(vma);
2111	return -ENOMEM;
2112	}
2113	error = 0;
2114
2115	/* Somebody else might have raced and expanded it already */
2116	if (address > vma->vm_end) {
2117	unsigned long size, grow;
2118
2119	size = address - vma->vm_start;
2120	grow = (address - vma->vm_end) >> PAGE_SHIFT;
2121
2122	error = -ENOMEM;
2123	if (vma->vm_pgoff + (size >> PAGE_SHIFT) >= vma->vm_pgoff) {
2124	error = acct_stack_growth(vma, size, grow);
2125	if (!error) {
2126	/*
2127	* vma_gap_update() doesn't support concurrent
2128	* updates, but we only hold a shared mmap_sem
2129	* lock here, so we need to protect against
2130	* concurrent vma expansions.
2131	* vma_lock_anon_vma() doesn't help here, as
2132	* we don't guarantee that all growable vmas
2133	* in a mm share the same root anon vma.
2134	* So, we reuse mm->page_table_lock to guard
2135	* against concurrent vma expansions.
2136	*/
2137	spin_lock(&vma->vm_mm->page_table_lock);
2138	anon_vma_interval_tree_pre_update_vma(vma);
2139	vma->vm_end = address;
2140	anon_vma_interval_tree_post_update_vma(vma);
2141	if (vma->vm_next)
2142	vma_gap_update(vma->vm_next);
2143	else
2144	vma->vm_mm->highest_vm_end = address;
2145	spin_unlock(&vma->vm_mm->page_table_lock);
2146
2147	perf_event_mmap(vma);
2148	}
2149	}
2150	}
2151	vma_unlock_anon_vma(vma);
2152	khugepaged_enter_vma_merge(vma);
2153	validate_mm(vma->vm_mm);
2154	return error;
2155	}
2156	#endif /* CONFIG_STACK_GROWSUP \|\| CONFIG_IA64 */
2157
2158	/*
2159	* vma is the first one with address < vma->vm_start. Have to extend vma.
2160	*/
2161	int expand_downwards(struct vm_area_struct *vma,
2162	unsigned long address)
2163	{
2164	int error;
2165
2166	/*
2167	* We must make sure the anon_vma is allocated
2168	* so that the anon_vma locking is not a noop.
2169	*/
2170	if (unlikely(anon_vma_prepare(vma)))
2171	return -ENOMEM;
2172
2173	address &= PAGE_MASK;
2174	error = security_mmap_addr(address);
2175	if (error)
2176	return error;
2177
2178	vma_lock_anon_vma(vma);
2179
2180	/*
2181	* vma->vm_start/vm_end cannot change under us because the caller
2182	* is required to hold the mmap_sem in read mode. We need the
2183	* anon_vma lock to serialize against concurrent expand_stacks.
2184	*/
2185
2186	/* Somebody else might have raced and expanded it already */
2187	if (address < vma->vm_start) {
2188	unsigned long size, grow;
2189
2190	size = vma->vm_end - address;
2191	grow = (vma->vm_start - address) >> PAGE_SHIFT;
2192
2193	error = -ENOMEM;
2194	if (grow <= vma->vm_pgoff) {
2195	error = acct_stack_growth(vma, size, grow);
2196	if (!error) {
2197	/*
2198	* vma_gap_update() doesn't support concurrent
2199	* updates, but we only hold a shared mmap_sem
2200	* lock here, so we need to protect against
2201	* concurrent vma expansions.
2202	* vma_lock_anon_vma() doesn't help here, as
2203	* we don't guarantee that all growable vmas
2204	* in a mm share the same root anon vma.
2205	* So, we reuse mm->page_table_lock to guard
2206	* against concurrent vma expansions.
2207	*/
2208	spin_lock(&vma->vm_mm->page_table_lock);
2209	anon_vma_interval_tree_pre_update_vma(vma);
2210	vma->vm_start = address;
2211	vma->vm_pgoff -= grow;
2212	anon_vma_interval_tree_post_update_vma(vma);
2213	vma_gap_update(vma);
2214	spin_unlock(&vma->vm_mm->page_table_lock);
2215
2216	perf_event_mmap(vma);
2217	}
2218	}
2219	}
2220	vma_unlock_anon_vma(vma);
2221	khugepaged_enter_vma_merge(vma);
2222	validate_mm(vma->vm_mm);
2223	return error;
2224	}
2225
2226	/*
2227	* Note how expand_stack() refuses to expand the stack all the way to
2228	* abut the next virtual mapping, unless that mapping itself is also
2229	* a stack mapping. We want to leave room for a guard page, after all
2230	* (the guard page itself is not added here, that is done by the
2231	* actual page faulting logic)
2232	*
2233	* This matches the behavior of the guard page logic (see mm/memory.c:
2234	* check_stack_guard_page()), which only allows the guard page to be
2235	* removed under these circumstances.
2236	*/
2237	#ifdef CONFIG_STACK_GROWSUP
2238	int expand_stack(struct vm_area_struct *vma, unsigned long address)
2239	{
2240	struct vm_area_struct *next;
2241
2242	address &= PAGE_MASK;
2243	next = vma->vm_next;
2244	if (next && next->vm_start == address + PAGE_SIZE) {
2245	if (!(next->vm_flags & VM_GROWSUP))
2246	return -ENOMEM;
2247	}
2248	return expand_upwards(vma, address);
2249	}
2250
2251	struct vm_area_struct *
2252	find_extend_vma(struct mm_struct *mm, unsigned long addr)
2253	{
2254	struct vm_area_struct vma, prev;
2255
2256	addr &= PAGE_MASK;
2257	vma = find_vma_prev(mm, addr, &prev);
2258	if (vma && (vma->vm_start <= addr))
2259	return vma;
2260	if (!prev \|\| expand_stack(prev, addr))
2261	return NULL;
2262	if (prev->vm_flags & VM_LOCKED)
2263	__mlock_vma_pages_range(prev, addr, prev->vm_end, NULL);
2264	return prev;
2265	}
2266	#else
2267	int expand_stack(struct vm_area_struct *vma, unsigned long address)
2268	{
2269	struct vm_area_struct *prev;
2270
2271	address &= PAGE_MASK;
2272	prev = vma->vm_prev;
2273	if (prev && prev->vm_end == address) {
2274	if (!(prev->vm_flags & VM_GROWSDOWN))
2275	return -ENOMEM;
2276	}
2277	return expand_downwards(vma, address);
2278	}
2279
2280	struct vm_area_struct *
2281	find_extend_vma(struct mm_struct * mm, unsigned long addr)
2282	{
2283	struct vm_area_struct * vma;
2284	unsigned long start;
2285
2286	addr &= PAGE_MASK;
2287	vma = find_vma(mm,addr);
2288	if (!vma)
2289	return NULL;
2290	if (vma->vm_start <= addr)
2291	return vma;
2292	if (!(vma->vm_flags & VM_GROWSDOWN))
2293	return NULL;
2294	start = vma->vm_start;
2295	if (expand_stack(vma, addr))
2296	return NULL;
2297	if (vma->vm_flags & VM_LOCKED)
2298	__mlock_vma_pages_range(vma, addr, start, NULL);
2299	return vma;
2300	}
2301	#endif
2302
2303	/*
2304	* Ok - we have the memory areas we should free on the vma list,
2305	* so release them, and do the vma updates.
2306	*
2307	* Called with the mm semaphore held.
2308	*/
2309	static void remove_vma_list(struct mm_struct mm, struct vm_area_struct vma)
2310	{
2311	unsigned long nr_accounted = 0;
2312
2313	/* Update high watermark before we lower total_vm */
2314	update_hiwater_vm(mm);
2315	do {
2316	long nrpages = vma_pages(vma);
2317
2318	if (vma->vm_flags & VM_ACCOUNT)
2319	nr_accounted += nrpages;
2320	vm_stat_account(mm, vma->vm_flags, vma->vm_file, -nrpages);
2321	vma = remove_vma(vma);
2322	} while (vma);
2323	vm_unacct_memory(nr_accounted);
2324	validate_mm(mm);
2325	}
2326
2327	/*
2328	* Get rid of page table information in the indicated region.
2329	*
2330	* Called with the mm semaphore held.
2331	*/
2332	static void unmap_region(struct mm_struct *mm,
2333	struct vm_area_struct vma, struct vm_area_struct prev,
2334	unsigned long start, unsigned long end)
2335	{
2336	struct vm_area_struct *next = prev? prev->vm_next: mm->mmap;
2337	struct mmu_gather tlb;
2338
2339	lru_add_drain();
2340	tlb_gather_mmu(&tlb, mm, start, end);
2341	update_hiwater_rss(mm);
2342	unmap_vmas(&tlb, vma, start, end);
2343	free_pgtables(&tlb, vma, prev ? prev->vm_end : FIRST_USER_ADDRESS,
2344	next ? next->vm_start : USER_PGTABLES_CEILING);
2345	tlb_finish_mmu(&tlb, start, end);
2346	}
2347
2348	/*
2349	* Create a list of vma's touched by the unmap, removing them from the mm's
2350	* vma list as we go..
2351	*/
2352	static void
2353	detach_vmas_to_be_unmapped(struct mm_struct mm, struct vm_area_struct vma,
2354	struct vm_area_struct *prev, unsigned long end)
2355	{
2356	struct vm_area_struct **insertion_point;
2357	struct vm_area_struct *tail_vma = NULL;
2358
2359	insertion_point = (prev ? &prev->vm_next : &mm->mmap);
2360	vma->vm_prev = NULL;
2361	do {
2362	vma_rb_erase(vma, &mm->mm_rb);
2363	mm->map_count--;
2364	tail_vma = vma;
2365	vma = vma->vm_next;
2366	} while (vma && vma->vm_start < end);
2367	*insertion_point = vma;
2368	if (vma) {
2369	vma->vm_prev = prev;
2370	vma_gap_update(vma);
2371	} else
2372	mm->highest_vm_end = prev ? prev->vm_end : 0;
2373	tail_vma->vm_next = NULL;
2374	mm->mmap_cache = NULL; /* Kill the cache. */
2375	}
2376
2377	/*
2378	* __split_vma() bypasses sysctl_max_map_count checking. We use this on the
2379	* munmap path where it doesn't make sense to fail.
2380	*/
2381	static int __split_vma(struct mm_struct * mm, struct vm_area_struct * vma,
2382	unsigned long addr, int new_below)
2383	{
2384	struct vm_area_struct *new;
2385	int err = -ENOMEM;
2386
2387	if (is_vm_hugetlb_page(vma) && (addr &
2388	~(huge_page_mask(hstate_vma(vma)))))
2389	return -EINVAL;
2390
2391	new = kmem_cache_alloc(vm_area_cachep, GFP_KERNEL);
2392	if (!new)
2393	goto out_err;
2394
2395	/* most fields are the same, copy all, and then fixup */
2396	new = vma;
2397
2398	INIT_LIST_HEAD(&new->anon_vma_chain);
2399
2400	if (new_below)
2401	new->vm_end = addr;
2402	else {
2403	new->vm_start = addr;
2404	new->vm_pgoff += ((addr - vma->vm_start) >> PAGE_SHIFT);
2405	}
2406
2407	err = vma_dup_policy(vma, new);
2408	if (err)
2409	goto out_free_vma;
2410
2411	if (anon_vma_clone(new, vma))
2412	goto out_free_mpol;
2413
2414	if (new->vm_file)
2415	get_file(new->vm_file);
2416
2417	if (new->vm_ops && new->vm_ops->open)
2418	new->vm_ops->open(new);
2419
2420	if (new_below)
2421	err = vma_adjust(vma, addr, vma->vm_end, vma->vm_pgoff +
2422	((addr - new->vm_start) >> PAGE_SHIFT), new);
2423	else
2424	err = vma_adjust(vma, vma->vm_start, addr, vma->vm_pgoff, new);
2425
2426	/* Success. */
2427	if (!err)
2428	return 0;
2429
2430	/* Clean everything up if vma_adjust failed. */
2431	if (new->vm_ops && new->vm_ops->close)
2432	new->vm_ops->close(new);
2433	if (new->vm_file)
2434	fput(new->vm_file);
2435	unlink_anon_vmas(new);
2436	out_free_mpol:
2437	mpol_put(vma_policy(new));
2438	out_free_vma:
2439	kmem_cache_free(vm_area_cachep, new);
2440	out_err:
2441	return err;
2442	}
2443
2444	/*
2445	* Split a vma into two pieces at address 'addr', a new vma is allocated
2446	* either for the first part or the tail.
2447	*/
2448	int split_vma(struct mm_struct mm, struct vm_area_struct vma,
2449	unsigned long addr, int new_below)
2450	{
2451	if (mm->map_count >= sysctl_max_map_count)
2452	return -ENOMEM;
2453
2454	return __split_vma(mm, vma, addr, new_below);
2455	}
2456
2457	/* Munmap is split into 2 main parts -- this part which finds
2458	* what needs doing, and the areas themselves, which do the
2459	* work. This now handles partial unmappings.
2460	* Jeremy Fitzhardinge <jeremy@goop.org>
2461	*/
2462	int do_munmap(struct mm_struct *mm, unsigned long start, size_t len)
2463	{
2464	unsigned long end;
2465	struct vm_area_struct vma, prev, *last;
2466
2467	if ((start & ~PAGE_MASK) \|\| start > TASK_SIZE \|\| len > TASK_SIZE-start)
2468	return -EINVAL;
2469
2470	if ((len = PAGE_ALIGN(len)) == 0)
2471	return -EINVAL;
2472
2473	/* Find the first overlapping VMA */
2474	vma = find_vma(mm, start);
2475	if (!vma)
2476	return 0;
2477	prev = vma->vm_prev;
2478	/* we have start < vma->vm_end */
2479
2480	/* if it doesn't overlap, we have nothing.. */
2481	end = start + len;
2482	if (vma->vm_start >= end)
2483	return 0;
2484
2485	/*
2486	* If we need to split any vma, do it now to save pain later.
2487	*
2488	* Note: mremap's move_vma VM_ACCOUNT handling assumes a partially
2489	* unmapped vm_area_struct will remain in use: so lower split_vma
2490	* places tmp vma above, and higher split_vma places tmp vma below.
2491	*/
2492	if (start > vma->vm_start) {
2493	int error;
2494
2495	/*
2496	* Make sure that map_count on return from munmap() will
2497	* not exceed its limit; but let map_count go just above
2498	* its limit temporarily, to help free resources as expected.
2499	*/
2500	if (end < vma->vm_end && mm->map_count >= sysctl_max_map_count)
2501	return -ENOMEM;
2502
2503	error = __split_vma(mm, vma, start, 0);
2504	if (error)
2505	return error;
2506	prev = vma;
2507	}
2508
2509	/* Does it split the last one? */
2510	last = find_vma(mm, end);
2511	if (last && end > last->vm_start) {
2512	int error = __split_vma(mm, last, end, 1);
2513	if (error)
2514	return error;
2515	}
2516	vma = prev? prev->vm_next: mm->mmap;
2517
2518	/*
2519	* unlock any mlock()ed ranges before detaching vmas
2520	*/
2521	if (mm->locked_vm) {
2522	struct vm_area_struct *tmp = vma;
2523	while (tmp && tmp->vm_start < end) {
2524	if (tmp->vm_flags & VM_LOCKED) {
2525	mm->locked_vm -= vma_pages(tmp);
2526	munlock_vma_pages_all(tmp);
2527	}
2528	tmp = tmp->vm_next;
2529	}
2530	}
2531
2532	/*
2533	* Remove the vma's, and unmap the actual pages
2534	*/
2535	detach_vmas_to_be_unmapped(mm, vma, prev, end);
2536	unmap_region(mm, vma, prev, start, end);
2537
2538	/* Fix up all other VM information */
2539	remove_vma_list(mm, vma);
2540
2541	return 0;
2542	}
2543
2544	int vm_munmap(unsigned long start, size_t len)
2545	{
2546	int ret;
2547	struct mm_struct *mm = current->mm;
2548
2549	down_write(&mm->mmap_sem);
2550	ret = do_munmap(mm, start, len);
2551	up_write(&mm->mmap_sem);
2552	return ret;
2553	}
2554	EXPORT_SYMBOL(vm_munmap);
2555
2556	SYSCALL_DEFINE2(munmap, unsigned long, addr, size_t, len)
2557	{
2558	profile_munmap(addr);
2559	return vm_munmap(addr, len);
2560	}
2561
2562	static inline void verify_mm_writelocked(struct mm_struct *mm)
2563	{
2564	#ifdef CONFIG_DEBUG_VM
2565	if (unlikely(down_read_trylock(&mm->mmap_sem))) {
2566	WARN_ON(1);
2567	up_read(&mm->mmap_sem);
2568	}
2569	#endif
2570	}
2571
2572	/*
2573	* this is really a simplified "do_mmap". it only handles
2574	* anonymous maps. eventually we may be able to do some
2575	* brk-specific accounting here.
2576	*/
2577	static unsigned long do_brk(unsigned long addr, unsigned long len)
2578	{
2579	struct mm_struct * mm = current->mm;
2580	struct vm_area_struct * vma, * prev;
2581	unsigned long flags;
2582	struct rb_node ** rb_link, * rb_parent;
2583	pgoff_t pgoff = addr >> PAGE_SHIFT;
2584	int error;
2585
2586	len = PAGE_ALIGN(len);
2587	if (!len)
2588	return addr;
2589
2590	flags = VM_DATA_DEFAULT_FLAGS \| VM_ACCOUNT \| mm->def_flags;
2591
2592	error = get_unmapped_area(NULL, addr, len, 0, MAP_FIXED);
2593	if (error & ~PAGE_MASK)
2594	return error;
2595
2596	/*
2597	* mlock MCL_FUTURE?
2598	*/
2599	if (mm->def_flags & VM_LOCKED) {
2600	unsigned long locked, lock_limit;
2601	locked = len >> PAGE_SHIFT;
2602	locked += mm->locked_vm;
2603	lock_limit = rlimit(RLIMIT_MEMLOCK);
2604	lock_limit >>= PAGE_SHIFT;
2605	if (locked > lock_limit && !capable(CAP_IPC_LOCK))
2606	return -EAGAIN;
2607	}
2608
2609	/*
2610	* mm->mmap_sem is required to protect against another thread
2611	* changing the mappings in case we sleep.
2612	*/
2613	verify_mm_writelocked(mm);
2614
2615	/*
2616	* Clear old maps. this also does some error checking for us
2617	*/
2618	munmap_back:
2619	if (find_vma_links(mm, addr, addr + len, &prev, &rb_link, &rb_parent)) {
2620	if (do_munmap(mm, addr, len))
2621	return -ENOMEM;
2622	goto munmap_back;
2623	}
2624
2625	/* Check against address space limits after clearing old maps... */
2626	if (!may_expand_vm(mm, len >> PAGE_SHIFT))
2627	return -ENOMEM;
2628
2629	if (mm->map_count > sysctl_max_map_count)
2630	return -ENOMEM;
2631
2632	if (security_vm_enough_memory_mm(mm, len >> PAGE_SHIFT))
2633	return -ENOMEM;
2634
2635	/* Can we just expand an old private anonymous mapping? */
2636	vma = vma_merge(mm, prev, addr, addr + len, flags,
2637	NULL, NULL, pgoff, NULL);
2638	if (vma)
2639	goto out;
2640
2641	/*
2642	* create a vma struct for an anonymous mapping
2643	*/
2644	vma = kmem_cache_zalloc(vm_area_cachep, GFP_KERNEL);
2645	if (!vma) {
2646	vm_unacct_memory(len >> PAGE_SHIFT);
2647	return -ENOMEM;
2648	}
2649
2650	INIT_LIST_HEAD(&vma->anon_vma_chain);
2651	vma->vm_mm = mm;
2652	vma->vm_start = addr;
2653	vma->vm_end = addr + len;
2654	vma->vm_pgoff = pgoff;
2655	vma->vm_flags = flags;
2656	vma->vm_page_prot = vm_get_page_prot(flags);
2657	vma_link(mm, vma, prev, rb_link, rb_parent);
2658	out:
2659	perf_event_mmap(vma);
2660	mm->total_vm += len >> PAGE_SHIFT;
2661	if (flags & VM_LOCKED)
2662	mm->locked_vm += (len >> PAGE_SHIFT);
2663	vma->vm_flags \|= VM_SOFTDIRTY;
2664	return addr;
2665	}
2666
2667	unsigned long vm_brk(unsigned long addr, unsigned long len)
2668	{
2669	struct mm_struct *mm = current->mm;
2670	unsigned long ret;
2671	bool populate;
2672
2673	down_write(&mm->mmap_sem);
2674	ret = do_brk(addr, len);
2675	populate = ((mm->def_flags & VM_LOCKED) != 0);
2676	up_write(&mm->mmap_sem);
2677	if (populate)
2678	mm_populate(addr, len);
2679	return ret;
2680	}
2681	EXPORT_SYMBOL(vm_brk);
2682
2683	/* Release all mmaps. */
2684	void exit_mmap(struct mm_struct *mm)
2685	{
2686	struct mmu_gather tlb;
2687	struct vm_area_struct *vma;
2688	unsigned long nr_accounted = 0;
2689
2690	/* mm's last user has gone, and its about to be pulled down */
2691	mmu_notifier_release(mm);
2692
2693	if (mm->locked_vm) {
2694	vma = mm->mmap;
2695	while (vma) {
2696	if (vma->vm_flags & VM_LOCKED)
2697	munlock_vma_pages_all(vma);
2698	vma = vma->vm_next;
2699	}
2700	}
2701
2702	arch_exit_mmap(mm);
2703
2704	vma = mm->mmap;
2705	if (!vma) /* Can happen if dup_mmap() received an OOM */
2706	return;
2707
2708	lru_add_drain();
2709	flush_cache_mm(mm);
2710	tlb_gather_mmu(&tlb, mm, 0, -1);
2711	/* update_hiwater_rss(mm) here? but nobody should be looking */
2712	/* Use -1 here to ensure all VMAs in the mm are unmapped */
2713	unmap_vmas(&tlb, vma, 0, -1);
2714
2715	free_pgtables(&tlb, vma, FIRST_USER_ADDRESS, USER_PGTABLES_CEILING);
2716	tlb_finish_mmu(&tlb, 0, -1);
2717
2718	/*
2719	* Walk the list again, actually closing and freeing it,
2720	* with preemption enabled, without holding any MM locks.
2721	*/
2722	while (vma) {
2723	if (vma->vm_flags & VM_ACCOUNT)
2724	nr_accounted += vma_pages(vma);
2725	vma = remove_vma(vma);
2726	}
2727	vm_unacct_memory(nr_accounted);
2728
2729	WARN_ON(mm->nr_ptes > (FIRST_USER_ADDRESS+PMD_SIZE-1)>>PMD_SHIFT);
2730	}
2731
2732	/* Insert vm structure into process list sorted by address
2733	* and into the inode's i_mmap tree. If vm_file is non-NULL
2734	* then i_mmap_mutex is taken here.
2735	*/
2736	int insert_vm_struct(struct mm_struct mm, struct vm_area_struct vma)
2737	{
2738	struct vm_area_struct *prev;
2739	struct rb_node *rb_link, rb_parent;
2740
2741	/*
2742	* The vm_pgoff of a purely anonymous vma should be irrelevant
2743	* until its first write fault, when page's anon_vma and index
2744	* are set. But now set the vm_pgoff it will almost certainly
2745	* end up with (unless mremap moves it elsewhere before that
2746	* first wfault), so /proc/pid/maps tells a consistent story.
2747	*
2748	* By setting it to reflect the virtual start address of the
2749	* vma, merges and splits can happen in a seamless way, just
2750	* using the existing file pgoff checks and manipulations.
2751	* Similarly in do_mmap_pgoff and in do_brk.
2752	*/
2753	if (!vma->vm_file) {
2754	BUG_ON(vma->anon_vma);
2755	vma->vm_pgoff = vma->vm_start >> PAGE_SHIFT;
2756	}
2757	if (find_vma_links(mm, vma->vm_start, vma->vm_end,
2758	&prev, &rb_link, &rb_parent))
2759	return -ENOMEM;
2760	if ((vma->vm_flags & VM_ACCOUNT) &&
2761	security_vm_enough_memory_mm(mm, vma_pages(vma)))
2762	return -ENOMEM;
2763
2764	vma_link(mm, vma, prev, rb_link, rb_parent);
2765	return 0;
2766	}
2767
2768	/*
2769	* Copy the vma structure to a new location in the same mm,
2770	* prior to moving page table entries, to effect an mremap move.
2771	*/
2772	struct vm_area_struct copy_vma(struct vm_area_struct *vmap,
2773	unsigned long addr, unsigned long len, pgoff_t pgoff,
2774	bool *need_rmap_locks)
2775	{
2776	struct vm_area_struct vma = vmap;
2777	unsigned long vma_start = vma->vm_start;
2778	struct mm_struct *mm = vma->vm_mm;
2779	struct vm_area_struct new_vma, prev;
2780	struct rb_node *rb_link, rb_parent;
2781	bool faulted_in_anon_vma = true;
2782
2783	/*
2784	* If anonymous vma has not yet been faulted, update new pgoff
2785	* to match new location, to increase its chance of merging.
2786	*/
2787	if (unlikely(!vma->vm_file && !vma->anon_vma)) {
2788	pgoff = addr >> PAGE_SHIFT;
2789	faulted_in_anon_vma = false;
2790	}
2791
2792	if (find_vma_links(mm, addr, addr + len, &prev, &rb_link, &rb_parent))
2793	return NULL; /* should never get here */
2794	new_vma = vma_merge(mm, prev, addr, addr + len, vma->vm_flags,
2795	vma->anon_vma, vma->vm_file, pgoff, vma_policy(vma));
2796	if (new_vma) {
2797	/*
2798	* Source vma may have been merged into new_vma
2799	*/
2800	if (unlikely(vma_start >= new_vma->vm_start &&
2801	vma_start < new_vma->vm_end)) {
2802	/*
2803	* The only way we can get a vma_merge with
2804	* self during an mremap is if the vma hasn't
2805	* been faulted in yet and we were allowed to
2806	* reset the dst vma->vm_pgoff to the
2807	* destination address of the mremap to allow
2808	* the merge to happen. mremap must change the
2809	* vm_pgoff linearity between src and dst vmas
2810	* (in turn preventing a vma_merge) to be
2811	* safe. It is only safe to keep the vm_pgoff
2812	* linear if there are no pages mapped yet.
2813	*/
2814	VM_BUG_ON(faulted_in_anon_vma);
2815	*vmap = vma = new_vma;
2816	}
2817	*need_rmap_locks = (new_vma->vm_pgoff <= vma->vm_pgoff);
2818	} else {
2819	new_vma = kmem_cache_alloc(vm_area_cachep, GFP_KERNEL);
2820	if (new_vma) {
2821	new_vma = vma;
2822	new_vma->vm_start = addr;
2823	new_vma->vm_end = addr + len;
2824	new_vma->vm_pgoff = pgoff;
2825	if (vma_dup_policy(vma, new_vma))
2826	goto out_free_vma;
2827	INIT_LIST_HEAD(&new_vma->anon_vma_chain);
2828	if (anon_vma_clone(new_vma, vma))
2829	goto out_free_mempol;
2830	if (new_vma->vm_file)
2831	get_file(new_vma->vm_file);
2832	if (new_vma->vm_ops && new_vma->vm_ops->open)
2833	new_vma->vm_ops->open(new_vma);
2834	vma_link(mm, new_vma, prev, rb_link, rb_parent);
2835	*need_rmap_locks = false;
2836	}
2837	}
2838	return new_vma;
2839
2840	out_free_mempol:
2841	mpol_put(vma_policy(new_vma));
2842	out_free_vma:
2843	kmem_cache_free(vm_area_cachep, new_vma);
2844	return NULL;
2845	}
2846
2847	/*
2848	* Return true if the calling process may expand its vm space by the passed
2849	* number of pages
2850	*/
2851	int may_expand_vm(struct mm_struct *mm, unsigned long npages)
2852	{
2853	unsigned long cur = mm->total_vm; /* pages */
2854	unsigned long lim;
2855
2856	lim = rlimit(RLIMIT_AS) >> PAGE_SHIFT;
2857
2858	if (cur + npages > lim)
2859	return 0;
2860	return 1;
2861	}
2862
2863
2864	static int special_mapping_fault(struct vm_area_struct *vma,
2865	struct vm_fault *vmf)
2866	{
2867	pgoff_t pgoff;
2868	struct page **pages;
2869
2870	/*
2871	* special mappings have no vm_file, and in that case, the mm
2872	* uses vm_pgoff internally. So we have to subtract it from here.
2873	* We are allowed to do this because we are the mm; do not copy
2874	* this code into drivers!
2875	*/
2876	pgoff = vmf->pgoff - vma->vm_pgoff;
2877
2878	for (pages = vma->vm_private_data; pgoff && *pages; ++pages)
2879	pgoff--;
2880
2881	if (*pages) {
2882	struct page page = pages;
2883	get_page(page);
2884	vmf->page = page;
2885	return 0;
2886	}
2887
2888	return VM_FAULT_SIGBUS;
2889	}
2890
2891	/*
2892	* Having a close hook prevents vma merging regardless of flags.
2893	*/
2894	static void special_mapping_close(struct vm_area_struct *vma)
2895	{
2896	}
2897
2898	static const struct vm_operations_struct special_mapping_vmops = {
2899	.close = special_mapping_close,
2900	.fault = special_mapping_fault,
2901	};
2902
2903	/*
2904	* Called with mm->mmap_sem held for writing.
2905	* Insert a new vma covering the given region, with the given flags.
2906	* Its pages are supplied by the given array of struct page *.
2907	* The array can be shorter than len >> PAGE_SHIFT if it's null-terminated.
2908	* The region past the last page supplied will always produce SIGBUS.
2909	* The array pointer and the pages it points to are assumed to stay alive
2910	* for as long as this mapping might exist.
2911	*/
2912	int install_special_mapping(struct mm_struct *mm,
2913	unsigned long addr, unsigned long len,
2914	unsigned long vm_flags, struct page **pages)
2915	{
2916	int ret;
2917	struct vm_area_struct *vma;
2918
2919	vma = kmem_cache_zalloc(vm_area_cachep, GFP_KERNEL);
2920	if (unlikely(vma == NULL))
2921	return -ENOMEM;
2922
2923	INIT_LIST_HEAD(&vma->anon_vma_chain);
2924	vma->vm_mm = mm;
2925	vma->vm_start = addr;
2926	vma->vm_end = addr + len;
2927
2928	vma->vm_flags = vm_flags \| mm->def_flags \| VM_DONTEXPAND \| VM_SOFTDIRTY;
2929	vma->vm_page_prot = vm_get_page_prot(vma->vm_flags);
2930
2931	vma->vm_ops = &special_mapping_vmops;
2932	vma->vm_private_data = pages;
2933
2934	ret = insert_vm_struct(mm, vma);
2935	if (ret)
2936	goto out;
2937
2938	mm->total_vm += len >> PAGE_SHIFT;
2939
2940	perf_event_mmap(vma);
2941
2942	return 0;
2943
2944	out:
2945	kmem_cache_free(vm_area_cachep, vma);
2946	return ret;
2947	}
2948
2949	static DEFINE_MUTEX(mm_all_locks_mutex);
2950
2951	static void vm_lock_anon_vma(struct mm_struct mm, struct anon_vma anon_vma)
2952	{
2953	if (!test_bit(0, (unsigned long *) &anon_vma->root->rb_root.rb_node)) {
2954	/*
2955	* The LSB of head.next can't change from under us
2956	* because we hold the mm_all_locks_mutex.
2957	*/
2958	down_write_nest_lock(&anon_vma->root->rwsem, &mm->mmap_sem);
2959	/*
2960	* We can safely modify head.next after taking the
2961	* anon_vma->root->rwsem. If some other vma in this mm shares
2962	* the same anon_vma we won't take it again.
2963	*
2964	* No need of atomic instructions here, head.next
2965	* can't change from under us thanks to the
2966	* anon_vma->root->rwsem.
2967	*/
2968	if (__test_and_set_bit(0, (unsigned long *)
2969	&anon_vma->root->rb_root.rb_node))
2970	BUG();
2971	}
2972	}
2973
2974	static void vm_lock_mapping(struct mm_struct mm, struct address_space mapping)
2975	{
2976	if (!test_bit(AS_MM_ALL_LOCKS, &mapping->flags)) {
2977	/*
2978	* AS_MM_ALL_LOCKS can't change from under us because
2979	* we hold the mm_all_locks_mutex.
2980	*
2981	* Operations on ->flags have to be atomic because
2982	* even if AS_MM_ALL_LOCKS is stable thanks to the
2983	* mm_all_locks_mutex, there may be other cpus
2984	* changing other bitflags in parallel to us.
2985	*/
2986	if (test_and_set_bit(AS_MM_ALL_LOCKS, &mapping->flags))
2987	BUG();
2988	mutex_lock_nest_lock(&mapping->i_mmap_mutex, &mm->mmap_sem);
2989	}
2990	}
2991
2992	/*
2993	* This operation locks against the VM for all pte/vma/mm related
2994	* operations that could ever happen on a certain mm. This includes
2995	* vmtruncate, try_to_unmap, and all page faults.
2996	*
2997	* The caller must take the mmap_sem in write mode before calling
2998	* mm_take_all_locks(). The caller isn't allowed to release the
2999	* mmap_sem until mm_drop_all_locks() returns.
3000	*
3001	* mmap_sem in write mode is required in order to block all operations
3002	* that could modify pagetables and free pages without need of
3003	* altering the vma layout (for example populate_range() with
3004	* nonlinear vmas). It's also needed in write mode to avoid new
3005	* anon_vmas to be associated with existing vmas.
3006	*
3007	* A single task can't take more than one mm_take_all_locks() in a row
3008	* or it would deadlock.
3009	*
3010	* The LSB in anon_vma->rb_root.rb_node and the AS_MM_ALL_LOCKS bitflag in
3011	* mapping->flags avoid to take the same lock twice, if more than one
3012	* vma in this mm is backed by the same anon_vma or address_space.
3013	*
3014	* We can take all the locks in random order because the VM code
3015	* taking i_mmap_mutex or anon_vma->rwsem outside the mmap_sem never
3016	* takes more than one of them in a row. Secondly we're protected
3017	* against a concurrent mm_take_all_locks() by the mm_all_locks_mutex.
3018	*
3019	* mm_take_all_locks() and mm_drop_all_locks are expensive operations
3020	* that may have to take thousand of locks.
3021	*
3022	* mm_take_all_locks() can fail if it's interrupted by signals.
3023	*/
3024	int mm_take_all_locks(struct mm_struct *mm)
3025	{
3026	struct vm_area_struct *vma;
3027	struct anon_vma_chain *avc;
3028
3029	BUG_ON(down_read_trylock(&mm->mmap_sem));
3030
3031	mutex_lock(&mm_all_locks_mutex);
3032
3033	for (vma = mm->mmap; vma; vma = vma->vm_next) {
3034	if (signal_pending(current))
3035	goto out_unlock;
3036	if (vma->vm_file && vma->vm_file->f_mapping)
3037	vm_lock_mapping(mm, vma->vm_file->f_mapping);
3038	}
3039
3040	for (vma = mm->mmap; vma; vma = vma->vm_next) {
3041	if (signal_pending(current))
3042	goto out_unlock;
3043	if (vma->anon_vma)
3044	list_for_each_entry(avc, &vma->anon_vma_chain, same_vma)
3045	vm_lock_anon_vma(mm, avc->anon_vma);
3046	}
3047
3048	return 0;
3049
3050	out_unlock:
3051	mm_drop_all_locks(mm);
3052	return -EINTR;
3053	}
3054
3055	static void vm_unlock_anon_vma(struct anon_vma *anon_vma)
3056	{
3057	if (test_bit(0, (unsigned long *) &anon_vma->root->rb_root.rb_node)) {
3058	/*
3059	* The LSB of head.next can't change to 0 from under
3060	* us because we hold the mm_all_locks_mutex.
3061	*
3062	* We must however clear the bitflag before unlocking
3063	* the vma so the users using the anon_vma->rb_root will
3064	* never see our bitflag.
3065	*
3066	* No need of atomic instructions here, head.next
3067	* can't change from under us until we release the
3068	* anon_vma->root->rwsem.
3069	*/
3070	if (!__test_and_clear_bit(0, (unsigned long *)
3071	&anon_vma->root->rb_root.rb_node))
3072	BUG();
3073	anon_vma_unlock_write(anon_vma);
3074	}
3075	}
3076
3077	static void vm_unlock_mapping(struct address_space *mapping)
3078	{
3079	if (test_bit(AS_MM_ALL_LOCKS, &mapping->flags)) {
3080	/*
3081	* AS_MM_ALL_LOCKS can't change to 0 from under us
3082	* because we hold the mm_all_locks_mutex.
3083	*/
3084	mutex_unlock(&mapping->i_mmap_mutex);
3085	if (!test_and_clear_bit(AS_MM_ALL_LOCKS,
3086	&mapping->flags))
3087	BUG();
3088	}
3089	}
3090
3091	/*
3092	* The mmap_sem cannot be released by the caller until
3093	* mm_drop_all_locks() returns.
3094	*/
3095	void mm_drop_all_locks(struct mm_struct *mm)
3096	{
3097	struct vm_area_struct *vma;
3098	struct anon_vma_chain *avc;
3099
3100	BUG_ON(down_read_trylock(&mm->mmap_sem));
3101	BUG_ON(!mutex_is_locked(&mm_all_locks_mutex));
3102
3103	for (vma = mm->mmap; vma; vma = vma->vm_next) {
3104	if (vma->anon_vma)
3105	list_for_each_entry(avc, &vma->anon_vma_chain, same_vma)
3106	vm_unlock_anon_vma(avc->anon_vma);
3107	if (vma->vm_file && vma->vm_file->f_mapping)
3108	vm_unlock_mapping(vma->vm_file->f_mapping);
3109	}
3110
3111	mutex_unlock(&mm_all_locks_mutex);
3112	}
3113
3114	/*
3115	* initialise the VMA slab
3116	*/
3117	void __init mmap_init(void)
3118	{
3119	int ret;
3120
3121	ret = percpu_counter_init(&vm_committed_as, 0);
3122	VM_BUG_ON(ret);
3123	}
3124
3125	/*
3126	* Initialise sysctl_user_reserve_kbytes.
3127	*
3128	* This is intended to prevent a user from starting a single memory hogging
3129	* process, such that they cannot recover (kill the hog) in OVERCOMMIT_NEVER
3130	* mode.
3131	*
3132	* The default value is min(3% of free memory, 128MB)
3133	* 128MB is enough to recover with sshd/login, bash, and top/kill.
3134	*/
3135	static int init_user_reserve(void)
3136	{
3137	unsigned long free_kbytes;
3138
3139	free_kbytes = global_page_state(NR_FREE_PAGES) << (PAGE_SHIFT - 10);
3140
3141	sysctl_user_reserve_kbytes = min(free_kbytes / 32, 1UL << 17);
3142	return 0;
3143	}
3144	module_init(init_user_reserve)
3145
3146	/*
3147	* Initialise sysctl_admin_reserve_kbytes.
3148	*
3149	* The purpose of sysctl_admin_reserve_kbytes is to allow the sys admin
3150	* to log in and kill a memory hogging process.
3151	*
3152	* Systems with more than 256MB will reserve 8MB, enough to recover
3153	* with sshd, bash, and top in OVERCOMMIT_GUESS. Smaller systems will
3154	* only reserve 3% of free pages by default.
3155	*/
3156	static int init_admin_reserve(void)
3157	{
3158	unsigned long free_kbytes;
3159
3160	free_kbytes = global_page_state(NR_FREE_PAGES) << (PAGE_SHIFT - 10);
3161
3162	sysctl_admin_reserve_kbytes = min(free_kbytes / 32, 1UL << 13);
3163	return 0;
3164	}
3165	module_init(init_admin_reserve)
3166
3167	/*
3168	* Reinititalise user and admin reserves if memory is added or removed.
3169	*
3170	* The default user reserve max is 128MB, and the default max for the
3171	* admin reserve is 8MB. These are usually, but not always, enough to
3172	* enable recovery from a memory hogging process using login/sshd, a shell,
3173	* and tools like top. It may make sense to increase or even disable the
3174	* reserve depending on the existence of swap or variations in the recovery
3175	* tools. So, the admin may have changed them.
3176	*
3177	* If memory is added and the reserves have been eliminated or increased above
3178	* the default max, then we'll trust the admin.
3179	*
3180	* If memory is removed and there isn't enough free memory, then we
3181	* need to reset the reserves.
3182	*
3183	* Otherwise keep the reserve set by the admin.
3184	*/
3185	static int reserve_mem_notifier(struct notifier_block *nb,
3186	unsigned long action, void *data)
3187	{
3188	unsigned long tmp, free_kbytes;
3189
3190	switch (action) {
3191	case MEM_ONLINE:
3192	/* Default max is 128MB. Leave alone if modified by operator. */
3193	tmp = sysctl_user_reserve_kbytes;
3194	if (0 < tmp && tmp < (1UL << 17))
3195	init_user_reserve();
3196
3197	/* Default max is 8MB. Leave alone if modified by operator. */
3198	tmp = sysctl_admin_reserve_kbytes;
3199	if (0 < tmp && tmp < (1UL << 13))
3200	init_admin_reserve();
3201
3202	break;
3203	case MEM_OFFLINE:
3204	free_kbytes = global_page_state(NR_FREE_PAGES) << (PAGE_SHIFT - 10);
3205
3206	if (sysctl_user_reserve_kbytes > free_kbytes) {
3207	init_user_reserve();
3208	pr_info("vm.user_reserve_kbytes reset to %lu\n",
3209	sysctl_user_reserve_kbytes);
3210	}
3211
3212	if (sysctl_admin_reserve_kbytes > free_kbytes) {
3213	init_admin_reserve();
3214	pr_info("vm.admin_reserve_kbytes reset to %lu\n",
3215	sysctl_admin_reserve_kbytes);
3216	}
3217	break;
3218	default:
3219	break;
3220	}
3221	return NOTIFY_OK;
3222	}
3223
3224	static struct notifier_block reserve_mem_nb = {
3225	.notifier_call = reserve_mem_notifier,
3226	};
3227
3228	static int __meminit init_reserve_notifier(void)
3229	{
3230	if (register_hotmemory_notifier(&reserve_mem_nb))
3231	printk("Failed registering memory add/remove notifier for admin reserve");
3232
3233	return 0;
3234	}
3235	module_init(init_reserve_notifier)
3236

Download this file

Branches:
ben-wpan
ben-wpan-stefan
javiroman/ks7010
jz-2.6.34
jz-2.6.34-rc5
jz-2.6.34-rc6
jz-2.6.34-rc7
jz-2.6.35
jz-2.6.36
jz-2.6.37
jz-2.6.38
jz-2.6.39
jz-3.0
jz-3.1
jz-3.11
jz-3.12
jz-3.13
jz-3.15
jz-3.16
jz-3.18-dt
jz-3.2
jz-3.3
jz-3.4
jz-3.5
jz-3.6
jz-3.6-rc2-pwm
jz-3.9
jz-3.9-clk
jz-3.9-rc8
jz47xx
jz47xx-2.6.38
master

Tags:
od-2011-09-04
od-2011-09-18
v2.6.34-rc5
v2.6.34-rc6
v2.6.34-rc7
v3.9

Kernel

Kernel Git Source Tree

Root/mm/mmap.c

interactive