Kernel

Kernel Git Source Tree

Root/mm/hugetlb.c

1	/*
2	* Generic hugetlb support.
3	* (C) Nadia Yvette Chambers, April 2004
4	*/
5	#include <linux/list.h>
6	#include <linux/init.h>
7	#include <linux/module.h>
8	#include <linux/mm.h>
9	#include <linux/seq_file.h>
10	#include <linux/sysctl.h>
11	#include <linux/highmem.h>
12	#include <linux/mmu_notifier.h>
13	#include <linux/nodemask.h>
14	#include <linux/pagemap.h>
15	#include <linux/mempolicy.h>
16	#include <linux/cpuset.h>
17	#include <linux/mutex.h>
18	#include <linux/bootmem.h>
19	#include <linux/sysfs.h>
20	#include <linux/slab.h>
21	#include <linux/rmap.h>
22	#include <linux/swap.h>
23	#include <linux/swapops.h>
24	#include <linux/page-isolation.h>
25
26	#include <asm/page.h>
27	#include <asm/pgtable.h>
28	#include <asm/tlb.h>
29
30	#include <linux/io.h>
31	#include <linux/hugetlb.h>
32	#include <linux/hugetlb_cgroup.h>
33	#include <linux/node.h>
34	#include "internal.h"
35
36	const unsigned long hugetlb_zero = 0, hugetlb_infinity = ~0UL;
37	unsigned long hugepages_treat_as_movable;
38
39	int hugetlb_max_hstate __read_mostly;
40	unsigned int default_hstate_idx;
41	struct hstate hstates[HUGE_MAX_HSTATE];
42
43	__initdata LIST_HEAD(huge_boot_pages);
44
45	/* for command line parsing */
46	static struct hstate * __initdata parsed_hstate;
47	static unsigned long __initdata default_hstate_max_huge_pages;
48	static unsigned long __initdata default_hstate_size;
49
50	/*
51	* Protects updates to hugepage_freelists, hugepage_activelist, nr_huge_pages,
52	* free_huge_pages, and surplus_huge_pages.
53	*/
54	DEFINE_SPINLOCK(hugetlb_lock);
55
56	static inline void unlock_or_release_subpool(struct hugepage_subpool *spool)
57	{
58	bool free = (spool->count == 0) && (spool->used_hpages == 0);
59
60	spin_unlock(&spool->lock);
61
62	/* If no pages are used, and no other handles to the subpool
63	* remain, free the subpool the subpool remain */
64	if (free)
65	kfree(spool);
66	}
67
68	struct hugepage_subpool *hugepage_new_subpool(long nr_blocks)
69	{
70	struct hugepage_subpool *spool;
71
72	spool = kmalloc(sizeof(*spool), GFP_KERNEL);
73	if (!spool)
74	return NULL;
75
76	spin_lock_init(&spool->lock);
77	spool->count = 1;
78	spool->max_hpages = nr_blocks;
79	spool->used_hpages = 0;
80
81	return spool;
82	}
83
84	void hugepage_put_subpool(struct hugepage_subpool *spool)
85	{
86	spin_lock(&spool->lock);
87	BUG_ON(!spool->count);
88	spool->count--;
89	unlock_or_release_subpool(spool);
90	}
91
92	static int hugepage_subpool_get_pages(struct hugepage_subpool *spool,
93	long delta)
94	{
95	int ret = 0;
96
97	if (!spool)
98	return 0;
99
100	spin_lock(&spool->lock);
101	if ((spool->used_hpages + delta) <= spool->max_hpages) {
102	spool->used_hpages += delta;
103	} else {
104	ret = -ENOMEM;
105	}
106	spin_unlock(&spool->lock);
107
108	return ret;
109	}
110
111	static void hugepage_subpool_put_pages(struct hugepage_subpool *spool,
112	long delta)
113	{
114	if (!spool)
115	return;
116
117	spin_lock(&spool->lock);
118	spool->used_hpages -= delta;
119	/* If hugetlbfs_put_super couldn't free spool due to
120	* an outstanding quota reference, free it now. */
121	unlock_or_release_subpool(spool);
122	}
123
124	static inline struct hugepage_subpool subpool_inode(struct inode inode)
125	{
126	return HUGETLBFS_SB(inode->i_sb)->spool;
127	}
128
129	static inline struct hugepage_subpool subpool_vma(struct vm_area_struct vma)
130	{
131	return subpool_inode(file_inode(vma->vm_file));
132	}
133
134	/*
135	* Region tracking -- allows tracking of reservations and instantiated pages
136	* across the pages in a mapping.
137	*
138	* The region data structures are protected by a combination of the mmap_sem
139	* and the hugetlb_instantiation_mutex. To access or modify a region the caller
140	* must either hold the mmap_sem for write, or the mmap_sem for read and
141	* the hugetlb_instantiation_mutex:
142	*
143	* down_write(&mm->mmap_sem);
144	* or
145	* down_read(&mm->mmap_sem);
146	* mutex_lock(&hugetlb_instantiation_mutex);
147	*/
148	struct file_region {
149	struct list_head link;
150	long from;
151	long to;
152	};
153
154	static long region_add(struct list_head *head, long f, long t)
155	{
156	struct file_region rg, nrg, *trg;
157
158	/* Locate the region we are either in or before. */
159	list_for_each_entry(rg, head, link)
160	if (f <= rg->to)
161	break;
162
163	/* Round our left edge to the current segment if it encloses us. */
164	if (f > rg->from)
165	f = rg->from;
166
167	/* Check for and consume any regions we now overlap with. */
168	nrg = rg;
169	list_for_each_entry_safe(rg, trg, rg->link.prev, link) {
170	if (&rg->link == head)
171	break;
172	if (rg->from > t)
173	break;
174
175	/* If this area reaches higher then extend our area to
176	* include it completely. If this is not the first area
177	* which we intend to reuse, free it. */
178	if (rg->to > t)
179	t = rg->to;
180	if (rg != nrg) {
181	list_del(&rg->link);
182	kfree(rg);
183	}
184	}
185	nrg->from = f;
186	nrg->to = t;
187	return 0;
188	}
189
190	static long region_chg(struct list_head *head, long f, long t)
191	{
192	struct file_region rg, nrg;
193	long chg = 0;
194
195	/* Locate the region we are before or in. */
196	list_for_each_entry(rg, head, link)
197	if (f <= rg->to)
198	break;
199
200	/* If we are below the current region then a new region is required.
201	* Subtle, allocate a new region at the position but make it zero
202	* size such that we can guarantee to record the reservation. */
203	if (&rg->link == head \|\| t < rg->from) {
204	nrg = kmalloc(sizeof(*nrg), GFP_KERNEL);
205	if (!nrg)
206	return -ENOMEM;
207	nrg->from = f;
208	nrg->to = f;
209	INIT_LIST_HEAD(&nrg->link);
210	list_add(&nrg->link, rg->link.prev);
211
212	return t - f;
213	}
214
215	/* Round our left edge to the current segment if it encloses us. */
216	if (f > rg->from)
217	f = rg->from;
218	chg = t - f;
219
220	/* Check for and consume any regions we now overlap with. */
221	list_for_each_entry(rg, rg->link.prev, link) {
222	if (&rg->link == head)
223	break;
224	if (rg->from > t)
225	return chg;
226
227	/* We overlap with this area, if it extends further than
228	* us then we must extend ourselves. Account for its
229	* existing reservation. */
230	if (rg->to > t) {
231	chg += rg->to - t;
232	t = rg->to;
233	}
234	chg -= rg->to - rg->from;
235	}
236	return chg;
237	}
238
239	static long region_truncate(struct list_head *head, long end)
240	{
241	struct file_region rg, trg;
242	long chg = 0;
243
244	/* Locate the region we are either in or before. */
245	list_for_each_entry(rg, head, link)
246	if (end <= rg->to)
247	break;
248	if (&rg->link == head)
249	return 0;
250
251	/* If we are in the middle of a region then adjust it. */
252	if (end > rg->from) {
253	chg = rg->to - end;
254	rg->to = end;
255	rg = list_entry(rg->link.next, typeof(*rg), link);
256	}
257
258	/* Drop any remaining regions. */
259	list_for_each_entry_safe(rg, trg, rg->link.prev, link) {
260	if (&rg->link == head)
261	break;
262	chg += rg->to - rg->from;
263	list_del(&rg->link);
264	kfree(rg);
265	}
266	return chg;
267	}
268
269	static long region_count(struct list_head *head, long f, long t)
270	{
271	struct file_region *rg;
272	long chg = 0;
273
274	/* Locate each segment we overlap with, and count that overlap. */
275	list_for_each_entry(rg, head, link) {
276	long seg_from;
277	long seg_to;
278
279	if (rg->to <= f)
280	continue;
281	if (rg->from >= t)
282	break;
283
284	seg_from = max(rg->from, f);
285	seg_to = min(rg->to, t);
286
287	chg += seg_to - seg_from;
288	}
289
290	return chg;
291	}
292
293	/*
294	* Convert the address within this vma to the page offset within
295	* the mapping, in pagecache page units; huge pages here.
296	*/
297	static pgoff_t vma_hugecache_offset(struct hstate *h,
298	struct vm_area_struct *vma, unsigned long address)
299	{
300	return ((address - vma->vm_start) >> huge_page_shift(h)) +
301	(vma->vm_pgoff >> huge_page_order(h));
302	}
303
304	pgoff_t linear_hugepage_index(struct vm_area_struct *vma,
305	unsigned long address)
306	{
307	return vma_hugecache_offset(hstate_vma(vma), vma, address);
308	}
309
310	/*
311	* Return the size of the pages allocated when backing a VMA. In the majority
312	* cases this will be same size as used by the page table entries.
313	*/
314	unsigned long vma_kernel_pagesize(struct vm_area_struct *vma)
315	{
316	struct hstate *hstate;
317
318	if (!is_vm_hugetlb_page(vma))
319	return PAGE_SIZE;
320
321	hstate = hstate_vma(vma);
322
323	return 1UL << huge_page_shift(hstate);
324	}
325	EXPORT_SYMBOL_GPL(vma_kernel_pagesize);
326
327	/*
328	* Return the page size being used by the MMU to back a VMA. In the majority
329	* of cases, the page size used by the kernel matches the MMU size. On
330	* architectures where it differs, an architecture-specific version of this
331	* function is required.
332	*/
333	#ifndef vma_mmu_pagesize
334	unsigned long vma_mmu_pagesize(struct vm_area_struct *vma)
335	{
336	return vma_kernel_pagesize(vma);
337	}
338	#endif
339
340	/*
341	* Flags for MAP_PRIVATE reservations. These are stored in the bottom
342	* bits of the reservation map pointer, which are always clear due to
343	* alignment.
344	*/
345	#define HPAGE_RESV_OWNER (1UL << 0)
346	#define HPAGE_RESV_UNMAPPED (1UL << 1)
347	#define HPAGE_RESV_MASK (HPAGE_RESV_OWNER \| HPAGE_RESV_UNMAPPED)
348
349	/*
350	* These helpers are used to track how many pages are reserved for
351	* faults in a MAP_PRIVATE mapping. Only the process that called mmap()
352	* is guaranteed to have their future faults succeed.
353	*
354	* With the exception of reset_vma_resv_huge_pages() which is called at fork(),
355	* the reserve counters are updated with the hugetlb_lock held. It is safe
356	* to reset the VMA at fork() time as it is not in use yet and there is no
357	* chance of the global counters getting corrupted as a result of the values.
358	*
359	* The private mapping reservation is represented in a subtly different
360	* manner to a shared mapping. A shared mapping has a region map associated
361	* with the underlying file, this region map represents the backing file
362	* pages which have ever had a reservation assigned which this persists even
363	* after the page is instantiated. A private mapping has a region map
364	* associated with the original mmap which is attached to all VMAs which
365	* reference it, this region map represents those offsets which have consumed
366	* reservation ie. where pages have been instantiated.
367	*/
368	static unsigned long get_vma_private_data(struct vm_area_struct *vma)
369	{
370	return (unsigned long)vma->vm_private_data;
371	}
372
373	static void set_vma_private_data(struct vm_area_struct *vma,
374	unsigned long value)
375	{
376	vma->vm_private_data = (void *)value;
377	}
378
379	struct resv_map {
380	struct kref refs;
381	struct list_head regions;
382	};
383
384	static struct resv_map *resv_map_alloc(void)
385	{
386	struct resv_map resv_map = kmalloc(sizeof(resv_map), GFP_KERNEL);
387	if (!resv_map)
388	return NULL;
389
390	kref_init(&resv_map->refs);
391	INIT_LIST_HEAD(&resv_map->regions);
392
393	return resv_map;
394	}
395
396	static void resv_map_release(struct kref *ref)
397	{
398	struct resv_map *resv_map = container_of(ref, struct resv_map, refs);
399
400	/* Clear out any active regions before we release the map. */
401	region_truncate(&resv_map->regions, 0);
402	kfree(resv_map);
403	}
404
405	static struct resv_map vma_resv_map(struct vm_area_struct vma)
406	{
407	VM_BUG_ON(!is_vm_hugetlb_page(vma));
408	if (!(vma->vm_flags & VM_MAYSHARE))
409	return (struct resv_map *)(get_vma_private_data(vma) &
410	~HPAGE_RESV_MASK);
411	return NULL;
412	}
413
414	static void set_vma_resv_map(struct vm_area_struct vma, struct resv_map map)
415	{
416	VM_BUG_ON(!is_vm_hugetlb_page(vma));
417	VM_BUG_ON(vma->vm_flags & VM_MAYSHARE);
418
419	set_vma_private_data(vma, (get_vma_private_data(vma) &
420	HPAGE_RESV_MASK) \| (unsigned long)map);
421	}
422
423	static void set_vma_resv_flags(struct vm_area_struct *vma, unsigned long flags)
424	{
425	VM_BUG_ON(!is_vm_hugetlb_page(vma));
426	VM_BUG_ON(vma->vm_flags & VM_MAYSHARE);
427
428	set_vma_private_data(vma, get_vma_private_data(vma) \| flags);
429	}
430
431	static int is_vma_resv_set(struct vm_area_struct *vma, unsigned long flag)
432	{
433	VM_BUG_ON(!is_vm_hugetlb_page(vma));
434
435	return (get_vma_private_data(vma) & flag) != 0;
436	}
437
438	/* Reset counters to 0 and clear all HPAGE_RESV_* flags */
439	void reset_vma_resv_huge_pages(struct vm_area_struct *vma)
440	{
441	VM_BUG_ON(!is_vm_hugetlb_page(vma));
442	if (!(vma->vm_flags & VM_MAYSHARE))
443	vma->vm_private_data = (void *)0;
444	}
445
446	/* Returns true if the VMA has associated reserve pages */
447	static int vma_has_reserves(struct vm_area_struct *vma, long chg)
448	{
449	if (vma->vm_flags & VM_NORESERVE) {
450	/*
451	* This address is already reserved by other process(chg == 0),
452	* so, we should decrement reserved count. Without decrementing,
453	* reserve count remains after releasing inode, because this
454	* allocated page will go into page cache and is regarded as
455	* coming from reserved pool in releasing step. Currently, we
456	* don't have any other solution to deal with this situation
457	* properly, so add work-around here.
458	*/
459	if (vma->vm_flags & VM_MAYSHARE && chg == 0)
460	return 1;
461	else
462	return 0;
463	}
464
465	/* Shared mappings always use reserves */
466	if (vma->vm_flags & VM_MAYSHARE)
467	return 1;
468
469	/*
470	* Only the process that called mmap() has reserves for
471	* private mappings.
472	*/
473	if (is_vma_resv_set(vma, HPAGE_RESV_OWNER))
474	return 1;
475
476	return 0;
477	}
478
479	static void copy_gigantic_page(struct page dst, struct page src)
480	{
481	int i;
482	struct hstate *h = page_hstate(src);
483	struct page *dst_base = dst;
484	struct page *src_base = src;
485
486	for (i = 0; i < pages_per_huge_page(h); ) {
487	cond_resched();
488	copy_highpage(dst, src);
489
490	i++;
491	dst = mem_map_next(dst, dst_base, i);
492	src = mem_map_next(src, src_base, i);
493	}
494	}
495
496	void copy_huge_page(struct page dst, struct page src)
497	{
498	int i;
499	struct hstate *h = page_hstate(src);
500
501	if (unlikely(pages_per_huge_page(h) > MAX_ORDER_NR_PAGES)) {
502	copy_gigantic_page(dst, src);
503	return;
504	}
505
506	might_sleep();
507	for (i = 0; i < pages_per_huge_page(h); i++) {
508	cond_resched();
509	copy_highpage(dst + i, src + i);
510	}
511	}
512
513	static void enqueue_huge_page(struct hstate h, struct page page)
514	{
515	int nid = page_to_nid(page);
516	list_move(&page->lru, &h->hugepage_freelists[nid]);
517	h->free_huge_pages++;
518	h->free_huge_pages_node[nid]++;
519	}
520
521	static struct page dequeue_huge_page_node(struct hstate h, int nid)
522	{
523	struct page *page;
524
525	list_for_each_entry(page, &h->hugepage_freelists[nid], lru)
526	if (!is_migrate_isolate_page(page))
527	break;
528	/*
529	* if 'non-isolated free hugepage' not found on the list,
530	* the allocation fails.
531	*/
532	if (&h->hugepage_freelists[nid] == &page->lru)
533	return NULL;
534	list_move(&page->lru, &h->hugepage_activelist);
535	set_page_refcounted(page);
536	h->free_huge_pages--;
537	h->free_huge_pages_node[nid]--;
538	return page;
539	}
540
541	/* Movability of hugepages depends on migration support. */
542	static inline gfp_t htlb_alloc_mask(struct hstate *h)
543	{
544	if (hugepages_treat_as_movable \|\| hugepage_migration_support(h))
545	return GFP_HIGHUSER_MOVABLE;
546	else
547	return GFP_HIGHUSER;
548	}
549
550	static struct page dequeue_huge_page_vma(struct hstate h,
551	struct vm_area_struct *vma,
552	unsigned long address, int avoid_reserve,
553	long chg)
554	{
555	struct page *page = NULL;
556	struct mempolicy *mpol;
557	nodemask_t *nodemask;
558	struct zonelist *zonelist;
559	struct zone *zone;
560	struct zoneref *z;
561	unsigned int cpuset_mems_cookie;
562
563	/*
564	* A child process with MAP_PRIVATE mappings created by their parent
565	* have no page reserves. This check ensures that reservations are
566	* not "stolen". The child may still get SIGKILLed
567	*/
568	if (!vma_has_reserves(vma, chg) &&
569	h->free_huge_pages - h->resv_huge_pages == 0)
570	goto err;
571
572	/* If reserves cannot be used, ensure enough pages are in the pool */
573	if (avoid_reserve && h->free_huge_pages - h->resv_huge_pages == 0)
574	goto err;
575
576	retry_cpuset:
577	cpuset_mems_cookie = get_mems_allowed();
578	zonelist = huge_zonelist(vma, address,
579	htlb_alloc_mask(h), &mpol, &nodemask);
580
581	for_each_zone_zonelist_nodemask(zone, z, zonelist,
582	MAX_NR_ZONES - 1, nodemask) {
583	if (cpuset_zone_allowed_softwall(zone, htlb_alloc_mask(h))) {
584	page = dequeue_huge_page_node(h, zone_to_nid(zone));
585	if (page) {
586	if (avoid_reserve)
587	break;
588	if (!vma_has_reserves(vma, chg))
589	break;
590
591	SetPagePrivate(page);
592	h->resv_huge_pages--;
593	break;
594	}
595	}
596	}
597
598	mpol_cond_put(mpol);
599	if (unlikely(!put_mems_allowed(cpuset_mems_cookie) && !page))
600	goto retry_cpuset;
601	return page;
602
603	err:
604	return NULL;
605	}
606
607	static void update_and_free_page(struct hstate h, struct page page)
608	{
609	int i;
610
611	VM_BUG_ON(h->order >= MAX_ORDER);
612
613	h->nr_huge_pages--;
614	h->nr_huge_pages_node[page_to_nid(page)]--;
615	for (i = 0; i < pages_per_huge_page(h); i++) {
616	page[i].flags &= ~(1 << PG_locked \| 1 << PG_error \|
617	1 << PG_referenced \| 1 << PG_dirty \|
618	1 << PG_active \| 1 << PG_reserved \|
619	1 << PG_private \| 1 << PG_writeback);
620	}
621	VM_BUG_ON(hugetlb_cgroup_from_page(page));
622	set_compound_page_dtor(page, NULL);
623	set_page_refcounted(page);
624	arch_release_hugepage(page);
625	__free_pages(page, huge_page_order(h));
626	}
627
628	struct hstate *size_to_hstate(unsigned long size)
629	{
630	struct hstate *h;
631
632	for_each_hstate(h) {
633	if (huge_page_size(h) == size)
634	return h;
635	}
636	return NULL;
637	}
638
639	static void free_huge_page(struct page *page)
640	{
641	/*
642	* Can't pass hstate in here because it is called from the
643	* compound page destructor.
644	*/
645	struct hstate *h = page_hstate(page);
646	int nid = page_to_nid(page);
647	struct hugepage_subpool *spool =
648	(struct hugepage_subpool *)page_private(page);
649	bool restore_reserve;
650
651	set_page_private(page, 0);
652	page->mapping = NULL;
653	BUG_ON(page_count(page));
654	BUG_ON(page_mapcount(page));
655	restore_reserve = PagePrivate(page);
656	ClearPagePrivate(page);
657
658	spin_lock(&hugetlb_lock);
659	hugetlb_cgroup_uncharge_page(hstate_index(h),
660	pages_per_huge_page(h), page);
661	if (restore_reserve)
662	h->resv_huge_pages++;
663
664	if (h->surplus_huge_pages_node[nid] && huge_page_order(h) < MAX_ORDER) {
665	/* remove the page from active list */
666	list_del(&page->lru);
667	update_and_free_page(h, page);
668	h->surplus_huge_pages--;
669	h->surplus_huge_pages_node[nid]--;
670	} else {
671	arch_clear_hugepage_flags(page);
672	enqueue_huge_page(h, page);
673	}
674	spin_unlock(&hugetlb_lock);
675	hugepage_subpool_put_pages(spool, 1);
676	}
677
678	static void prep_new_huge_page(struct hstate h, struct page page, int nid)
679	{
680	INIT_LIST_HEAD(&page->lru);
681	set_compound_page_dtor(page, free_huge_page);
682	spin_lock(&hugetlb_lock);
683	set_hugetlb_cgroup(page, NULL);
684	h->nr_huge_pages++;
685	h->nr_huge_pages_node[nid]++;
686	spin_unlock(&hugetlb_lock);
687	put_page(page); /* free it into the hugepage allocator */
688	}
689
690	static void prep_compound_gigantic_page(struct page *page, unsigned long order)
691	{
692	int i;
693	int nr_pages = 1 << order;
694	struct page *p = page + 1;
695
696	/* we rely on prep_new_huge_page to set the destructor */
697	set_compound_order(page, order);
698	__SetPageHead(page);
699	__ClearPageReserved(page);
700	for (i = 1; i < nr_pages; i++, p = mem_map_next(p, page, i)) {
701	__SetPageTail(p);
702	/*
703	* For gigantic hugepages allocated through bootmem at
704	* boot, it's safer to be consistent with the not-gigantic
705	* hugepages and clear the PG_reserved bit from all tail pages
706	* too. Otherwse drivers using get_user_pages() to access tail
707	* pages may get the reference counting wrong if they see
708	* PG_reserved set on a tail page (despite the head page not
709	* having PG_reserved set). Enforcing this consistency between
710	* head and tail pages allows drivers to optimize away a check
711	* on the head page when they need know if put_page() is needed
712	* after get_user_pages().
713	*/
714	__ClearPageReserved(p);
715	set_page_count(p, 0);
716	p->first_page = page;
717	}
718	}
719
720	/*
721	* PageHuge() only returns true for hugetlbfs pages, but not for normal or
722	* transparent huge pages. See the PageTransHuge() documentation for more
723	* details.
724	*/
725	int PageHuge(struct page *page)
726	{
727	compound_page_dtor *dtor;
728
729	if (!PageCompound(page))
730	return 0;
731
732	page = compound_head(page);
733	dtor = get_compound_page_dtor(page);
734
735	return dtor == free_huge_page;
736	}
737	EXPORT_SYMBOL_GPL(PageHuge);
738
739	pgoff_t __basepage_index(struct page *page)
740	{
741	struct page *page_head = compound_head(page);
742	pgoff_t index = page_index(page_head);
743	unsigned long compound_idx;
744
745	if (!PageHuge(page_head))
746	return page_index(page);
747
748	if (compound_order(page_head) >= MAX_ORDER)
749	compound_idx = page_to_pfn(page) - page_to_pfn(page_head);
750	else
751	compound_idx = page - page_head;
752
753	return (index << compound_order(page_head)) + compound_idx;
754	}
755
756	static struct page alloc_fresh_huge_page_node(struct hstate h, int nid)
757	{
758	struct page *page;
759
760	if (h->order >= MAX_ORDER)
761	return NULL;
762
763	page = alloc_pages_exact_node(nid,
764	htlb_alloc_mask(h)\|__GFP_COMP\|__GFP_THISNODE\|
765	__GFP_REPEAT\|__GFP_NOWARN,
766	huge_page_order(h));
767	if (page) {
768	if (arch_prepare_hugepage(page)) {
769	__free_pages(page, huge_page_order(h));
770	return NULL;
771	}
772	prep_new_huge_page(h, page, nid);
773	}
774
775	return page;
776	}
777
778	/*
779	* common helper functions for hstate_next_node_to_{alloc\|free}.
780	* We may have allocated or freed a huge page based on a different
781	* nodes_allowed previously, so h->next_node_to_{alloc\|free} might
782	* be outside of *nodes_allowed. Ensure that we use an allowed
783	* node for alloc or free.
784	*/
785	static int next_node_allowed(int nid, nodemask_t *nodes_allowed)
786	{
787	nid = next_node(nid, *nodes_allowed);
788	if (nid == MAX_NUMNODES)
789	nid = first_node(*nodes_allowed);
790	VM_BUG_ON(nid >= MAX_NUMNODES);
791
792	return nid;
793	}
794
795	static int get_valid_node_allowed(int nid, nodemask_t *nodes_allowed)
796	{
797	if (!node_isset(nid, *nodes_allowed))
798	nid = next_node_allowed(nid, nodes_allowed);
799	return nid;
800	}
801
802	/*
803	* returns the previously saved node ["this node"] from which to
804	* allocate a persistent huge page for the pool and advance the
805	* next node from which to allocate, handling wrap at end of node
806	* mask.
807	*/
808	static int hstate_next_node_to_alloc(struct hstate *h,
809	nodemask_t *nodes_allowed)
810	{
811	int nid;
812
813	VM_BUG_ON(!nodes_allowed);
814
815	nid = get_valid_node_allowed(h->next_nid_to_alloc, nodes_allowed);
816	h->next_nid_to_alloc = next_node_allowed(nid, nodes_allowed);
817
818	return nid;
819	}
820
821	/*
822	* helper for free_pool_huge_page() - return the previously saved
823	* node ["this node"] from which to free a huge page. Advance the
824	* next node id whether or not we find a free huge page to free so
825	* that the next attempt to free addresses the next node.
826	*/
827	static int hstate_next_node_to_free(struct hstate h, nodemask_t nodes_allowed)
828	{
829	int nid;
830
831	VM_BUG_ON(!nodes_allowed);
832
833	nid = get_valid_node_allowed(h->next_nid_to_free, nodes_allowed);
834	h->next_nid_to_free = next_node_allowed(nid, nodes_allowed);
835
836	return nid;
837	}
838
839	#define for_each_node_mask_to_alloc(hs, nr_nodes, node, mask) \
840	for (nr_nodes = nodes_weight(*mask); \
841	nr_nodes > 0 && \
842	((node = hstate_next_node_to_alloc(hs, mask)) \|\| 1); \
843	nr_nodes--)
844
845	#define for_each_node_mask_to_free(hs, nr_nodes, node, mask) \
846	for (nr_nodes = nodes_weight(*mask); \
847	nr_nodes > 0 && \
848	((node = hstate_next_node_to_free(hs, mask)) \|\| 1); \
849	nr_nodes--)
850
851	static int alloc_fresh_huge_page(struct hstate h, nodemask_t nodes_allowed)
852	{
853	struct page *page;
854	int nr_nodes, node;
855	int ret = 0;
856
857	for_each_node_mask_to_alloc(h, nr_nodes, node, nodes_allowed) {
858	page = alloc_fresh_huge_page_node(h, node);
859	if (page) {
860	ret = 1;
861	break;
862	}
863	}
864
865	if (ret)
866	count_vm_event(HTLB_BUDDY_PGALLOC);
867	else
868	count_vm_event(HTLB_BUDDY_PGALLOC_FAIL);
869
870	return ret;
871	}
872
873	/*
874	* Free huge page from pool from next node to free.
875	* Attempt to keep persistent huge pages more or less
876	* balanced over allowed nodes.
877	* Called with hugetlb_lock locked.
878	*/
879	static int free_pool_huge_page(struct hstate h, nodemask_t nodes_allowed,
880	bool acct_surplus)
881	{
882	int nr_nodes, node;
883	int ret = 0;
884
885	for_each_node_mask_to_free(h, nr_nodes, node, nodes_allowed) {
886	/*
887	* If we're returning unused surplus pages, only examine
888	* nodes with surplus pages.
889	*/
890	if ((!acct_surplus \|\| h->surplus_huge_pages_node[node]) &&
891	!list_empty(&h->hugepage_freelists[node])) {
892	struct page *page =
893	list_entry(h->hugepage_freelists[node].next,
894	struct page, lru);
895	list_del(&page->lru);
896	h->free_huge_pages--;
897	h->free_huge_pages_node[node]--;
898	if (acct_surplus) {
899	h->surplus_huge_pages--;
900	h->surplus_huge_pages_node[node]--;
901	}
902	update_and_free_page(h, page);
903	ret = 1;
904	break;
905	}
906	}
907
908	return ret;
909	}
910
911	/*
912	* Dissolve a given free hugepage into free buddy pages. This function does
913	* nothing for in-use (including surplus) hugepages.
914	*/
915	static void dissolve_free_huge_page(struct page *page)
916	{
917	spin_lock(&hugetlb_lock);
918	if (PageHuge(page) && !page_count(page)) {
919	struct hstate *h = page_hstate(page);
920	int nid = page_to_nid(page);
921	list_del(&page->lru);
922	h->free_huge_pages--;
923	h->free_huge_pages_node[nid]--;
924	update_and_free_page(h, page);
925	}
926	spin_unlock(&hugetlb_lock);
927	}
928
929	/*
930	* Dissolve free hugepages in a given pfn range. Used by memory hotplug to
931	* make specified memory blocks removable from the system.
932	* Note that start_pfn should aligned with (minimum) hugepage size.
933	*/
934	void dissolve_free_huge_pages(unsigned long start_pfn, unsigned long end_pfn)
935	{
936	unsigned int order = 8 * sizeof(void *);
937	unsigned long pfn;
938	struct hstate *h;
939
940	/* Set scan step to minimum hugepage size */
941	for_each_hstate(h)
942	if (order > huge_page_order(h))
943	order = huge_page_order(h);
944	VM_BUG_ON(!IS_ALIGNED(start_pfn, 1 << order));
945	for (pfn = start_pfn; pfn < end_pfn; pfn += 1 << order)
946	dissolve_free_huge_page(pfn_to_page(pfn));
947	}
948
949	static struct page alloc_buddy_huge_page(struct hstate h, int nid)
950	{
951	struct page *page;
952	unsigned int r_nid;
953
954	if (h->order >= MAX_ORDER)
955	return NULL;
956
957	/*
958	* Assume we will successfully allocate the surplus page to
959	* prevent racing processes from causing the surplus to exceed
960	* overcommit
961	*
962	* This however introduces a different race, where a process B
963	* tries to grow the static hugepage pool while alloc_pages() is
964	* called by process A. B will only examine the per-node
965	* counters in determining if surplus huge pages can be
966	* converted to normal huge pages in adjust_pool_surplus(). A
967	* won't be able to increment the per-node counter, until the
968	* lock is dropped by B, but B doesn't drop hugetlb_lock until
969	* no more huge pages can be converted from surplus to normal
970	* state (and doesn't try to convert again). Thus, we have a
971	* case where a surplus huge page exists, the pool is grown, and
972	* the surplus huge page still exists after, even though it
973	* should just have been converted to a normal huge page. This
974	* does not leak memory, though, as the hugepage will be freed
975	* once it is out of use. It also does not allow the counters to
976	* go out of whack in adjust_pool_surplus() as we don't modify
977	* the node values until we've gotten the hugepage and only the
978	* per-node value is checked there.
979	*/
980	spin_lock(&hugetlb_lock);
981	if (h->surplus_huge_pages >= h->nr_overcommit_huge_pages) {
982	spin_unlock(&hugetlb_lock);
983	return NULL;
984	} else {
985	h->nr_huge_pages++;
986	h->surplus_huge_pages++;
987	}
988	spin_unlock(&hugetlb_lock);
989
990	if (nid == NUMA_NO_NODE)
991	page = alloc_pages(htlb_alloc_mask(h)\|__GFP_COMP\|
992	__GFP_REPEAT\|__GFP_NOWARN,
993	huge_page_order(h));
994	else
995	page = alloc_pages_exact_node(nid,
996	htlb_alloc_mask(h)\|__GFP_COMP\|__GFP_THISNODE\|
997	__GFP_REPEAT\|__GFP_NOWARN, huge_page_order(h));
998
999	if (page && arch_prepare_hugepage(page)) {
1000	__free_pages(page, huge_page_order(h));
1001	page = NULL;
1002	}
1003
1004	spin_lock(&hugetlb_lock);
1005	if (page) {
1006	INIT_LIST_HEAD(&page->lru);
1007	r_nid = page_to_nid(page);
1008	set_compound_page_dtor(page, free_huge_page);
1009	set_hugetlb_cgroup(page, NULL);
1010	/*
1011	* We incremented the global counters already
1012	*/
1013	h->nr_huge_pages_node[r_nid]++;
1014	h->surplus_huge_pages_node[r_nid]++;
1015	__count_vm_event(HTLB_BUDDY_PGALLOC);
1016	} else {
1017	h->nr_huge_pages--;
1018	h->surplus_huge_pages--;
1019	__count_vm_event(HTLB_BUDDY_PGALLOC_FAIL);
1020	}
1021	spin_unlock(&hugetlb_lock);
1022
1023	return page;
1024	}
1025
1026	/*
1027	* This allocation function is useful in the context where vma is irrelevant.
1028	* E.g. soft-offlining uses this function because it only cares physical
1029	* address of error page.
1030	*/
1031	struct page alloc_huge_page_node(struct hstate h, int nid)
1032	{
1033	struct page *page = NULL;
1034
1035	spin_lock(&hugetlb_lock);
1036	if (h->free_huge_pages - h->resv_huge_pages > 0)
1037	page = dequeue_huge_page_node(h, nid);
1038	spin_unlock(&hugetlb_lock);
1039
1040	if (!page)
1041	page = alloc_buddy_huge_page(h, nid);
1042
1043	return page;
1044	}
1045
1046	/*
1047	* Increase the hugetlb pool such that it can accommodate a reservation
1048	* of size 'delta'.
1049	*/
1050	static int gather_surplus_pages(struct hstate *h, int delta)
1051	{
1052	struct list_head surplus_list;
1053	struct page page, tmp;
1054	int ret, i;
1055	int needed, allocated;
1056	bool alloc_ok = true;
1057
1058	needed = (h->resv_huge_pages + delta) - h->free_huge_pages;
1059	if (needed <= 0) {
1060	h->resv_huge_pages += delta;
1061	return 0;
1062	}
1063
1064	allocated = 0;
1065	INIT_LIST_HEAD(&surplus_list);
1066
1067	ret = -ENOMEM;
1068	retry:
1069	spin_unlock(&hugetlb_lock);
1070	for (i = 0; i < needed; i++) {
1071	page = alloc_buddy_huge_page(h, NUMA_NO_NODE);
1072	if (!page) {
1073	alloc_ok = false;
1074	break;
1075	}
1076	list_add(&page->lru, &surplus_list);
1077	}
1078	allocated += i;
1079
1080	/*
1081	* After retaking hugetlb_lock, we need to recalculate 'needed'
1082	* because either resv_huge_pages or free_huge_pages may have changed.
1083	*/
1084	spin_lock(&hugetlb_lock);
1085	needed = (h->resv_huge_pages + delta) -
1086	(h->free_huge_pages + allocated);
1087	if (needed > 0) {
1088	if (alloc_ok)
1089	goto retry;
1090	/*
1091	* We were not able to allocate enough pages to
1092	* satisfy the entire reservation so we free what
1093	* we've allocated so far.
1094	*/
1095	goto free;
1096	}
1097	/*
1098	* The surplus_list now contains _at_least_ the number of extra pages
1099	* needed to accommodate the reservation. Add the appropriate number
1100	* of pages to the hugetlb pool and free the extras back to the buddy
1101	* allocator. Commit the entire reservation here to prevent another
1102	* process from stealing the pages as they are added to the pool but
1103	* before they are reserved.
1104	*/
1105	needed += allocated;
1106	h->resv_huge_pages += delta;
1107	ret = 0;
1108
1109	/* Free the needed pages to the hugetlb pool */
1110	list_for_each_entry_safe(page, tmp, &surplus_list, lru) {
1111	if ((--needed) < 0)
1112	break;
1113	/*
1114	* This page is now managed by the hugetlb allocator and has
1115	* no users -- drop the buddy allocator's reference.
1116	*/
1117	put_page_testzero(page);
1118	VM_BUG_ON(page_count(page));
1119	enqueue_huge_page(h, page);
1120	}
1121	free:
1122	spin_unlock(&hugetlb_lock);
1123
1124	/* Free unnecessary surplus pages to the buddy allocator */
1125	list_for_each_entry_safe(page, tmp, &surplus_list, lru)
1126	put_page(page);
1127	spin_lock(&hugetlb_lock);
1128
1129	return ret;
1130	}
1131
1132	/*
1133	* When releasing a hugetlb pool reservation, any surplus pages that were
1134	* allocated to satisfy the reservation must be explicitly freed if they were
1135	* never used.
1136	* Called with hugetlb_lock held.
1137	*/
1138	static void return_unused_surplus_pages(struct hstate *h,
1139	unsigned long unused_resv_pages)
1140	{
1141	unsigned long nr_pages;
1142
1143	/* Uncommit the reservation */
1144	h->resv_huge_pages -= unused_resv_pages;
1145
1146	/* Cannot return gigantic pages currently */
1147	if (h->order >= MAX_ORDER)
1148	return;
1149
1150	nr_pages = min(unused_resv_pages, h->surplus_huge_pages);
1151
1152	/*
1153	* We want to release as many surplus pages as possible, spread
1154	* evenly across all nodes with memory. Iterate across these nodes
1155	* until we can no longer free unreserved surplus pages. This occurs
1156	* when the nodes with surplus pages have no free pages.
1157	* free_pool_huge_page() will balance the the freed pages across the
1158	* on-line nodes with memory and will handle the hstate accounting.
1159	*/
1160	while (nr_pages--) {
1161	if (!free_pool_huge_page(h, &node_states[N_MEMORY], 1))
1162	break;
1163	}
1164	}
1165
1166	/*
1167	* Determine if the huge page at addr within the vma has an associated
1168	* reservation. Where it does not we will need to logically increase
1169	* reservation and actually increase subpool usage before an allocation
1170	* can occur. Where any new reservation would be required the
1171	* reservation change is prepared, but not committed. Once the page
1172	* has been allocated from the subpool and instantiated the change should
1173	* be committed via vma_commit_reservation. No action is required on
1174	* failure.
1175	*/
1176	static long vma_needs_reservation(struct hstate *h,
1177	struct vm_area_struct *vma, unsigned long addr)
1178	{
1179	struct address_space *mapping = vma->vm_file->f_mapping;
1180	struct inode *inode = mapping->host;
1181
1182	if (vma->vm_flags & VM_MAYSHARE) {
1183	pgoff_t idx = vma_hugecache_offset(h, vma, addr);
1184	return region_chg(&inode->i_mapping->private_list,
1185	idx, idx + 1);
1186
1187	} else if (!is_vma_resv_set(vma, HPAGE_RESV_OWNER)) {
1188	return 1;
1189
1190	} else {
1191	long err;
1192	pgoff_t idx = vma_hugecache_offset(h, vma, addr);
1193	struct resv_map *resv = vma_resv_map(vma);
1194
1195	err = region_chg(&resv->regions, idx, idx + 1);
1196	if (err < 0)
1197	return err;
1198	return 0;
1199	}
1200	}
1201	static void vma_commit_reservation(struct hstate *h,
1202	struct vm_area_struct *vma, unsigned long addr)
1203	{
1204	struct address_space *mapping = vma->vm_file->f_mapping;
1205	struct inode *inode = mapping->host;
1206
1207	if (vma->vm_flags & VM_MAYSHARE) {
1208	pgoff_t idx = vma_hugecache_offset(h, vma, addr);
1209	region_add(&inode->i_mapping->private_list, idx, idx + 1);
1210
1211	} else if (is_vma_resv_set(vma, HPAGE_RESV_OWNER)) {
1212	pgoff_t idx = vma_hugecache_offset(h, vma, addr);
1213	struct resv_map *resv = vma_resv_map(vma);
1214
1215	/* Mark this page used in the map. */
1216	region_add(&resv->regions, idx, idx + 1);
1217	}
1218	}
1219
1220	static struct page alloc_huge_page(struct vm_area_struct vma,
1221	unsigned long addr, int avoid_reserve)
1222	{
1223	struct hugepage_subpool *spool = subpool_vma(vma);
1224	struct hstate *h = hstate_vma(vma);
1225	struct page *page;
1226	long chg;
1227	int ret, idx;
1228	struct hugetlb_cgroup *h_cg;
1229
1230	idx = hstate_index(h);
1231	/*
1232	* Processes that did not create the mapping will have no
1233	* reserves and will not have accounted against subpool
1234	* limit. Check that the subpool limit can be made before
1235	* satisfying the allocation MAP_NORESERVE mappings may also
1236	* need pages and subpool limit allocated allocated if no reserve
1237	* mapping overlaps.
1238	*/
1239	chg = vma_needs_reservation(h, vma, addr);
1240	if (chg < 0)
1241	return ERR_PTR(-ENOMEM);
1242	if (chg \|\| avoid_reserve)
1243	if (hugepage_subpool_get_pages(spool, 1))
1244	return ERR_PTR(-ENOSPC);
1245
1246	ret = hugetlb_cgroup_charge_cgroup(idx, pages_per_huge_page(h), &h_cg);
1247	if (ret) {
1248	if (chg \|\| avoid_reserve)
1249	hugepage_subpool_put_pages(spool, 1);
1250	return ERR_PTR(-ENOSPC);
1251	}
1252	spin_lock(&hugetlb_lock);
1253	page = dequeue_huge_page_vma(h, vma, addr, avoid_reserve, chg);
1254	if (!page) {
1255	spin_unlock(&hugetlb_lock);
1256	page = alloc_buddy_huge_page(h, NUMA_NO_NODE);
1257	if (!page) {
1258	hugetlb_cgroup_uncharge_cgroup(idx,
1259	pages_per_huge_page(h),
1260	h_cg);
1261	if (chg \|\| avoid_reserve)
1262	hugepage_subpool_put_pages(spool, 1);
1263	return ERR_PTR(-ENOSPC);
1264	}
1265	spin_lock(&hugetlb_lock);
1266	list_move(&page->lru, &h->hugepage_activelist);
1267	/* Fall through */
1268	}
1269	hugetlb_cgroup_commit_charge(idx, pages_per_huge_page(h), h_cg, page);
1270	spin_unlock(&hugetlb_lock);
1271
1272	set_page_private(page, (unsigned long)spool);
1273
1274	vma_commit_reservation(h, vma, addr);
1275	return page;
1276	}
1277
1278	/*
1279	* alloc_huge_page()'s wrapper which simply returns the page if allocation
1280	* succeeds, otherwise NULL. This function is called from new_vma_page(),
1281	* where no ERR_VALUE is expected to be returned.
1282	*/
1283	struct page alloc_huge_page_noerr(struct vm_area_struct vma,
1284	unsigned long addr, int avoid_reserve)
1285	{
1286	struct page *page = alloc_huge_page(vma, addr, avoid_reserve);
1287	if (IS_ERR(page))
1288	page = NULL;
1289	return page;
1290	}
1291
1292	int __weak alloc_bootmem_huge_page(struct hstate *h)
1293	{
1294	struct huge_bootmem_page *m;
1295	int nr_nodes, node;
1296
1297	for_each_node_mask_to_alloc(h, nr_nodes, node, &node_states[N_MEMORY]) {
1298	void *addr;
1299
1300	addr = __alloc_bootmem_node_nopanic(NODE_DATA(node),
1301	huge_page_size(h), huge_page_size(h), 0);
1302
1303	if (addr) {
1304	/*
1305	* Use the beginning of the huge page to store the
1306	* huge_bootmem_page struct (until gather_bootmem
1307	* puts them into the mem_map).
1308	*/
1309	m = addr;
1310	goto found;
1311	}
1312	}
1313	return 0;
1314
1315	found:
1316	BUG_ON((unsigned long)virt_to_phys(m) & (huge_page_size(h) - 1));
1317	/* Put them into a private list first because mem_map is not up yet */
1318	list_add(&m->list, &huge_boot_pages);
1319	m->hstate = h;
1320	return 1;
1321	}
1322
1323	static void prep_compound_huge_page(struct page *page, int order)
1324	{
1325	if (unlikely(order > (MAX_ORDER - 1)))
1326	prep_compound_gigantic_page(page, order);
1327	else
1328	prep_compound_page(page, order);
1329	}
1330
1331	/* Put bootmem huge pages into the standard lists after mem_map is up */
1332	static void __init gather_bootmem_prealloc(void)
1333	{
1334	struct huge_bootmem_page *m;
1335
1336	list_for_each_entry(m, &huge_boot_pages, list) {
1337	struct hstate *h = m->hstate;
1338	struct page *page;
1339
1340	#ifdef CONFIG_HIGHMEM
1341	page = pfn_to_page(m->phys >> PAGE_SHIFT);
1342	free_bootmem_late((unsigned long)m,
1343	sizeof(struct huge_bootmem_page));
1344	#else
1345	page = virt_to_page(m);
1346	#endif
1347	WARN_ON(page_count(page) != 1);
1348	prep_compound_huge_page(page, h->order);
1349	WARN_ON(PageReserved(page));
1350	prep_new_huge_page(h, page, page_to_nid(page));
1351	/*
1352	* If we had gigantic hugepages allocated at boot time, we need
1353	* to restore the 'stolen' pages to totalram_pages in order to
1354	* fix confusing memory reports from free(1) and another
1355	* side-effects, like CommitLimit going negative.
1356	*/
1357	if (h->order > (MAX_ORDER - 1))
1358	adjust_managed_page_count(page, 1 << h->order);
1359	}
1360	}
1361
1362	static void __init hugetlb_hstate_alloc_pages(struct hstate *h)
1363	{
1364	unsigned long i;
1365
1366	for (i = 0; i < h->max_huge_pages; ++i) {
1367	if (h->order >= MAX_ORDER) {
1368	if (!alloc_bootmem_huge_page(h))
1369	break;
1370	} else if (!alloc_fresh_huge_page(h,
1371	&node_states[N_MEMORY]))
1372	break;
1373	}
1374	h->max_huge_pages = i;
1375	}
1376
1377	static void __init hugetlb_init_hstates(void)
1378	{
1379	struct hstate *h;
1380
1381	for_each_hstate(h) {
1382	/* oversize hugepages were init'ed in early boot */
1383	if (h->order < MAX_ORDER)
1384	hugetlb_hstate_alloc_pages(h);
1385	}
1386	}
1387
1388	static char * __init memfmt(char *buf, unsigned long n)
1389	{
1390	if (n >= (1UL << 30))
1391	sprintf(buf, "%lu GB", n >> 30);
1392	else if (n >= (1UL << 20))
1393	sprintf(buf, "%lu MB", n >> 20);
1394	else
1395	sprintf(buf, "%lu KB", n >> 10);
1396	return buf;
1397	}
1398
1399	static void __init report_hugepages(void)
1400	{
1401	struct hstate *h;
1402
1403	for_each_hstate(h) {
1404	char buf[32];
1405	pr_info("HugeTLB registered %s page size, pre-allocated %ld pages\n",
1406	memfmt(buf, huge_page_size(h)),
1407	h->free_huge_pages);
1408	}
1409	}
1410
1411	#ifdef CONFIG_HIGHMEM
1412	static void try_to_free_low(struct hstate *h, unsigned long count,
1413	nodemask_t *nodes_allowed)
1414	{
1415	int i;
1416
1417	if (h->order >= MAX_ORDER)
1418	return;
1419
1420	for_each_node_mask(i, *nodes_allowed) {
1421	struct page page, next;
1422	struct list_head *freel = &h->hugepage_freelists[i];
1423	list_for_each_entry_safe(page, next, freel, lru) {
1424	if (count >= h->nr_huge_pages)
1425	return;
1426	if (PageHighMem(page))
1427	continue;
1428	list_del(&page->lru);
1429	update_and_free_page(h, page);
1430	h->free_huge_pages--;
1431	h->free_huge_pages_node[page_to_nid(page)]--;
1432	}
1433	}
1434	}
1435	#else
1436	static inline void try_to_free_low(struct hstate *h, unsigned long count,
1437	nodemask_t *nodes_allowed)
1438	{
1439	}
1440	#endif
1441
1442	/*
1443	* Increment or decrement surplus_huge_pages. Keep node-specific counters
1444	* balanced by operating on them in a round-robin fashion.
1445	* Returns 1 if an adjustment was made.
1446	*/
1447	static int adjust_pool_surplus(struct hstate h, nodemask_t nodes_allowed,
1448	int delta)
1449	{
1450	int nr_nodes, node;
1451
1452	VM_BUG_ON(delta != -1 && delta != 1);
1453
1454	if (delta < 0) {
1455	for_each_node_mask_to_alloc(h, nr_nodes, node, nodes_allowed) {
1456	if (h->surplus_huge_pages_node[node])
1457	goto found;
1458	}
1459	} else {
1460	for_each_node_mask_to_free(h, nr_nodes, node, nodes_allowed) {
1461	if (h->surplus_huge_pages_node[node] <
1462	h->nr_huge_pages_node[node])
1463	goto found;
1464	}
1465	}
1466	return 0;
1467
1468	found:
1469	h->surplus_huge_pages += delta;
1470	h->surplus_huge_pages_node[node] += delta;
1471	return 1;
1472	}
1473
1474	#define persistent_huge_pages(h) (h->nr_huge_pages - h->surplus_huge_pages)
1475	static unsigned long set_max_huge_pages(struct hstate *h, unsigned long count,
1476	nodemask_t *nodes_allowed)
1477	{
1478	unsigned long min_count, ret;
1479
1480	if (h->order >= MAX_ORDER)
1481	return h->max_huge_pages;
1482
1483	/*
1484	* Increase the pool size
1485	* First take pages out of surplus state. Then make up the
1486	* remaining difference by allocating fresh huge pages.
1487	*
1488	* We might race with alloc_buddy_huge_page() here and be unable
1489	* to convert a surplus huge page to a normal huge page. That is
1490	* not critical, though, it just means the overall size of the
1491	* pool might be one hugepage larger than it needs to be, but
1492	* within all the constraints specified by the sysctls.
1493	*/
1494	spin_lock(&hugetlb_lock);
1495	while (h->surplus_huge_pages && count > persistent_huge_pages(h)) {
1496	if (!adjust_pool_surplus(h, nodes_allowed, -1))
1497	break;
1498	}
1499
1500	while (count > persistent_huge_pages(h)) {
1501	/*
1502	* If this allocation races such that we no longer need the
1503	* page, free_huge_page will handle it by freeing the page
1504	* and reducing the surplus.
1505	*/
1506	spin_unlock(&hugetlb_lock);
1507	ret = alloc_fresh_huge_page(h, nodes_allowed);
1508	spin_lock(&hugetlb_lock);
1509	if (!ret)
1510	goto out;
1511
1512	/* Bail for signals. Probably ctrl-c from user */
1513	if (signal_pending(current))
1514	goto out;
1515	}
1516
1517	/*
1518	* Decrease the pool size
1519	* First return free pages to the buddy allocator (being careful
1520	* to keep enough around to satisfy reservations). Then place
1521	* pages into surplus state as needed so the pool will shrink
1522	* to the desired size as pages become free.
1523	*
1524	* By placing pages into the surplus state independent of the
1525	* overcommit value, we are allowing the surplus pool size to
1526	* exceed overcommit. There are few sane options here. Since
1527	* alloc_buddy_huge_page() is checking the global counter,
1528	* though, we'll note that we're not allowed to exceed surplus
1529	* and won't grow the pool anywhere else. Not until one of the
1530	* sysctls are changed, or the surplus pages go out of use.
1531	*/
1532	min_count = h->resv_huge_pages + h->nr_huge_pages - h->free_huge_pages;
1533	min_count = max(count, min_count);
1534	try_to_free_low(h, min_count, nodes_allowed);
1535	while (min_count < persistent_huge_pages(h)) {
1536	if (!free_pool_huge_page(h, nodes_allowed, 0))
1537	break;
1538	}
1539	while (count < persistent_huge_pages(h)) {
1540	if (!adjust_pool_surplus(h, nodes_allowed, 1))
1541	break;
1542	}
1543	out:
1544	ret = persistent_huge_pages(h);
1545	spin_unlock(&hugetlb_lock);
1546	return ret;
1547	}
1548
1549	#define HSTATE_ATTR_RO(_name) \
1550	static struct kobj_attribute _name##_attr = __ATTR_RO(_name)
1551
1552	#define HSTATE_ATTR(_name) \
1553	static struct kobj_attribute _name##_attr = \
1554	__ATTR(_name, 0644, _name##_show, _name##_store)
1555
1556	static struct kobject *hugepages_kobj;
1557	static struct kobject *hstate_kobjs[HUGE_MAX_HSTATE];
1558
1559	static struct hstate kobj_to_node_hstate(struct kobject kobj, int *nidp);
1560
1561	static struct hstate kobj_to_hstate(struct kobject kobj, int *nidp)
1562	{
1563	int i;
1564
1565	for (i = 0; i < HUGE_MAX_HSTATE; i++)
1566	if (hstate_kobjs[i] == kobj) {
1567	if (nidp)
1568	*nidp = NUMA_NO_NODE;
1569	return &hstates[i];
1570	}
1571
1572	return kobj_to_node_hstate(kobj, nidp);
1573	}
1574
1575	static ssize_t nr_hugepages_show_common(struct kobject *kobj,
1576	struct kobj_attribute attr, char buf)
1577	{
1578	struct hstate *h;
1579	unsigned long nr_huge_pages;
1580	int nid;
1581
1582	h = kobj_to_hstate(kobj, &nid);
1583	if (nid == NUMA_NO_NODE)
1584	nr_huge_pages = h->nr_huge_pages;
1585	else
1586	nr_huge_pages = h->nr_huge_pages_node[nid];
1587
1588	return sprintf(buf, "%lu\n", nr_huge_pages);
1589	}
1590
1591	static ssize_t nr_hugepages_store_common(bool obey_mempolicy,
1592	struct kobject kobj, struct kobj_attribute attr,
1593	const char *buf, size_t len)
1594	{
1595	int err;
1596	int nid;
1597	unsigned long count;
1598	struct hstate *h;
1599	NODEMASK_ALLOC(nodemask_t, nodes_allowed, GFP_KERNEL \| __GFP_NORETRY);
1600
1601	err = kstrtoul(buf, 10, &count);
1602	if (err)
1603	goto out;
1604
1605	h = kobj_to_hstate(kobj, &nid);
1606	if (h->order >= MAX_ORDER) {
1607	err = -EINVAL;
1608	goto out;
1609	}
1610
1611	if (nid == NUMA_NO_NODE) {
1612	/*
1613	* global hstate attribute
1614	*/
1615	if (!(obey_mempolicy &&
1616	init_nodemask_of_mempolicy(nodes_allowed))) {
1617	NODEMASK_FREE(nodes_allowed);
1618	nodes_allowed = &node_states[N_MEMORY];
1619	}
1620	} else if (nodes_allowed) {
1621	/*
1622	* per node hstate attribute: adjust count to global,
1623	* but restrict alloc/free to the specified node.
1624	*/
1625	count += h->nr_huge_pages - h->nr_huge_pages_node[nid];
1626	init_nodemask_of_node(nodes_allowed, nid);
1627	} else
1628	nodes_allowed = &node_states[N_MEMORY];
1629
1630	h->max_huge_pages = set_max_huge_pages(h, count, nodes_allowed);
1631
1632	if (nodes_allowed != &node_states[N_MEMORY])
1633	NODEMASK_FREE(nodes_allowed);
1634
1635	return len;
1636	out:
1637	NODEMASK_FREE(nodes_allowed);
1638	return err;
1639	}
1640
1641	static ssize_t nr_hugepages_show(struct kobject *kobj,
1642	struct kobj_attribute attr, char buf)
1643	{
1644	return nr_hugepages_show_common(kobj, attr, buf);
1645	}
1646
1647	static ssize_t nr_hugepages_store(struct kobject *kobj,
1648	struct kobj_attribute attr, const char buf, size_t len)
1649	{
1650	return nr_hugepages_store_common(false, kobj, attr, buf, len);
1651	}
1652	HSTATE_ATTR(nr_hugepages);
1653
1654	#ifdef CONFIG_NUMA
1655
1656	/*
1657	* hstate attribute for optionally mempolicy-based constraint on persistent
1658	* huge page alloc/free.
1659	*/
1660	static ssize_t nr_hugepages_mempolicy_show(struct kobject *kobj,
1661	struct kobj_attribute attr, char buf)
1662	{
1663	return nr_hugepages_show_common(kobj, attr, buf);
1664	}
1665
1666	static ssize_t nr_hugepages_mempolicy_store(struct kobject *kobj,
1667	struct kobj_attribute attr, const char buf, size_t len)
1668	{
1669	return nr_hugepages_store_common(true, kobj, attr, buf, len);
1670	}
1671	HSTATE_ATTR(nr_hugepages_mempolicy);
1672	#endif
1673
1674
1675	static ssize_t nr_overcommit_hugepages_show(struct kobject *kobj,
1676	struct kobj_attribute attr, char buf)
1677	{
1678	struct hstate *h = kobj_to_hstate(kobj, NULL);
1679	return sprintf(buf, "%lu\n", h->nr_overcommit_huge_pages);
1680	}
1681
1682	static ssize_t nr_overcommit_hugepages_store(struct kobject *kobj,
1683	struct kobj_attribute attr, const char buf, size_t count)
1684	{
1685	int err;
1686	unsigned long input;
1687	struct hstate *h = kobj_to_hstate(kobj, NULL);
1688
1689	if (h->order >= MAX_ORDER)
1690	return -EINVAL;
1691
1692	err = kstrtoul(buf, 10, &input);
1693	if (err)
1694	return err;
1695
1696	spin_lock(&hugetlb_lock);
1697	h->nr_overcommit_huge_pages = input;
1698	spin_unlock(&hugetlb_lock);
1699
1700	return count;
1701	}
1702	HSTATE_ATTR(nr_overcommit_hugepages);
1703
1704	static ssize_t free_hugepages_show(struct kobject *kobj,
1705	struct kobj_attribute attr, char buf)
1706	{
1707	struct hstate *h;
1708	unsigned long free_huge_pages;
1709	int nid;
1710
1711	h = kobj_to_hstate(kobj, &nid);
1712	if (nid == NUMA_NO_NODE)
1713	free_huge_pages = h->free_huge_pages;
1714	else
1715	free_huge_pages = h->free_huge_pages_node[nid];
1716
1717	return sprintf(buf, "%lu\n", free_huge_pages);
1718	}
1719	HSTATE_ATTR_RO(free_hugepages);
1720
1721	static ssize_t resv_hugepages_show(struct kobject *kobj,
1722	struct kobj_attribute attr, char buf)
1723	{
1724	struct hstate *h = kobj_to_hstate(kobj, NULL);
1725	return sprintf(buf, "%lu\n", h->resv_huge_pages);
1726	}
1727	HSTATE_ATTR_RO(resv_hugepages);
1728
1729	static ssize_t surplus_hugepages_show(struct kobject *kobj,
1730	struct kobj_attribute attr, char buf)
1731	{
1732	struct hstate *h;
1733	unsigned long surplus_huge_pages;
1734	int nid;
1735
1736	h = kobj_to_hstate(kobj, &nid);
1737	if (nid == NUMA_NO_NODE)
1738	surplus_huge_pages = h->surplus_huge_pages;
1739	else
1740	surplus_huge_pages = h->surplus_huge_pages_node[nid];
1741
1742	return sprintf(buf, "%lu\n", surplus_huge_pages);
1743	}
1744	HSTATE_ATTR_RO(surplus_hugepages);
1745
1746	static struct attribute *hstate_attrs[] = {
1747	&nr_hugepages_attr.attr,
1748	&nr_overcommit_hugepages_attr.attr,
1749	&free_hugepages_attr.attr,
1750	&resv_hugepages_attr.attr,
1751	&surplus_hugepages_attr.attr,
1752	#ifdef CONFIG_NUMA
1753	&nr_hugepages_mempolicy_attr.attr,
1754	#endif
1755	NULL,
1756	};
1757
1758	static struct attribute_group hstate_attr_group = {
1759	.attrs = hstate_attrs,
1760	};
1761
1762	static int hugetlb_sysfs_add_hstate(struct hstate h, struct kobject parent,
1763	struct kobject **hstate_kobjs,
1764	struct attribute_group *hstate_attr_group)
1765	{
1766	int retval;
1767	int hi = hstate_index(h);
1768
1769	hstate_kobjs[hi] = kobject_create_and_add(h->name, parent);
1770	if (!hstate_kobjs[hi])
1771	return -ENOMEM;
1772
1773	retval = sysfs_create_group(hstate_kobjs[hi], hstate_attr_group);
1774	if (retval)
1775	kobject_put(hstate_kobjs[hi]);
1776
1777	return retval;
1778	}
1779
1780	static void __init hugetlb_sysfs_init(void)
1781	{
1782	struct hstate *h;
1783	int err;
1784
1785	hugepages_kobj = kobject_create_and_add("hugepages", mm_kobj);
1786	if (!hugepages_kobj)
1787	return;
1788
1789	for_each_hstate(h) {
1790	err = hugetlb_sysfs_add_hstate(h, hugepages_kobj,
1791	hstate_kobjs, &hstate_attr_group);
1792	if (err)
1793	pr_err("Hugetlb: Unable to add hstate %s", h->name);
1794	}
1795	}
1796
1797	#ifdef CONFIG_NUMA
1798
1799	/*
1800	* node_hstate/s - associate per node hstate attributes, via their kobjects,
1801	* with node devices in node_devices[] using a parallel array. The array
1802	* index of a node device or _hstate == node id.
1803	* This is here to avoid any static dependency of the node device driver, in
1804	* the base kernel, on the hugetlb module.
1805	*/
1806	struct node_hstate {
1807	struct kobject *hugepages_kobj;
1808	struct kobject *hstate_kobjs[HUGE_MAX_HSTATE];
1809	};
1810	struct node_hstate node_hstates[MAX_NUMNODES];
1811
1812	/*
1813	* A subset of global hstate attributes for node devices
1814	*/
1815	static struct attribute *per_node_hstate_attrs[] = {
1816	&nr_hugepages_attr.attr,
1817	&free_hugepages_attr.attr,
1818	&surplus_hugepages_attr.attr,
1819	NULL,
1820	};
1821
1822	static struct attribute_group per_node_hstate_attr_group = {
1823	.attrs = per_node_hstate_attrs,
1824	};
1825
1826	/*
1827	* kobj_to_node_hstate - lookup global hstate for node device hstate attr kobj.
1828	* Returns node id via non-NULL nidp.
1829	*/
1830	static struct hstate kobj_to_node_hstate(struct kobject kobj, int *nidp)
1831	{
1832	int nid;
1833
1834	for (nid = 0; nid < nr_node_ids; nid++) {
1835	struct node_hstate *nhs = &node_hstates[nid];
1836	int i;
1837	for (i = 0; i < HUGE_MAX_HSTATE; i++)
1838	if (nhs->hstate_kobjs[i] == kobj) {
1839	if (nidp)
1840	*nidp = nid;
1841	return &hstates[i];
1842	}
1843	}
1844
1845	BUG();
1846	return NULL;
1847	}
1848
1849	/*
1850	* Unregister hstate attributes from a single node device.
1851	* No-op if no hstate attributes attached.
1852	*/
1853	static void hugetlb_unregister_node(struct node *node)
1854	{
1855	struct hstate *h;
1856	struct node_hstate *nhs = &node_hstates[node->dev.id];
1857
1858	if (!nhs->hugepages_kobj)
1859	return; /* no hstate attributes */
1860
1861	for_each_hstate(h) {
1862	int idx = hstate_index(h);
1863	if (nhs->hstate_kobjs[idx]) {
1864	kobject_put(nhs->hstate_kobjs[idx]);
1865	nhs->hstate_kobjs[idx] = NULL;
1866	}
1867	}
1868
1869	kobject_put(nhs->hugepages_kobj);
1870	nhs->hugepages_kobj = NULL;
1871	}
1872
1873	/*
1874	* hugetlb module exit: unregister hstate attributes from node devices
1875	* that have them.
1876	*/
1877	static void hugetlb_unregister_all_nodes(void)
1878	{
1879	int nid;
1880
1881	/*
1882	* disable node device registrations.
1883	*/
1884	register_hugetlbfs_with_node(NULL, NULL);
1885
1886	/*
1887	* remove hstate attributes from any nodes that have them.
1888	*/
1889	for (nid = 0; nid < nr_node_ids; nid++)
1890	hugetlb_unregister_node(node_devices[nid]);
1891	}
1892
1893	/*
1894	* Register hstate attributes for a single node device.
1895	* No-op if attributes already registered.
1896	*/
1897	static void hugetlb_register_node(struct node *node)
1898	{
1899	struct hstate *h;
1900	struct node_hstate *nhs = &node_hstates[node->dev.id];
1901	int err;
1902
1903	if (nhs->hugepages_kobj)
1904	return; /* already allocated */
1905
1906	nhs->hugepages_kobj = kobject_create_and_add("hugepages",
1907	&node->dev.kobj);
1908	if (!nhs->hugepages_kobj)
1909	return;
1910
1911	for_each_hstate(h) {
1912	err = hugetlb_sysfs_add_hstate(h, nhs->hugepages_kobj,
1913	nhs->hstate_kobjs,
1914	&per_node_hstate_attr_group);
1915	if (err) {
1916	pr_err("Hugetlb: Unable to add hstate %s for node %d\n",
1917	h->name, node->dev.id);
1918	hugetlb_unregister_node(node);
1919	break;
1920	}
1921	}
1922	}
1923
1924	/*
1925	* hugetlb init time: register hstate attributes for all registered node
1926	* devices of nodes that have memory. All on-line nodes should have
1927	* registered their associated device by this time.
1928	*/
1929	static void hugetlb_register_all_nodes(void)
1930	{
1931	int nid;
1932
1933	for_each_node_state(nid, N_MEMORY) {
1934	struct node *node = node_devices[nid];
1935	if (node->dev.id == nid)
1936	hugetlb_register_node(node);
1937	}
1938
1939	/*
1940	* Let the node device driver know we're here so it can
1941	* [un]register hstate attributes on node hotplug.
1942	*/
1943	register_hugetlbfs_with_node(hugetlb_register_node,
1944	hugetlb_unregister_node);
1945	}
1946	#else /* !CONFIG_NUMA */
1947
1948	static struct hstate kobj_to_node_hstate(struct kobject kobj, int *nidp)
1949	{
1950	BUG();
1951	if (nidp)
1952	*nidp = -1;
1953	return NULL;
1954	}
1955
1956	static void hugetlb_unregister_all_nodes(void) { }
1957
1958	static void hugetlb_register_all_nodes(void) { }
1959
1960	#endif
1961
1962	static void __exit hugetlb_exit(void)
1963	{
1964	struct hstate *h;
1965
1966	hugetlb_unregister_all_nodes();
1967
1968	for_each_hstate(h) {
1969	kobject_put(hstate_kobjs[hstate_index(h)]);
1970	}
1971
1972	kobject_put(hugepages_kobj);
1973	}
1974	module_exit(hugetlb_exit);
1975
1976	static int __init hugetlb_init(void)
1977	{
1978	/* Some platform decide whether they support huge pages at boot
1979	* time. On these, such as powerpc, HPAGE_SHIFT is set to 0 when
1980	* there is no such support
1981	*/
1982	if (HPAGE_SHIFT == 0)
1983	return 0;
1984
1985	if (!size_to_hstate(default_hstate_size)) {
1986	default_hstate_size = HPAGE_SIZE;
1987	if (!size_to_hstate(default_hstate_size))
1988	hugetlb_add_hstate(HUGETLB_PAGE_ORDER);
1989	}
1990	default_hstate_idx = hstate_index(size_to_hstate(default_hstate_size));
1991	if (default_hstate_max_huge_pages)
1992	default_hstate.max_huge_pages = default_hstate_max_huge_pages;
1993
1994	hugetlb_init_hstates();
1995	gather_bootmem_prealloc();
1996	report_hugepages();
1997
1998	hugetlb_sysfs_init();
1999	hugetlb_register_all_nodes();
2000	hugetlb_cgroup_file_init();
2001
2002	return 0;
2003	}
2004	module_init(hugetlb_init);
2005
2006	/* Should be called on processing a hugepagesz=... option */
2007	void __init hugetlb_add_hstate(unsigned order)
2008	{
2009	struct hstate *h;
2010	unsigned long i;
2011
2012	if (size_to_hstate(PAGE_SIZE << order)) {
2013	pr_warning("hugepagesz= specified twice, ignoring\n");
2014	return;
2015	}
2016	BUG_ON(hugetlb_max_hstate >= HUGE_MAX_HSTATE);
2017	BUG_ON(order == 0);
2018	h = &hstates[hugetlb_max_hstate++];
2019	h->order = order;
2020	h->mask = ~((1ULL << (order + PAGE_SHIFT)) - 1);
2021	h->nr_huge_pages = 0;
2022	h->free_huge_pages = 0;
2023	for (i = 0; i < MAX_NUMNODES; ++i)
2024	INIT_LIST_HEAD(&h->hugepage_freelists[i]);
2025	INIT_LIST_HEAD(&h->hugepage_activelist);
2026	h->next_nid_to_alloc = first_node(node_states[N_MEMORY]);
2027	h->next_nid_to_free = first_node(node_states[N_MEMORY]);
2028	snprintf(h->name, HSTATE_NAME_LEN, "hugepages-%lukB",
2029	huge_page_size(h)/1024);
2030
2031	parsed_hstate = h;
2032	}
2033
2034	static int __init hugetlb_nrpages_setup(char *s)
2035	{
2036	unsigned long *mhp;
2037	static unsigned long *last_mhp;
2038
2039	/*
2040	* !hugetlb_max_hstate means we haven't parsed a hugepagesz= parameter yet,
2041	* so this hugepages= parameter goes to the "default hstate".
2042	*/
2043	if (!hugetlb_max_hstate)
2044	mhp = &default_hstate_max_huge_pages;
2045	else
2046	mhp = &parsed_hstate->max_huge_pages;
2047
2048	if (mhp == last_mhp) {
2049	pr_warning("hugepages= specified twice without "
2050	"interleaving hugepagesz=, ignoring\n");
2051	return 1;
2052	}
2053
2054	if (sscanf(s, "%lu", mhp) <= 0)
2055	*mhp = 0;
2056
2057	/*
2058	* Global state is always initialized later in hugetlb_init.
2059	* But we need to allocate >= MAX_ORDER hstates here early to still
2060	* use the bootmem allocator.
2061	*/
2062	if (hugetlb_max_hstate && parsed_hstate->order >= MAX_ORDER)
2063	hugetlb_hstate_alloc_pages(parsed_hstate);
2064
2065	last_mhp = mhp;
2066
2067	return 1;
2068	}
2069	__setup("hugepages=", hugetlb_nrpages_setup);
2070
2071	static int __init hugetlb_default_setup(char *s)
2072	{
2073	default_hstate_size = memparse(s, &s);
2074	return 1;
2075	}
2076	__setup("default_hugepagesz=", hugetlb_default_setup);
2077
2078	static unsigned int cpuset_mems_nr(unsigned int *array)
2079	{
2080	int node;
2081	unsigned int nr = 0;
2082
2083	for_each_node_mask(node, cpuset_current_mems_allowed)
2084	nr += array[node];
2085
2086	return nr;
2087	}
2088
2089	#ifdef CONFIG_SYSCTL
2090	static int hugetlb_sysctl_handler_common(bool obey_mempolicy,
2091	struct ctl_table *table, int write,
2092	void __user buffer, size_t length, loff_t *ppos)
2093	{
2094	struct hstate *h = &default_hstate;
2095	unsigned long tmp;
2096	int ret;
2097
2098	tmp = h->max_huge_pages;
2099
2100	if (write && h->order >= MAX_ORDER)
2101	return -EINVAL;
2102
2103	table->data = &tmp;
2104	table->maxlen = sizeof(unsigned long);
2105	ret = proc_doulongvec_minmax(table, write, buffer, length, ppos);
2106	if (ret)
2107	goto out;
2108
2109	if (write) {
2110	NODEMASK_ALLOC(nodemask_t, nodes_allowed,
2111	GFP_KERNEL \| __GFP_NORETRY);
2112	if (!(obey_mempolicy &&
2113	init_nodemask_of_mempolicy(nodes_allowed))) {
2114	NODEMASK_FREE(nodes_allowed);
2115	nodes_allowed = &node_states[N_MEMORY];
2116	}
2117	h->max_huge_pages = set_max_huge_pages(h, tmp, nodes_allowed);
2118
2119	if (nodes_allowed != &node_states[N_MEMORY])
2120	NODEMASK_FREE(nodes_allowed);
2121	}
2122	out:
2123	return ret;
2124	}
2125
2126	int hugetlb_sysctl_handler(struct ctl_table *table, int write,
2127	void __user buffer, size_t length, loff_t *ppos)
2128	{
2129
2130	return hugetlb_sysctl_handler_common(false, table, write,
2131	buffer, length, ppos);
2132	}
2133
2134	#ifdef CONFIG_NUMA
2135	int hugetlb_mempolicy_sysctl_handler(struct ctl_table *table, int write,
2136	void __user buffer, size_t length, loff_t *ppos)
2137	{
2138	return hugetlb_sysctl_handler_common(true, table, write,
2139	buffer, length, ppos);
2140	}
2141	#endif /* CONFIG_NUMA */
2142
2143	int hugetlb_overcommit_handler(struct ctl_table *table, int write,
2144	void __user *buffer,
2145	size_t length, loff_t ppos)
2146	{
2147	struct hstate *h = &default_hstate;
2148	unsigned long tmp;
2149	int ret;
2150
2151	tmp = h->nr_overcommit_huge_pages;
2152
2153	if (write && h->order >= MAX_ORDER)
2154	return -EINVAL;
2155
2156	table->data = &tmp;
2157	table->maxlen = sizeof(unsigned long);
2158	ret = proc_doulongvec_minmax(table, write, buffer, length, ppos);
2159	if (ret)
2160	goto out;
2161
2162	if (write) {
2163	spin_lock(&hugetlb_lock);
2164	h->nr_overcommit_huge_pages = tmp;
2165	spin_unlock(&hugetlb_lock);
2166	}
2167	out:
2168	return ret;
2169	}
2170
2171	#endif /* CONFIG_SYSCTL */
2172
2173	void hugetlb_report_meminfo(struct seq_file *m)
2174	{
2175	struct hstate *h = &default_hstate;
2176	seq_printf(m,
2177	"HugePages_Total: %5lu\n"
2178	"HugePages_Free: %5lu\n"
2179	"HugePages_Rsvd: %5lu\n"
2180	"HugePages_Surp: %5lu\n"
2181	"Hugepagesize: %8lu kB\n",
2182	h->nr_huge_pages,
2183	h->free_huge_pages,
2184	h->resv_huge_pages,
2185	h->surplus_huge_pages,
2186	1UL << (huge_page_order(h) + PAGE_SHIFT - 10));
2187	}
2188
2189	int hugetlb_report_node_meminfo(int nid, char *buf)
2190	{
2191	struct hstate *h = &default_hstate;
2192	return sprintf(buf,
2193	"Node %d HugePages_Total: %5u\n"
2194	"Node %d HugePages_Free: %5u\n"
2195	"Node %d HugePages_Surp: %5u\n",
2196	nid, h->nr_huge_pages_node[nid],
2197	nid, h->free_huge_pages_node[nid],
2198	nid, h->surplus_huge_pages_node[nid]);
2199	}
2200
2201	void hugetlb_show_meminfo(void)
2202	{
2203	struct hstate *h;
2204	int nid;
2205
2206	for_each_node_state(nid, N_MEMORY)
2207	for_each_hstate(h)
2208	pr_info("Node %d hugepages_total=%u hugepages_free=%u hugepages_surp=%u hugepages_size=%lukB\n",
2209	nid,
2210	h->nr_huge_pages_node[nid],
2211	h->free_huge_pages_node[nid],
2212	h->surplus_huge_pages_node[nid],
2213	1UL << (huge_page_order(h) + PAGE_SHIFT - 10));
2214	}
2215
2216	/* Return the number pages of memory we physically have, in PAGE_SIZE units. */
2217	unsigned long hugetlb_total_pages(void)
2218	{
2219	struct hstate *h;
2220	unsigned long nr_total_pages = 0;
2221
2222	for_each_hstate(h)
2223	nr_total_pages += h->nr_huge_pages * pages_per_huge_page(h);
2224	return nr_total_pages;
2225	}
2226
2227	static int hugetlb_acct_memory(struct hstate *h, long delta)
2228	{
2229	int ret = -ENOMEM;
2230
2231	spin_lock(&hugetlb_lock);
2232	/*
2233	* When cpuset is configured, it breaks the strict hugetlb page
2234	* reservation as the accounting is done on a global variable. Such
2235	* reservation is completely rubbish in the presence of cpuset because
2236	* the reservation is not checked against page availability for the
2237	* current cpuset. Application can still potentially OOM'ed by kernel
2238	* with lack of free htlb page in cpuset that the task is in.
2239	* Attempt to enforce strict accounting with cpuset is almost
2240	* impossible (or too ugly) because cpuset is too fluid that
2241	* task or memory node can be dynamically moved between cpusets.
2242	*
2243	* The change of semantics for shared hugetlb mapping with cpuset is
2244	* undesirable. However, in order to preserve some of the semantics,
2245	* we fall back to check against current free page availability as
2246	* a best attempt and hopefully to minimize the impact of changing
2247	* semantics that cpuset has.
2248	*/
2249	if (delta > 0) {
2250	if (gather_surplus_pages(h, delta) < 0)
2251	goto out;
2252
2253	if (delta > cpuset_mems_nr(h->free_huge_pages_node)) {
2254	return_unused_surplus_pages(h, delta);
2255	goto out;
2256	}
2257	}
2258
2259	ret = 0;
2260	if (delta < 0)
2261	return_unused_surplus_pages(h, (unsigned long) -delta);
2262
2263	out:
2264	spin_unlock(&hugetlb_lock);
2265	return ret;
2266	}
2267
2268	static void hugetlb_vm_op_open(struct vm_area_struct *vma)
2269	{
2270	struct resv_map *resv = vma_resv_map(vma);
2271
2272	/*
2273	* This new VMA should share its siblings reservation map if present.
2274	* The VMA will only ever have a valid reservation map pointer where
2275	* it is being copied for another still existing VMA. As that VMA
2276	* has a reference to the reservation map it cannot disappear until
2277	* after this open call completes. It is therefore safe to take a
2278	* new reference here without additional locking.
2279	*/
2280	if (resv)
2281	kref_get(&resv->refs);
2282	}
2283
2284	static void resv_map_put(struct vm_area_struct *vma)
2285	{
2286	struct resv_map *resv = vma_resv_map(vma);
2287
2288	if (!resv)
2289	return;
2290	kref_put(&resv->refs, resv_map_release);
2291	}
2292
2293	static void hugetlb_vm_op_close(struct vm_area_struct *vma)
2294	{
2295	struct hstate *h = hstate_vma(vma);
2296	struct resv_map *resv = vma_resv_map(vma);
2297	struct hugepage_subpool *spool = subpool_vma(vma);
2298	unsigned long reserve;
2299	unsigned long start;
2300	unsigned long end;
2301
2302	if (resv) {
2303	start = vma_hugecache_offset(h, vma, vma->vm_start);
2304	end = vma_hugecache_offset(h, vma, vma->vm_end);
2305
2306	reserve = (end - start) -
2307	region_count(&resv->regions, start, end);
2308
2309	resv_map_put(vma);
2310
2311	if (reserve) {
2312	hugetlb_acct_memory(h, -reserve);
2313	hugepage_subpool_put_pages(spool, reserve);
2314	}
2315	}
2316	}
2317
2318	/*
2319	* We cannot handle pagefaults against hugetlb pages at all. They cause
2320	* handle_mm_fault() to try to instantiate regular-sized pages in the
2321	* hugegpage VMA. do_page_fault() is supposed to trap this, so BUG is we get
2322	* this far.
2323	*/
2324	static int hugetlb_vm_op_fault(struct vm_area_struct vma, struct vm_fault vmf)
2325	{
2326	BUG();
2327	return 0;
2328	}
2329
2330	const struct vm_operations_struct hugetlb_vm_ops = {
2331	.fault = hugetlb_vm_op_fault,
2332	.open = hugetlb_vm_op_open,
2333	.close = hugetlb_vm_op_close,
2334	};
2335
2336	static pte_t make_huge_pte(struct vm_area_struct vma, struct page page,
2337	int writable)
2338	{
2339	pte_t entry;
2340
2341	if (writable) {
2342	entry = huge_pte_mkwrite(huge_pte_mkdirty(mk_huge_pte(page,
2343	vma->vm_page_prot)));
2344	} else {
2345	entry = huge_pte_wrprotect(mk_huge_pte(page,
2346	vma->vm_page_prot));
2347	}
2348	entry = pte_mkyoung(entry);
2349	entry = pte_mkhuge(entry);
2350	entry = arch_make_huge_pte(entry, vma, page, writable);
2351
2352	return entry;
2353	}
2354
2355	static void set_huge_ptep_writable(struct vm_area_struct *vma,
2356	unsigned long address, pte_t *ptep)
2357	{
2358	pte_t entry;
2359
2360	entry = huge_pte_mkwrite(huge_pte_mkdirty(huge_ptep_get(ptep)));
2361	if (huge_ptep_set_access_flags(vma, address, ptep, entry, 1))
2362	update_mmu_cache(vma, address, ptep);
2363	}
2364
2365
2366	int copy_hugetlb_page_range(struct mm_struct dst, struct mm_struct src,
2367	struct vm_area_struct *vma)
2368	{
2369	pte_t src_pte, dst_pte, entry;
2370	struct page *ptepage;
2371	unsigned long addr;
2372	int cow;
2373	struct hstate *h = hstate_vma(vma);
2374	unsigned long sz = huge_page_size(h);
2375
2376	cow = (vma->vm_flags & (VM_SHARED \| VM_MAYWRITE)) == VM_MAYWRITE;
2377
2378	for (addr = vma->vm_start; addr < vma->vm_end; addr += sz) {
2379	src_pte = huge_pte_offset(src, addr);
2380	if (!src_pte)
2381	continue;
2382	dst_pte = huge_pte_alloc(dst, addr, sz);
2383	if (!dst_pte)
2384	goto nomem;
2385
2386	/* If the pagetables are shared don't copy or take references */
2387	if (dst_pte == src_pte)
2388	continue;
2389
2390	spin_lock(&dst->page_table_lock);
2391	spin_lock_nested(&src->page_table_lock, SINGLE_DEPTH_NESTING);
2392	if (!huge_pte_none(huge_ptep_get(src_pte))) {
2393	if (cow)
2394	huge_ptep_set_wrprotect(src, addr, src_pte);
2395	entry = huge_ptep_get(src_pte);
2396	ptepage = pte_page(entry);
2397	get_page(ptepage);
2398	page_dup_rmap(ptepage);
2399	set_huge_pte_at(dst, addr, dst_pte, entry);
2400	}
2401	spin_unlock(&src->page_table_lock);
2402	spin_unlock(&dst->page_table_lock);
2403	}
2404	return 0;
2405
2406	nomem:
2407	return -ENOMEM;
2408	}
2409
2410	static int is_hugetlb_entry_migration(pte_t pte)
2411	{
2412	swp_entry_t swp;
2413
2414	if (huge_pte_none(pte) \|\| pte_present(pte))
2415	return 0;
2416	swp = pte_to_swp_entry(pte);
2417	if (non_swap_entry(swp) && is_migration_entry(swp))
2418	return 1;
2419	else
2420	return 0;
2421	}
2422
2423	static int is_hugetlb_entry_hwpoisoned(pte_t pte)
2424	{
2425	swp_entry_t swp;
2426
2427	if (huge_pte_none(pte) \|\| pte_present(pte))
2428	return 0;
2429	swp = pte_to_swp_entry(pte);
2430	if (non_swap_entry(swp) && is_hwpoison_entry(swp))
2431	return 1;
2432	else
2433	return 0;
2434	}
2435
2436	void __unmap_hugepage_range(struct mmu_gather tlb, struct vm_area_struct vma,
2437	unsigned long start, unsigned long end,
2438	struct page *ref_page)
2439	{
2440	int force_flush = 0;
2441	struct mm_struct *mm = vma->vm_mm;
2442	unsigned long address;
2443	pte_t *ptep;
2444	pte_t pte;
2445	struct page *page;
2446	struct hstate *h = hstate_vma(vma);
2447	unsigned long sz = huge_page_size(h);
2448	const unsigned long mmun_start = start; /* For mmu_notifiers */
2449	const unsigned long mmun_end = end; /* For mmu_notifiers */
2450
2451	WARN_ON(!is_vm_hugetlb_page(vma));
2452	BUG_ON(start & ~huge_page_mask(h));
2453	BUG_ON(end & ~huge_page_mask(h));
2454
2455	tlb_start_vma(tlb, vma);
2456	mmu_notifier_invalidate_range_start(mm, mmun_start, mmun_end);
2457	again:
2458	spin_lock(&mm->page_table_lock);
2459	for (address = start; address < end; address += sz) {
2460	ptep = huge_pte_offset(mm, address);
2461	if (!ptep)
2462	continue;
2463
2464	if (huge_pmd_unshare(mm, &address, ptep))
2465	continue;
2466
2467	pte = huge_ptep_get(ptep);
2468	if (huge_pte_none(pte))
2469	continue;
2470
2471	/*
2472	* HWPoisoned hugepage is already unmapped and dropped reference
2473	*/
2474	if (unlikely(is_hugetlb_entry_hwpoisoned(pte))) {
2475	huge_pte_clear(mm, address, ptep);
2476	continue;
2477	}
2478
2479	page = pte_page(pte);
2480	/*
2481	* If a reference page is supplied, it is because a specific
2482	* page is being unmapped, not a range. Ensure the page we
2483	* are about to unmap is the actual page of interest.
2484	*/
2485	if (ref_page) {
2486	if (page != ref_page)
2487	continue;
2488
2489	/*
2490	* Mark the VMA as having unmapped its page so that
2491	* future faults in this VMA will fail rather than
2492	* looking like data was lost
2493	*/
2494	set_vma_resv_flags(vma, HPAGE_RESV_UNMAPPED);
2495	}
2496
2497	pte = huge_ptep_get_and_clear(mm, address, ptep);
2498	tlb_remove_tlb_entry(tlb, ptep, address);
2499	if (huge_pte_dirty(pte))
2500	set_page_dirty(page);
2501
2502	page_remove_rmap(page);
2503	force_flush = !__tlb_remove_page(tlb, page);
2504	if (force_flush)
2505	break;
2506	/* Bail out after unmapping reference page if supplied */
2507	if (ref_page)
2508	break;
2509	}
2510	spin_unlock(&mm->page_table_lock);
2511	/*
2512	* mmu_gather ran out of room to batch pages, we break out of
2513	* the PTE lock to avoid doing the potential expensive TLB invalidate
2514	* and page-free while holding it.
2515	*/
2516	if (force_flush) {
2517	force_flush = 0;
2518	tlb_flush_mmu(tlb);
2519	if (address < end && !ref_page)
2520	goto again;
2521	}
2522	mmu_notifier_invalidate_range_end(mm, mmun_start, mmun_end);
2523	tlb_end_vma(tlb, vma);
2524	}
2525
2526	void __unmap_hugepage_range_final(struct mmu_gather *tlb,
2527	struct vm_area_struct *vma, unsigned long start,
2528	unsigned long end, struct page *ref_page)
2529	{
2530	__unmap_hugepage_range(tlb, vma, start, end, ref_page);
2531
2532	/*
2533	* Clear this flag so that x86's huge_pmd_share page_table_shareable
2534	* test will fail on a vma being torn down, and not grab a page table
2535	* on its way out. We're lucky that the flag has such an appropriate
2536	* name, and can in fact be safely cleared here. We could clear it
2537	* before the __unmap_hugepage_range above, but all that's necessary
2538	* is to clear it before releasing the i_mmap_mutex. This works
2539	* because in the context this is called, the VMA is about to be
2540	* destroyed and the i_mmap_mutex is held.
2541	*/
2542	vma->vm_flags &= ~VM_MAYSHARE;
2543	}
2544
2545	void unmap_hugepage_range(struct vm_area_struct *vma, unsigned long start,
2546	unsigned long end, struct page *ref_page)
2547	{
2548	struct mm_struct *mm;
2549	struct mmu_gather tlb;
2550
2551	mm = vma->vm_mm;
2552
2553	tlb_gather_mmu(&tlb, mm, start, end);
2554	__unmap_hugepage_range(&tlb, vma, start, end, ref_page);
2555	tlb_finish_mmu(&tlb, start, end);
2556	}
2557
2558	/*
2559	* This is called when the original mapper is failing to COW a MAP_PRIVATE
2560	* mappping it owns the reserve page for. The intention is to unmap the page
2561	* from other VMAs and let the children be SIGKILLed if they are faulting the
2562	* same region.
2563	*/
2564	static int unmap_ref_private(struct mm_struct mm, struct vm_area_struct vma,
2565	struct page *page, unsigned long address)
2566	{
2567	struct hstate *h = hstate_vma(vma);
2568	struct vm_area_struct *iter_vma;
2569	struct address_space *mapping;
2570	pgoff_t pgoff;
2571
2572	/*
2573	* vm_pgoff is in PAGE_SIZE units, hence the different calculation
2574	* from page cache lookup which is in HPAGE_SIZE units.
2575	*/
2576	address = address & huge_page_mask(h);
2577	pgoff = ((address - vma->vm_start) >> PAGE_SHIFT) +
2578	vma->vm_pgoff;
2579	mapping = file_inode(vma->vm_file)->i_mapping;
2580
2581	/*
2582	* Take the mapping lock for the duration of the table walk. As
2583	* this mapping should be shared between all the VMAs,
2584	* __unmap_hugepage_range() is called as the lock is already held
2585	*/
2586	mutex_lock(&mapping->i_mmap_mutex);
2587	vma_interval_tree_foreach(iter_vma, &mapping->i_mmap, pgoff, pgoff) {
2588	/* Do not unmap the current VMA */
2589	if (iter_vma == vma)
2590	continue;
2591
2592	/*
2593	* Unmap the page from other VMAs without their own reserves.
2594	* They get marked to be SIGKILLed if they fault in these
2595	* areas. This is because a future no-page fault on this VMA
2596	* could insert a zeroed page instead of the data existing
2597	* from the time of fork. This would look like data corruption
2598	*/
2599	if (!is_vma_resv_set(iter_vma, HPAGE_RESV_OWNER))
2600	unmap_hugepage_range(iter_vma, address,
2601	address + huge_page_size(h), page);
2602	}
2603	mutex_unlock(&mapping->i_mmap_mutex);
2604
2605	return 1;
2606	}
2607
2608	/*
2609	* Hugetlb_cow() should be called with page lock of the original hugepage held.
2610	* Called with hugetlb_instantiation_mutex held and pte_page locked so we
2611	* cannot race with other handlers or page migration.
2612	* Keep the pte_same checks anyway to make transition from the mutex easier.
2613	*/
2614	static int hugetlb_cow(struct mm_struct mm, struct vm_area_struct vma,
2615	unsigned long address, pte_t *ptep, pte_t pte,
2616	struct page *pagecache_page)
2617	{
2618	struct hstate *h = hstate_vma(vma);
2619	struct page old_page, new_page;
2620	int outside_reserve = 0;
2621	unsigned long mmun_start; /* For mmu_notifiers */
2622	unsigned long mmun_end; /* For mmu_notifiers */
2623
2624	old_page = pte_page(pte);
2625
2626	retry_avoidcopy:
2627	/* If no-one else is actually using this page, avoid the copy
2628	* and just make the page writable */
2629	if (page_mapcount(old_page) == 1 && PageAnon(old_page)) {
2630	page_move_anon_rmap(old_page, vma, address);
2631	set_huge_ptep_writable(vma, address, ptep);
2632	return 0;
2633	}
2634
2635	/*
2636	* If the process that created a MAP_PRIVATE mapping is about to
2637	* perform a COW due to a shared page count, attempt to satisfy
2638	* the allocation without using the existing reserves. The pagecache
2639	* page is used to determine if the reserve at this address was
2640	* consumed or not. If reserves were used, a partial faulted mapping
2641	* at the time of fork() could consume its reserves on COW instead
2642	* of the full address range.
2643	*/
2644	if (is_vma_resv_set(vma, HPAGE_RESV_OWNER) &&
2645	old_page != pagecache_page)
2646	outside_reserve = 1;
2647
2648	page_cache_get(old_page);
2649
2650	/* Drop page_table_lock as buddy allocator may be called */
2651	spin_unlock(&mm->page_table_lock);
2652	new_page = alloc_huge_page(vma, address, outside_reserve);
2653
2654	if (IS_ERR(new_page)) {
2655	long err = PTR_ERR(new_page);
2656	page_cache_release(old_page);
2657
2658	/*
2659	* If a process owning a MAP_PRIVATE mapping fails to COW,
2660	* it is due to references held by a child and an insufficient
2661	* huge page pool. To guarantee the original mappers
2662	* reliability, unmap the page from child processes. The child
2663	* may get SIGKILLed if it later faults.
2664	*/
2665	if (outside_reserve) {
2666	BUG_ON(huge_pte_none(pte));
2667	if (unmap_ref_private(mm, vma, old_page, address)) {
2668	BUG_ON(huge_pte_none(pte));
2669	spin_lock(&mm->page_table_lock);
2670	ptep = huge_pte_offset(mm, address & huge_page_mask(h));
2671	if (likely(pte_same(huge_ptep_get(ptep), pte)))
2672	goto retry_avoidcopy;
2673	/*
2674	* race occurs while re-acquiring page_table_lock, and
2675	* our job is done.
2676	*/
2677	return 0;
2678	}
2679	WARN_ON_ONCE(1);
2680	}
2681
2682	/* Caller expects lock to be held */
2683	spin_lock(&mm->page_table_lock);
2684	if (err == -ENOMEM)
2685	return VM_FAULT_OOM;
2686	else
2687	return VM_FAULT_SIGBUS;
2688	}
2689
2690	/*
2691	* When the original hugepage is shared one, it does not have
2692	* anon_vma prepared.
2693	*/
2694	if (unlikely(anon_vma_prepare(vma))) {
2695	page_cache_release(new_page);
2696	page_cache_release(old_page);
2697	/* Caller expects lock to be held */
2698	spin_lock(&mm->page_table_lock);
2699	return VM_FAULT_OOM;
2700	}
2701
2702	copy_user_huge_page(new_page, old_page, address, vma,
2703	pages_per_huge_page(h));
2704	__SetPageUptodate(new_page);
2705
2706	mmun_start = address & huge_page_mask(h);
2707	mmun_end = mmun_start + huge_page_size(h);
2708	mmu_notifier_invalidate_range_start(mm, mmun_start, mmun_end);
2709	/*
2710	* Retake the page_table_lock to check for racing updates
2711	* before the page tables are altered
2712	*/
2713	spin_lock(&mm->page_table_lock);
2714	ptep = huge_pte_offset(mm, address & huge_page_mask(h));
2715	if (likely(pte_same(huge_ptep_get(ptep), pte))) {
2716	ClearPagePrivate(new_page);
2717
2718	/* Break COW */
2719	huge_ptep_clear_flush(vma, address, ptep);
2720	set_huge_pte_at(mm, address, ptep,
2721	make_huge_pte(vma, new_page, 1));
2722	page_remove_rmap(old_page);
2723	hugepage_add_new_anon_rmap(new_page, vma, address);
2724	/* Make the old page be freed below */
2725	new_page = old_page;
2726	}
2727	spin_unlock(&mm->page_table_lock);
2728	mmu_notifier_invalidate_range_end(mm, mmun_start, mmun_end);
2729	page_cache_release(new_page);
2730	page_cache_release(old_page);
2731
2732	/* Caller expects lock to be held */
2733	spin_lock(&mm->page_table_lock);
2734	return 0;
2735	}
2736
2737	/* Return the pagecache page at a given address within a VMA */
2738	static struct page hugetlbfs_pagecache_page(struct hstate h,
2739	struct vm_area_struct *vma, unsigned long address)
2740	{
2741	struct address_space *mapping;
2742	pgoff_t idx;
2743
2744	mapping = vma->vm_file->f_mapping;
2745	idx = vma_hugecache_offset(h, vma, address);
2746
2747	return find_lock_page(mapping, idx);
2748	}
2749
2750	/*
2751	* Return whether there is a pagecache page to back given address within VMA.
2752	* Caller follow_hugetlb_page() holds page_table_lock so we cannot lock_page.
2753	*/
2754	static bool hugetlbfs_pagecache_present(struct hstate *h,
2755	struct vm_area_struct *vma, unsigned long address)
2756	{
2757	struct address_space *mapping;
2758	pgoff_t idx;
2759	struct page *page;
2760
2761	mapping = vma->vm_file->f_mapping;
2762	idx = vma_hugecache_offset(h, vma, address);
2763
2764	page = find_get_page(mapping, idx);
2765	if (page)
2766	put_page(page);
2767	return page != NULL;
2768	}
2769
2770	static int hugetlb_no_page(struct mm_struct mm, struct vm_area_struct vma,
2771	unsigned long address, pte_t *ptep, unsigned int flags)
2772	{
2773	struct hstate *h = hstate_vma(vma);
2774	int ret = VM_FAULT_SIGBUS;
2775	int anon_rmap = 0;
2776	pgoff_t idx;
2777	unsigned long size;
2778	struct page *page;
2779	struct address_space *mapping;
2780	pte_t new_pte;
2781
2782	/*
2783	* Currently, we are forced to kill the process in the event the
2784	* original mapper has unmapped pages from the child due to a failed
2785	* COW. Warn that such a situation has occurred as it may not be obvious
2786	*/
2787	if (is_vma_resv_set(vma, HPAGE_RESV_UNMAPPED)) {
2788	pr_warning("PID %d killed due to inadequate hugepage pool\n",
2789	current->pid);
2790	return ret;
2791	}
2792
2793	mapping = vma->vm_file->f_mapping;
2794	idx = vma_hugecache_offset(h, vma, address);
2795
2796	/*
2797	* Use page lock to guard against racing truncation
2798	* before we get page_table_lock.
2799	*/
2800	retry:
2801	page = find_lock_page(mapping, idx);
2802	if (!page) {
2803	size = i_size_read(mapping->host) >> huge_page_shift(h);
2804	if (idx >= size)
2805	goto out;
2806	page = alloc_huge_page(vma, address, 0);
2807	if (IS_ERR(page)) {
2808	ret = PTR_ERR(page);
2809	if (ret == -ENOMEM)
2810	ret = VM_FAULT_OOM;
2811	else
2812	ret = VM_FAULT_SIGBUS;
2813	goto out;
2814	}
2815	clear_huge_page(page, address, pages_per_huge_page(h));
2816	__SetPageUptodate(page);
2817
2818	if (vma->vm_flags & VM_MAYSHARE) {
2819	int err;
2820	struct inode *inode = mapping->host;
2821
2822	err = add_to_page_cache(page, mapping, idx, GFP_KERNEL);
2823	if (err) {
2824	put_page(page);
2825	if (err == -EEXIST)
2826	goto retry;
2827	goto out;
2828	}
2829	ClearPagePrivate(page);
2830
2831	spin_lock(&inode->i_lock);
2832	inode->i_blocks += blocks_per_huge_page(h);
2833	spin_unlock(&inode->i_lock);
2834	} else {
2835	lock_page(page);
2836	if (unlikely(anon_vma_prepare(vma))) {
2837	ret = VM_FAULT_OOM;
2838	goto backout_unlocked;
2839	}
2840	anon_rmap = 1;
2841	}
2842	} else {
2843	/*
2844	* If memory error occurs between mmap() and fault, some process
2845	* don't have hwpoisoned swap entry for errored virtual address.
2846	* So we need to block hugepage fault by PG_hwpoison bit check.
2847	*/
2848	if (unlikely(PageHWPoison(page))) {
2849	ret = VM_FAULT_HWPOISON \|
2850	VM_FAULT_SET_HINDEX(hstate_index(h));
2851	goto backout_unlocked;
2852	}
2853	}
2854
2855	/*
2856	* If we are going to COW a private mapping later, we examine the
2857	* pending reservations for this page now. This will ensure that
2858	* any allocations necessary to record that reservation occur outside
2859	* the spinlock.
2860	*/
2861	if ((flags & FAULT_FLAG_WRITE) && !(vma->vm_flags & VM_SHARED))
2862	if (vma_needs_reservation(h, vma, address) < 0) {
2863	ret = VM_FAULT_OOM;
2864	goto backout_unlocked;
2865	}
2866
2867	spin_lock(&mm->page_table_lock);
2868	size = i_size_read(mapping->host) >> huge_page_shift(h);
2869	if (idx >= size)
2870	goto backout;
2871
2872	ret = 0;
2873	if (!huge_pte_none(huge_ptep_get(ptep)))
2874	goto backout;
2875
2876	if (anon_rmap) {
2877	ClearPagePrivate(page);
2878	hugepage_add_new_anon_rmap(page, vma, address);
2879	}
2880	else
2881	page_dup_rmap(page);
2882	new_pte = make_huge_pte(vma, page, ((vma->vm_flags & VM_WRITE)
2883	&& (vma->vm_flags & VM_SHARED)));
2884	set_huge_pte_at(mm, address, ptep, new_pte);
2885
2886	if ((flags & FAULT_FLAG_WRITE) && !(vma->vm_flags & VM_SHARED)) {
2887	/* Optimization, do the COW without a second fault */
2888	ret = hugetlb_cow(mm, vma, address, ptep, new_pte, page);
2889	}
2890
2891	spin_unlock(&mm->page_table_lock);
2892	unlock_page(page);
2893	out:
2894	return ret;
2895
2896	backout:
2897	spin_unlock(&mm->page_table_lock);
2898	backout_unlocked:
2899	unlock_page(page);
2900	put_page(page);
2901	goto out;
2902	}
2903
2904	int hugetlb_fault(struct mm_struct mm, struct vm_area_struct vma,
2905	unsigned long address, unsigned int flags)
2906	{
2907	pte_t *ptep;
2908	pte_t entry;
2909	int ret;
2910	struct page *page = NULL;
2911	struct page *pagecache_page = NULL;
2912	static DEFINE_MUTEX(hugetlb_instantiation_mutex);
2913	struct hstate *h = hstate_vma(vma);
2914
2915	address &= huge_page_mask(h);
2916
2917	ptep = huge_pte_offset(mm, address);
2918	if (ptep) {
2919	entry = huge_ptep_get(ptep);
2920	if (unlikely(is_hugetlb_entry_migration(entry))) {
2921	migration_entry_wait_huge(mm, ptep);
2922	return 0;
2923	} else if (unlikely(is_hugetlb_entry_hwpoisoned(entry)))
2924	return VM_FAULT_HWPOISON_LARGE \|
2925	VM_FAULT_SET_HINDEX(hstate_index(h));
2926	}
2927
2928	ptep = huge_pte_alloc(mm, address, huge_page_size(h));
2929	if (!ptep)
2930	return VM_FAULT_OOM;
2931
2932	/*
2933	* Serialize hugepage allocation and instantiation, so that we don't
2934	* get spurious allocation failures if two CPUs race to instantiate
2935	* the same page in the page cache.
2936	*/
2937	mutex_lock(&hugetlb_instantiation_mutex);
2938	entry = huge_ptep_get(ptep);
2939	if (huge_pte_none(entry)) {
2940	ret = hugetlb_no_page(mm, vma, address, ptep, flags);
2941	goto out_mutex;
2942	}
2943
2944	ret = 0;
2945
2946	/*
2947	* If we are going to COW the mapping later, we examine the pending
2948	* reservations for this page now. This will ensure that any
2949	* allocations necessary to record that reservation occur outside the
2950	* spinlock. For private mappings, we also lookup the pagecache
2951	* page now as it is used to determine if a reservation has been
2952	* consumed.
2953	*/
2954	if ((flags & FAULT_FLAG_WRITE) && !huge_pte_write(entry)) {
2955	if (vma_needs_reservation(h, vma, address) < 0) {
2956	ret = VM_FAULT_OOM;
2957	goto out_mutex;
2958	}
2959
2960	if (!(vma->vm_flags & VM_MAYSHARE))
2961	pagecache_page = hugetlbfs_pagecache_page(h,
2962	vma, address);
2963	}
2964
2965	/*
2966	* hugetlb_cow() requires page locks of pte_page(entry) and
2967	* pagecache_page, so here we need take the former one
2968	* when page != pagecache_page or !pagecache_page.
2969	* Note that locking order is always pagecache_page -> page,
2970	* so no worry about deadlock.
2971	*/
2972	page = pte_page(entry);
2973	get_page(page);
2974	if (page != pagecache_page)
2975	lock_page(page);
2976
2977	spin_lock(&mm->page_table_lock);
2978	/* Check for a racing update before calling hugetlb_cow */
2979	if (unlikely(!pte_same(entry, huge_ptep_get(ptep))))
2980	goto out_page_table_lock;
2981
2982
2983	if (flags & FAULT_FLAG_WRITE) {
2984	if (!huge_pte_write(entry)) {
2985	ret = hugetlb_cow(mm, vma, address, ptep, entry,
2986	pagecache_page);
2987	goto out_page_table_lock;
2988	}
2989	entry = huge_pte_mkdirty(entry);
2990	}
2991	entry = pte_mkyoung(entry);
2992	if (huge_ptep_set_access_flags(vma, address, ptep, entry,
2993	flags & FAULT_FLAG_WRITE))
2994	update_mmu_cache(vma, address, ptep);
2995
2996	out_page_table_lock:
2997	spin_unlock(&mm->page_table_lock);
2998
2999	if (pagecache_page) {
3000	unlock_page(pagecache_page);
3001	put_page(pagecache_page);
3002	}
3003	if (page != pagecache_page)
3004	unlock_page(page);
3005	put_page(page);
3006
3007	out_mutex:
3008	mutex_unlock(&hugetlb_instantiation_mutex);
3009
3010	return ret;
3011	}
3012
3013	long follow_hugetlb_page(struct mm_struct mm, struct vm_area_struct vma,
3014	struct page pages, struct vm_area_struct vmas,
3015	unsigned long position, unsigned long nr_pages,
3016	long i, unsigned int flags)
3017	{
3018	unsigned long pfn_offset;
3019	unsigned long vaddr = *position;
3020	unsigned long remainder = *nr_pages;
3021	struct hstate *h = hstate_vma(vma);
3022
3023	spin_lock(&mm->page_table_lock);
3024	while (vaddr < vma->vm_end && remainder) {
3025	pte_t *pte;
3026	int absent;
3027	struct page *page;
3028
3029	/*
3030	* Some archs (sparc64, sh*) have multiple pte_ts to
3031	* each hugepage. We have to make sure we get the
3032	* first, for the page indexing below to work.
3033	*/
3034	pte = huge_pte_offset(mm, vaddr & huge_page_mask(h));
3035	absent = !pte \|\| huge_pte_none(huge_ptep_get(pte));
3036
3037	/*
3038	* When coredumping, it suits get_dump_page if we just return
3039	* an error where there's an empty slot with no huge pagecache
3040	* to back it. This way, we avoid allocating a hugepage, and
3041	* the sparse dumpfile avoids allocating disk blocks, but its
3042	* huge holes still show up with zeroes where they need to be.
3043	*/
3044	if (absent && (flags & FOLL_DUMP) &&
3045	!hugetlbfs_pagecache_present(h, vma, vaddr)) {
3046	remainder = 0;
3047	break;
3048	}
3049
3050	/*
3051	* We need call hugetlb_fault for both hugepages under migration
3052	* (in which case hugetlb_fault waits for the migration,) and
3053	* hwpoisoned hugepages (in which case we need to prevent the
3054	* caller from accessing to them.) In order to do this, we use
3055	* here is_swap_pte instead of is_hugetlb_entry_migration and
3056	* is_hugetlb_entry_hwpoisoned. This is because it simply covers
3057	* both cases, and because we can't follow correct pages
3058	* directly from any kind of swap entries.
3059	*/
3060	if (absent \|\| is_swap_pte(huge_ptep_get(pte)) \|\|
3061	((flags & FOLL_WRITE) &&
3062	!huge_pte_write(huge_ptep_get(pte)))) {
3063	int ret;
3064
3065	spin_unlock(&mm->page_table_lock);
3066	ret = hugetlb_fault(mm, vma, vaddr,
3067	(flags & FOLL_WRITE) ? FAULT_FLAG_WRITE : 0);
3068	spin_lock(&mm->page_table_lock);
3069	if (!(ret & VM_FAULT_ERROR))
3070	continue;
3071
3072	remainder = 0;
3073	break;
3074	}
3075
3076	pfn_offset = (vaddr & ~huge_page_mask(h)) >> PAGE_SHIFT;
3077	page = pte_page(huge_ptep_get(pte));
3078	same_page:
3079	if (pages) {
3080	pages[i] = mem_map_offset(page, pfn_offset);
3081	get_page(pages[i]);
3082	}
3083
3084	if (vmas)
3085	vmas[i] = vma;
3086
3087	vaddr += PAGE_SIZE;
3088	++pfn_offset;
3089	--remainder;
3090	++i;
3091	if (vaddr < vma->vm_end && remainder &&
3092	pfn_offset < pages_per_huge_page(h)) {
3093	/*
3094	* We use pfn_offset to avoid touching the pageframes
3095	* of this compound page.
3096	*/
3097	goto same_page;
3098	}
3099	}
3100	spin_unlock(&mm->page_table_lock);
3101	*nr_pages = remainder;
3102	*position = vaddr;
3103
3104	return i ? i : -EFAULT;
3105	}
3106
3107	unsigned long hugetlb_change_protection(struct vm_area_struct *vma,
3108	unsigned long address, unsigned long end, pgprot_t newprot)
3109	{
3110	struct mm_struct *mm = vma->vm_mm;
3111	unsigned long start = address;
3112	pte_t *ptep;
3113	pte_t pte;
3114	struct hstate *h = hstate_vma(vma);
3115	unsigned long pages = 0;
3116
3117	BUG_ON(address >= end);
3118	flush_cache_range(vma, address, end);
3119
3120	mutex_lock(&vma->vm_file->f_mapping->i_mmap_mutex);
3121	spin_lock(&mm->page_table_lock);
3122	for (; address < end; address += huge_page_size(h)) {
3123	ptep = huge_pte_offset(mm, address);
3124	if (!ptep)
3125	continue;
3126	if (huge_pmd_unshare(mm, &address, ptep)) {
3127	pages++;
3128	continue;
3129	}
3130	if (!huge_pte_none(huge_ptep_get(ptep))) {
3131	pte = huge_ptep_get_and_clear(mm, address, ptep);
3132	pte = pte_mkhuge(huge_pte_modify(pte, newprot));
3133	pte = arch_make_huge_pte(pte, vma, NULL, 0);
3134	set_huge_pte_at(mm, address, ptep, pte);
3135	pages++;
3136	}
3137	}
3138	spin_unlock(&mm->page_table_lock);
3139	/*
3140	* Must flush TLB before releasing i_mmap_mutex: x86's huge_pmd_unshare
3141	* may have cleared our pud entry and done put_page on the page table:
3142	* once we release i_mmap_mutex, another task can do the final put_page
3143	* and that page table be reused and filled with junk.
3144	*/
3145	flush_tlb_range(vma, start, end);
3146	mutex_unlock(&vma->vm_file->f_mapping->i_mmap_mutex);
3147
3148	return pages << h->order;
3149	}
3150
3151	int hugetlb_reserve_pages(struct inode *inode,
3152	long from, long to,
3153	struct vm_area_struct *vma,
3154	vm_flags_t vm_flags)
3155	{
3156	long ret, chg;
3157	struct hstate *h = hstate_inode(inode);
3158	struct hugepage_subpool *spool = subpool_inode(inode);
3159
3160	/*
3161	* Only apply hugepage reservation if asked. At fault time, an
3162	* attempt will be made for VM_NORESERVE to allocate a page
3163	* without using reserves
3164	*/
3165	if (vm_flags & VM_NORESERVE)
3166	return 0;
3167
3168	/*
3169	* Shared mappings base their reservation on the number of pages that
3170	* are already allocated on behalf of the file. Private mappings need
3171	* to reserve the full area even if read-only as mprotect() may be
3172	* called to make the mapping read-write. Assume !vma is a shm mapping
3173	*/
3174	if (!vma \|\| vma->vm_flags & VM_MAYSHARE)
3175	chg = region_chg(&inode->i_mapping->private_list, from, to);
3176	else {
3177	struct resv_map *resv_map = resv_map_alloc();
3178	if (!resv_map)
3179	return -ENOMEM;
3180
3181	chg = to - from;
3182
3183	set_vma_resv_map(vma, resv_map);
3184	set_vma_resv_flags(vma, HPAGE_RESV_OWNER);
3185	}
3186
3187	if (chg < 0) {
3188	ret = chg;
3189	goto out_err;
3190	}
3191
3192	/* There must be enough pages in the subpool for the mapping */
3193	if (hugepage_subpool_get_pages(spool, chg)) {
3194	ret = -ENOSPC;
3195	goto out_err;
3196	}
3197
3198	/*
3199	* Check enough hugepages are available for the reservation.
3200	* Hand the pages back to the subpool if there are not
3201	*/
3202	ret = hugetlb_acct_memory(h, chg);
3203	if (ret < 0) {
3204	hugepage_subpool_put_pages(spool, chg);
3205	goto out_err;
3206	}
3207
3208	/*
3209	* Account for the reservations made. Shared mappings record regions
3210	* that have reservations as they are shared by multiple VMAs.
3211	* When the last VMA disappears, the region map says how much
3212	* the reservation was and the page cache tells how much of
3213	* the reservation was consumed. Private mappings are per-VMA and
3214	* only the consumed reservations are tracked. When the VMA
3215	* disappears, the original reservation is the VMA size and the
3216	* consumed reservations are stored in the map. Hence, nothing
3217	* else has to be done for private mappings here
3218	*/
3219	if (!vma \|\| vma->vm_flags & VM_MAYSHARE)
3220	region_add(&inode->i_mapping->private_list, from, to);
3221	return 0;
3222	out_err:
3223	if (vma)
3224	resv_map_put(vma);
3225	return ret;
3226	}
3227
3228	void hugetlb_unreserve_pages(struct inode *inode, long offset, long freed)
3229	{
3230	struct hstate *h = hstate_inode(inode);
3231	long chg = region_truncate(&inode->i_mapping->private_list, offset);
3232	struct hugepage_subpool *spool = subpool_inode(inode);
3233
3234	spin_lock(&inode->i_lock);
3235	inode->i_blocks -= (blocks_per_huge_page(h) * freed);
3236	spin_unlock(&inode->i_lock);
3237
3238	hugepage_subpool_put_pages(spool, (chg - freed));
3239	hugetlb_acct_memory(h, -(chg - freed));
3240	}
3241
3242	#ifdef CONFIG_ARCH_WANT_HUGE_PMD_SHARE
3243	static unsigned long page_table_shareable(struct vm_area_struct *svma,
3244	struct vm_area_struct *vma,
3245	unsigned long addr, pgoff_t idx)
3246	{
3247	unsigned long saddr = ((idx - svma->vm_pgoff) << PAGE_SHIFT) +
3248	svma->vm_start;
3249	unsigned long sbase = saddr & PUD_MASK;
3250	unsigned long s_end = sbase + PUD_SIZE;
3251
3252	/* Allow segments to share if only one is marked locked */
3253	unsigned long vm_flags = vma->vm_flags & ~VM_LOCKED;
3254	unsigned long svm_flags = svma->vm_flags & ~VM_LOCKED;
3255
3256	/*
3257	* match the virtual addresses, permission and the alignment of the
3258	* page table page.
3259	*/
3260	if (pmd_index(addr) != pmd_index(saddr) \|\|
3261	vm_flags != svm_flags \|\|
3262	sbase < svma->vm_start \|\| svma->vm_end < s_end)
3263	return 0;
3264
3265	return saddr;
3266	}
3267
3268	static int vma_shareable(struct vm_area_struct *vma, unsigned long addr)
3269	{
3270	unsigned long base = addr & PUD_MASK;
3271	unsigned long end = base + PUD_SIZE;
3272
3273	/*
3274	* check on proper vm_flags and page table alignment
3275	*/
3276	if (vma->vm_flags & VM_MAYSHARE &&
3277	vma->vm_start <= base && end <= vma->vm_end)
3278	return 1;
3279	return 0;
3280	}
3281
3282	/*
3283	* Search for a shareable pmd page for hugetlb. In any case calls pmd_alloc()
3284	* and returns the corresponding pte. While this is not necessary for the
3285	* !shared pmd case because we can allocate the pmd later as well, it makes the
3286	* code much cleaner. pmd allocation is essential for the shared case because
3287	* pud has to be populated inside the same i_mmap_mutex section - otherwise
3288	* racing tasks could either miss the sharing (see huge_pte_offset) or select a
3289	* bad pmd for sharing.
3290	*/
3291	pte_t huge_pmd_share(struct mm_struct mm, unsigned long addr, pud_t *pud)
3292	{
3293	struct vm_area_struct *vma = find_vma(mm, addr);
3294	struct address_space *mapping = vma->vm_file->f_mapping;
3295	pgoff_t idx = ((addr - vma->vm_start) >> PAGE_SHIFT) +
3296	vma->vm_pgoff;
3297	struct vm_area_struct *svma;
3298	unsigned long saddr;
3299	pte_t *spte = NULL;
3300	pte_t *pte;
3301
3302	if (!vma_shareable(vma, addr))
3303	return (pte_t *)pmd_alloc(mm, pud, addr);
3304
3305	mutex_lock(&mapping->i_mmap_mutex);
3306	vma_interval_tree_foreach(svma, &mapping->i_mmap, idx, idx) {
3307	if (svma == vma)
3308	continue;
3309
3310	saddr = page_table_shareable(svma, vma, addr, idx);
3311	if (saddr) {
3312	spte = huge_pte_offset(svma->vm_mm, saddr);
3313	if (spte) {
3314	get_page(virt_to_page(spte));
3315	break;
3316	}
3317	}
3318	}
3319
3320	if (!spte)
3321	goto out;
3322
3323	spin_lock(&mm->page_table_lock);
3324	if (pud_none(*pud))
3325	pud_populate(mm, pud,
3326	(pmd_t *)((unsigned long)spte & PAGE_MASK));
3327	else
3328	put_page(virt_to_page(spte));
3329	spin_unlock(&mm->page_table_lock);
3330	out:
3331	pte = (pte_t *)pmd_alloc(mm, pud, addr);
3332	mutex_unlock(&mapping->i_mmap_mutex);
3333	return pte;
3334	}
3335
3336	/*
3337	* unmap huge page backed by shared pte.
3338	*
3339	* Hugetlb pte page is ref counted at the time of mapping. If pte is shared
3340	* indicated by page_count > 1, unmap is achieved by clearing pud and
3341	* decrementing the ref count. If count == 1, the pte page is not shared.
3342	*
3343	* called with vma->vm_mm->page_table_lock held.
3344	*
3345	* returns: 1 successfully unmapped a shared pte page
3346	* 0 the underlying pte page is not shared, or it is the last user
3347	*/
3348	int huge_pmd_unshare(struct mm_struct mm, unsigned long addr, pte_t *ptep)
3349	{
3350	pgd_t pgd = pgd_offset(mm, addr);
3351	pud_t pud = pud_offset(pgd, addr);
3352
3353	BUG_ON(page_count(virt_to_page(ptep)) == 0);
3354	if (page_count(virt_to_page(ptep)) == 1)
3355	return 0;
3356
3357	pud_clear(pud);
3358	put_page(virt_to_page(ptep));
3359	addr = ALIGN(addr, HPAGE_SIZE * PTRS_PER_PTE) - HPAGE_SIZE;
3360	return 1;
3361	}
3362	#define want_pmd_share() (1)
3363	#else /* !CONFIG_ARCH_WANT_HUGE_PMD_SHARE */
3364	pte_t huge_pmd_share(struct mm_struct mm, unsigned long addr, pud_t *pud)
3365	{
3366	return NULL;
3367	}
3368	#define want_pmd_share() (0)
3369	#endif /* CONFIG_ARCH_WANT_HUGE_PMD_SHARE */
3370
3371	#ifdef CONFIG_ARCH_WANT_GENERAL_HUGETLB
3372	pte_t huge_pte_alloc(struct mm_struct mm,
3373	unsigned long addr, unsigned long sz)
3374	{
3375	pgd_t *pgd;
3376	pud_t *pud;
3377	pte_t *pte = NULL;
3378
3379	pgd = pgd_offset(mm, addr);
3380	pud = pud_alloc(mm, pgd, addr);
3381	if (pud) {
3382	if (sz == PUD_SIZE) {
3383	pte = (pte_t *)pud;
3384	} else {
3385	BUG_ON(sz != PMD_SIZE);
3386	if (want_pmd_share() && pud_none(*pud))
3387	pte = huge_pmd_share(mm, addr, pud);
3388	else
3389	pte = (pte_t *)pmd_alloc(mm, pud, addr);
3390	}
3391	}
3392	BUG_ON(pte && !pte_none(pte) && !pte_huge(pte));
3393
3394	return pte;
3395	}
3396
3397	pte_t huge_pte_offset(struct mm_struct mm, unsigned long addr)
3398	{
3399	pgd_t *pgd;
3400	pud_t *pud;
3401	pmd_t *pmd = NULL;
3402
3403	pgd = pgd_offset(mm, addr);
3404	if (pgd_present(*pgd)) {
3405	pud = pud_offset(pgd, addr);
3406	if (pud_present(*pud)) {
3407	if (pud_huge(*pud))
3408	return (pte_t *)pud;
3409	pmd = pmd_offset(pud, addr);
3410	}
3411	}
3412	return (pte_t *) pmd;
3413	}
3414
3415	struct page *
3416	follow_huge_pmd(struct mm_struct *mm, unsigned long address,
3417	pmd_t *pmd, int write)
3418	{
3419	struct page *page;
3420
3421	page = pte_page((pte_t )pmd);
3422	if (page)
3423	page += ((address & ~PMD_MASK) >> PAGE_SHIFT);
3424	return page;
3425	}
3426
3427	struct page *
3428	follow_huge_pud(struct mm_struct *mm, unsigned long address,
3429	pud_t *pud, int write)
3430	{
3431	struct page *page;
3432
3433	page = pte_page((pte_t )pud);
3434	if (page)
3435	page += ((address & ~PUD_MASK) >> PAGE_SHIFT);
3436	return page;
3437	}
3438
3439	#else /* !CONFIG_ARCH_WANT_GENERAL_HUGETLB */
3440
3441	/* Can be overriden by architectures */
3442	__attribute__((weak)) struct page *
3443	follow_huge_pud(struct mm_struct *mm, unsigned long address,
3444	pud_t *pud, int write)
3445	{
3446	BUG();
3447	return NULL;
3448	}
3449
3450	#endif /* CONFIG_ARCH_WANT_GENERAL_HUGETLB */
3451
3452	#ifdef CONFIG_MEMORY_FAILURE
3453
3454	/* Should be called in hugetlb_lock */
3455	static int is_hugepage_on_freelist(struct page *hpage)
3456	{
3457	struct page *page;
3458	struct page *tmp;
3459	struct hstate *h = page_hstate(hpage);
3460	int nid = page_to_nid(hpage);
3461
3462	list_for_each_entry_safe(page, tmp, &h->hugepage_freelists[nid], lru)
3463	if (page == hpage)
3464	return 1;
3465	return 0;
3466	}
3467
3468	/*
3469	* This function is called from memory failure code.
3470	* Assume the caller holds page lock of the head page.
3471	*/
3472	int dequeue_hwpoisoned_huge_page(struct page *hpage)
3473	{
3474	struct hstate *h = page_hstate(hpage);
3475	int nid = page_to_nid(hpage);
3476	int ret = -EBUSY;
3477
3478	spin_lock(&hugetlb_lock);
3479	if (is_hugepage_on_freelist(hpage)) {
3480	/*
3481	* Hwpoisoned hugepage isn't linked to activelist or freelist,
3482	* but dangling hpage->lru can trigger list-debug warnings
3483	* (this happens when we call unpoison_memory() on it),
3484	* so let it point to itself with list_del_init().
3485	*/
3486	list_del_init(&hpage->lru);
3487	set_page_refcounted(hpage);
3488	h->free_huge_pages--;
3489	h->free_huge_pages_node[nid]--;
3490	ret = 0;
3491	}
3492	spin_unlock(&hugetlb_lock);
3493	return ret;
3494	}
3495	#endif
3496
3497	bool isolate_huge_page(struct page page, struct list_head list)
3498	{
3499	VM_BUG_ON(!PageHead(page));
3500	if (!get_page_unless_zero(page))
3501	return false;
3502	spin_lock(&hugetlb_lock);
3503	list_move_tail(&page->lru, list);
3504	spin_unlock(&hugetlb_lock);
3505	return true;
3506	}
3507
3508	void putback_active_hugepage(struct page *page)
3509	{
3510	VM_BUG_ON(!PageHead(page));
3511	spin_lock(&hugetlb_lock);
3512	list_move_tail(&page->lru, &(page_hstate(page))->hugepage_activelist);
3513	spin_unlock(&hugetlb_lock);
3514	put_page(page);
3515	}
3516
3517	bool is_hugepage_active(struct page *page)
3518	{
3519	VM_BUG_ON(!PageHuge(page));
3520	/*
3521	* This function can be called for a tail page because the caller,
3522	* scan_movable_pages, scans through a given pfn-range which typically
3523	* covers one memory block. In systems using gigantic hugepage (1GB
3524	* for x86_64,) a hugepage is larger than a memory block, and we don't
3525	* support migrating such large hugepages for now, so return false
3526	* when called for tail pages.
3527	*/
3528	if (PageTail(page))
3529	return false;
3530	/*
3531	* Refcount of a hwpoisoned hugepages is 1, but they are not active,
3532	* so we should return false for them.
3533	*/
3534	if (unlikely(PageHWPoison(page)))
3535	return false;
3536	return page_count(page) > 0;
3537	}
3538

Download this file

Branches:
ben-wpan
ben-wpan-stefan
javiroman/ks7010
jz-2.6.34
jz-2.6.34-rc5
jz-2.6.34-rc6
jz-2.6.34-rc7
jz-2.6.35
jz-2.6.36
jz-2.6.37
jz-2.6.38
jz-2.6.39
jz-3.0
jz-3.1
jz-3.11
jz-3.12
jz-3.13
jz-3.15
jz-3.16
jz-3.18-dt
jz-3.2
jz-3.3
jz-3.4
jz-3.5
jz-3.6
jz-3.6-rc2-pwm
jz-3.9
jz-3.9-clk
jz-3.9-rc8
jz47xx
jz47xx-2.6.38
master

Tags:
od-2011-09-04
od-2011-09-18
v2.6.34-rc5
v2.6.34-rc6
v2.6.34-rc7
v3.9

Kernel

Kernel Git Source Tree

Root/mm/hugetlb.c

interactive