path: root/mm/hugetlb.c (plain)
blob: 75d8bd7e8798b5924b5bdce2002890e4525ffe66

1	/*
2	* Generic hugetlb support.
3	* (C) Nadia Yvette Chambers, April 2004
4	*/
5	#include <linux/list.h>
6	#include <linux/init.h>
7	#include <linux/mm.h>
8	#include <linux/seq_file.h>
9	#include <linux/sysctl.h>
10	#include <linux/highmem.h>
11	#include <linux/mmu_notifier.h>
12	#include <linux/nodemask.h>
13	#include <linux/pagemap.h>
14	#include <linux/mempolicy.h>
15	#include <linux/compiler.h>
16	#include <linux/cpuset.h>
17	#include <linux/mutex.h>
18	#include <linux/bootmem.h>
19	#include <linux/sysfs.h>
20	#include <linux/slab.h>
21	#include <linux/rmap.h>
22	#include <linux/swap.h>
23	#include <linux/swapops.h>
24	#include <linux/page-isolation.h>
25	#include <linux/jhash.h>
26
27	#include <asm/page.h>
28	#include <asm/pgtable.h>
29	#include <asm/tlb.h>
30
31	#include <linux/io.h>
32	#include <linux/hugetlb.h>
33	#include <linux/hugetlb_cgroup.h>
34	#include <linux/node.h>
35	#include "internal.h"
36
37	int hugepages_treat_as_movable;
38
39	int hugetlb_max_hstate __read_mostly;
40	unsigned int default_hstate_idx;
41	struct hstate hstates[HUGE_MAX_HSTATE];
42	/*
43	* Minimum page order among possible hugepage sizes, set to a proper value
44	* at boot time.
45	*/
46	static unsigned int minimum_order __read_mostly = UINT_MAX;
47
48	__initdata LIST_HEAD(huge_boot_pages);
49
50	/* for command line parsing */
51	static struct hstate * __initdata parsed_hstate;
52	static unsigned long __initdata default_hstate_max_huge_pages;
53	static unsigned long __initdata default_hstate_size;
54	static bool __initdata parsed_valid_hugepagesz = true;
55
56	/*
57	* Protects updates to hugepage_freelists, hugepage_activelist, nr_huge_pages,
58	* free_huge_pages, and surplus_huge_pages.
59	*/
60	DEFINE_SPINLOCK(hugetlb_lock);
61
62	/*
63	* Serializes faults on the same logical page. This is used to
64	* prevent spurious OOMs when the hugepage pool is fully utilized.
65	*/
66	static int num_fault_mutexes;
67	struct mutex *hugetlb_fault_mutex_table ____cacheline_aligned_in_smp;
68
69	/* Forward declaration */
70	static int hugetlb_acct_memory(struct hstate *h, long delta);
71
72	static inline void unlock_or_release_subpool(struct hugepage_subpool *spool)
73	{
74	bool free = (spool->count == 0) && (spool->used_hpages == 0);
75
76	spin_unlock(&spool->lock);
77
78	/* If no pages are used, and no other handles to the subpool
79	* remain, give up any reservations mased on minimum size and
80	* free the subpool */
81	if (free) {
82	if (spool->min_hpages != -1)
83	hugetlb_acct_memory(spool->hstate,
84	-spool->min_hpages);
85	kfree(spool);
86	}
87	}
88
89	struct hugepage_subpool *hugepage_new_subpool(struct hstate *h, long max_hpages,
90	long min_hpages)
91	{
92	struct hugepage_subpool *spool;
93
94	spool = kzalloc(sizeof(*spool), GFP_KERNEL);
95	if (!spool)
96	return NULL;
97
98	spin_lock_init(&spool->lock);
99	spool->count = 1;
100	spool->max_hpages = max_hpages;
101	spool->hstate = h;
102	spool->min_hpages = min_hpages;
103
104	if (min_hpages != -1 && hugetlb_acct_memory(h, min_hpages)) {
105	kfree(spool);
106	return NULL;
107	}
108	spool->rsv_hpages = min_hpages;
109
110	return spool;
111	}
112
113	void hugepage_put_subpool(struct hugepage_subpool *spool)
114	{
115	spin_lock(&spool->lock);
116	BUG_ON(!spool->count);
117	spool->count--;
118	unlock_or_release_subpool(spool);
119	}
120
121	/*
122	* Subpool accounting for allocating and reserving pages.
123	* Return -ENOMEM if there are not enough resources to satisfy the
124	* the request. Otherwise, return the number of pages by which the
125	* global pools must be adjusted (upward). The returned value may
126	* only be different than the passed value (delta) in the case where
127	* a subpool minimum size must be manitained.
128	*/
129	static long hugepage_subpool_get_pages(struct hugepage_subpool *spool,
130	long delta)
131	{
132	long ret = delta;
133
134	if (!spool)
135	return ret;
136
137	spin_lock(&spool->lock);
138
139	if (spool->max_hpages != -1) { /* maximum size accounting */
140	if ((spool->used_hpages + delta) <= spool->max_hpages)
141	spool->used_hpages += delta;
142	else {
143	ret = -ENOMEM;
144	goto unlock_ret;
145	}
146	}
147
148	/* minimum size accounting */
149	if (spool->min_hpages != -1 && spool->rsv_hpages) {
150	if (delta > spool->rsv_hpages) {
151	/*
152	* Asking for more reserves than those already taken on
153	* behalf of subpool. Return difference.
154	*/
155	ret = delta - spool->rsv_hpages;
156	spool->rsv_hpages = 0;
157	} else {
158	ret = 0; /* reserves already accounted for */
159	spool->rsv_hpages -= delta;
160	}
161	}
162
163	unlock_ret:
164	spin_unlock(&spool->lock);
165	return ret;
166	}
167
168	/*
169	* Subpool accounting for freeing and unreserving pages.
170	* Return the number of global page reservations that must be dropped.
171	* The return value may only be different than the passed value (delta)
172	* in the case where a subpool minimum size must be maintained.
173	*/
174	static long hugepage_subpool_put_pages(struct hugepage_subpool *spool,
175	long delta)
176	{
177	long ret = delta;
178
179	if (!spool)
180	return delta;
181
182	spin_lock(&spool->lock);
183
184	if (spool->max_hpages != -1) /* maximum size accounting */
185	spool->used_hpages -= delta;
186
187	/* minimum size accounting */
188	if (spool->min_hpages != -1 && spool->used_hpages < spool->min_hpages) {
189	if (spool->rsv_hpages + delta <= spool->min_hpages)
190	ret = 0;
191	else
192	ret = spool->rsv_hpages + delta - spool->min_hpages;
193
194	spool->rsv_hpages += delta;
195	if (spool->rsv_hpages > spool->min_hpages)
196	spool->rsv_hpages = spool->min_hpages;
197	}
198
199	/*
200	* If hugetlbfs_put_super couldn't free spool due to an outstanding
201	* quota reference, free it now.
202	*/
203	unlock_or_release_subpool(spool);
204
205	return ret;
206	}
207
208	static inline struct hugepage_subpool *subpool_inode(struct inode *inode)
209	{
210	return HUGETLBFS_SB(inode->i_sb)->spool;
211	}
212
213	static inline struct hugepage_subpool *subpool_vma(struct vm_area_struct *vma)
214	{
215	return subpool_inode(file_inode(vma->vm_file));
216	}
217
218	/*
219	* Region tracking -- allows tracking of reservations and instantiated pages
220	* across the pages in a mapping.
221	*
222	* The region data structures are embedded into a resv_map and protected
223	* by a resv_map's lock. The set of regions within the resv_map represent
224	* reservations for huge pages, or huge pages that have already been
225	* instantiated within the map. The from and to elements are huge page
226	* indicies into the associated mapping. from indicates the starting index
227	* of the region. to represents the first index past the end of the region.
228	*
229	* For example, a file region structure with from == 0 and to == 4 represents
230	* four huge pages in a mapping. It is important to note that the to element
231	* represents the first element past the end of the region. This is used in
232	* arithmetic as 4(to) - 0(from) = 4 huge pages in the region.
233	*
234	* Interval notation of the form [from, to) will be used to indicate that
235	* the endpoint from is inclusive and to is exclusive.
236	*/
237	struct file_region {
238	struct list_head link;
239	long from;
240	long to;
241	};
242
243	/*
244	* Add the huge page range represented by [f, t) to the reserve
245	* map. In the normal case, existing regions will be expanded
246	* to accommodate the specified range. Sufficient regions should
247	* exist for expansion due to the previous call to region_chg
248	* with the same range. However, it is possible that region_del
249	* could have been called after region_chg and modifed the map
250	* in such a way that no region exists to be expanded. In this
251	* case, pull a region descriptor from the cache associated with
252	* the map and use that for the new range.
253	*
254	* Return the number of new huge pages added to the map. This
255	* number is greater than or equal to zero.
256	*/
257	static long region_add(struct resv_map *resv, long f, long t)
258	{
259	struct list_head *head = &resv->regions;
260	struct file_region *rg, *nrg, *trg;
261	long add = 0;
262
263	spin_lock(&resv->lock);
264	/* Locate the region we are either in or before. */
265	list_for_each_entry(rg, head, link)
266	if (f <= rg->to)
267	break;
268
269	/*
270	* If no region exists which can be expanded to include the
271	* specified range, the list must have been modified by an
272	* interleving call to region_del(). Pull a region descriptor
273	* from the cache and use it for this range.
274	*/
275	if (&rg->link == head || t < rg->from) {
276	VM_BUG_ON(resv->region_cache_count <= 0);
277
278	resv->region_cache_count--;
279	nrg = list_first_entry(&resv->region_cache, struct file_region,
280	link);
281	list_del(&nrg->link);
282
283	nrg->from = f;
284	nrg->to = t;
285	list_add(&nrg->link, rg->link.prev);
286
287	add += t - f;
288	goto out_locked;
289	}
290
291	/* Round our left edge to the current segment if it encloses us. */
292	if (f > rg->from)
293	f = rg->from;
294
295	/* Check for and consume any regions we now overlap with. */
296	nrg = rg;
297	list_for_each_entry_safe(rg, trg, rg->link.prev, link) {
298	if (&rg->link == head)
299	break;
300	if (rg->from > t)
301	break;
302
303	/* If this area reaches higher then extend our area to
304	* include it completely. If this is not the first area
305	* which we intend to reuse, free it. */
306	if (rg->to > t)
307	t = rg->to;
308	if (rg != nrg) {
309	/* Decrement return value by the deleted range.
310	* Another range will span this area so that by
311	* end of routine add will be >= zero
312	*/
313	add -= (rg->to - rg->from);
314	list_del(&rg->link);
315	kfree(rg);
316	}
317	}
318
319	add += (nrg->from - f); /* Added to beginning of region */
320	nrg->from = f;
321	add += t - nrg->to; /* Added to end of region */
322	nrg->to = t;
323
324	out_locked:
325	resv->adds_in_progress--;
326	spin_unlock(&resv->lock);
327	VM_BUG_ON(add < 0);
328	return add;
329	}
330
331	/*
332	* Examine the existing reserve map and determine how many
333	* huge pages in the specified range [f, t) are NOT currently
334	* represented. This routine is called before a subsequent
335	* call to region_add that will actually modify the reserve
336	* map to add the specified range [f, t). region_chg does
337	* not change the number of huge pages represented by the
338	* map. However, if the existing regions in the map can not
339	* be expanded to represent the new range, a new file_region
340	* structure is added to the map as a placeholder. This is
341	* so that the subsequent region_add call will have all the
342	* regions it needs and will not fail.
343	*
344	* Upon entry, region_chg will also examine the cache of region descriptors
345	* associated with the map. If there are not enough descriptors cached, one
346	* will be allocated for the in progress add operation.
347	*
348	* Returns the number of huge pages that need to be added to the existing
349	* reservation map for the range [f, t). This number is greater or equal to
350	* zero. -ENOMEM is returned if a new file_region structure or cache entry
351	* is needed and can not be allocated.
352	*/
353	static long region_chg(struct resv_map *resv, long f, long t)
354	{
355	struct list_head *head = &resv->regions;
356	struct file_region *rg, *nrg = NULL;
357	long chg = 0;
358
359	retry:
360	spin_lock(&resv->lock);
361	retry_locked:
362	resv->adds_in_progress++;
363
364	/*
365	* Check for sufficient descriptors in the cache to accommodate
366	* the number of in progress add operations.
367	*/
368	if (resv->adds_in_progress > resv->region_cache_count) {
369	struct file_region *trg;
370
371	VM_BUG_ON(resv->adds_in_progress - resv->region_cache_count > 1);
372	/* Must drop lock to allocate a new descriptor. */
373	resv->adds_in_progress--;
374	spin_unlock(&resv->lock);
375
376	trg = kmalloc(sizeof(*trg), GFP_KERNEL);
377	if (!trg) {
378	kfree(nrg);
379	return -ENOMEM;
380	}
381
382	spin_lock(&resv->lock);
383	list_add(&trg->link, &resv->region_cache);
384	resv->region_cache_count++;
385	goto retry_locked;
386	}
387
388	/* Locate the region we are before or in. */
389	list_for_each_entry(rg, head, link)
390	if (f <= rg->to)
391	break;
392
393	/* If we are below the current region then a new region is required.
394	* Subtle, allocate a new region at the position but make it zero
395	* size such that we can guarantee to record the reservation. */
396	if (&rg->link == head || t < rg->from) {
397	if (!nrg) {
398	resv->adds_in_progress--;
399	spin_unlock(&resv->lock);
400	nrg = kmalloc(sizeof(*nrg), GFP_KERNEL);
401	if (!nrg)
402	return -ENOMEM;
403
404	nrg->from = f;
405	nrg->to = f;
406	INIT_LIST_HEAD(&nrg->link);
407	goto retry;
408	}
409
410	list_add(&nrg->link, rg->link.prev);
411	chg = t - f;
412	goto out_nrg;
413	}
414
415	/* Round our left edge to the current segment if it encloses us. */
416	if (f > rg->from)
417	f = rg->from;
418	chg = t - f;
419
420	/* Check for and consume any regions we now overlap with. */
421	list_for_each_entry(rg, rg->link.prev, link) {
422	if (&rg->link == head)
423	break;
424	if (rg->from > t)
425	goto out;
426
427	/* We overlap with this area, if it extends further than
428	* us then we must extend ourselves. Account for its
429	* existing reservation. */
430	if (rg->to > t) {
431	chg += rg->to - t;
432	t = rg->to;
433	}
434	chg -= rg->to - rg->from;
435	}
436
437	out:
438	spin_unlock(&resv->lock);
439	/* We already know we raced and no longer need the new region */
440	kfree(nrg);
441	return chg;
442	out_nrg:
443	spin_unlock(&resv->lock);
444	return chg;
445	}
446
447	/*
448	* Abort the in progress add operation. The adds_in_progress field
449	* of the resv_map keeps track of the operations in progress between
450	* calls to region_chg and region_add. Operations are sometimes
451	* aborted after the call to region_chg. In such cases, region_abort
452	* is called to decrement the adds_in_progress counter.
453	*
454	* NOTE: The range arguments [f, t) are not needed or used in this
455	* routine. They are kept to make reading the calling code easier as
456	* arguments will match the associated region_chg call.
457	*/
458	static void region_abort(struct resv_map *resv, long f, long t)
459	{
460	spin_lock(&resv->lock);
461	VM_BUG_ON(!resv->region_cache_count);
462	resv->adds_in_progress--;
463	spin_unlock(&resv->lock);
464	}
465
466	/*
467	* Delete the specified range [f, t) from the reserve map. If the
468	* t parameter is LONG_MAX, this indicates that ALL regions after f
469	* should be deleted. Locate the regions which intersect [f, t)
470	* and either trim, delete or split the existing regions.
471	*
472	* Returns the number of huge pages deleted from the reserve map.
473	* In the normal case, the return value is zero or more. In the
474	* case where a region must be split, a new region descriptor must
475	* be allocated. If the allocation fails, -ENOMEM will be returned.
476	* NOTE: If the parameter t == LONG_MAX, then we will never split
477	* a region and possibly return -ENOMEM. Callers specifying
478	* t == LONG_MAX do not need to check for -ENOMEM error.
479	*/
480	static long region_del(struct resv_map *resv, long f, long t)
481	{
482	struct list_head *head = &resv->regions;
483	struct file_region *rg, *trg;
484	struct file_region *nrg = NULL;
485	long del = 0;
486
487	retry:
488	spin_lock(&resv->lock);
489	list_for_each_entry_safe(rg, trg, head, link) {
490	/*
491	* Skip regions before the range to be deleted. file_region
492	* ranges are normally of the form [from, to). However, there
493	* may be a "placeholder" entry in the map which is of the form
494	* (from, to) with from == to. Check for placeholder entries
495	* at the beginning of the range to be deleted.
496	*/
497	if (rg->to <= f && (rg->to != rg->from || rg->to != f))
498	continue;
499
500	if (rg->from >= t)
501	break;
502
503	if (f > rg->from && t < rg->to) { /* Must split region */
504	/*
505	* Check for an entry in the cache before dropping
506	* lock and attempting allocation.
507	*/
508	if (!nrg &&
509	resv->region_cache_count > resv->adds_in_progress) {
510	nrg = list_first_entry(&resv->region_cache,
511	struct file_region,
512	link);
513	list_del(&nrg->link);
514	resv->region_cache_count--;
515	}
516
517	if (!nrg) {
518	spin_unlock(&resv->lock);
519	nrg = kmalloc(sizeof(*nrg), GFP_KERNEL);
520	if (!nrg)
521	return -ENOMEM;
522	goto retry;
523	}
524
525	del += t - f;
526
527	/* New entry for end of split region */
528	nrg->from = t;
529	nrg->to = rg->to;
530	INIT_LIST_HEAD(&nrg->link);
531
532	/* Original entry is trimmed */
533	rg->to = f;
534
535	list_add(&nrg->link, &rg->link);
536	nrg = NULL;
537	break;
538	}
539
540	if (f <= rg->from && t >= rg->to) { /* Remove entire region */
541	del += rg->to - rg->from;
542	list_del(&rg->link);
543	kfree(rg);
544	continue;
545	}
546
547	if (f <= rg->from) { /* Trim beginning of region */
548	del += t - rg->from;
549	rg->from = t;
550	} else { /* Trim end of region */
551	del += rg->to - f;
552	rg->to = f;
553	}
554	}
555
556	spin_unlock(&resv->lock);
557	kfree(nrg);
558	return del;
559	}
560
561	/*
562	* A rare out of memory error was encountered which prevented removal of
563	* the reserve map region for a page. The huge page itself was free'ed
564	* and removed from the page cache. This routine will adjust the subpool
565	* usage count, and the global reserve count if needed. By incrementing
566	* these counts, the reserve map entry which could not be deleted will
567	* appear as a "reserved" entry instead of simply dangling with incorrect
568	* counts.
569	*/
570	void hugetlb_fix_reserve_counts(struct inode *inode)
571	{
572	struct hugepage_subpool *spool = subpool_inode(inode);
573	long rsv_adjust;
574
575	rsv_adjust = hugepage_subpool_get_pages(spool, 1);
576	if (rsv_adjust) {
577	struct hstate *h = hstate_inode(inode);
578
579	hugetlb_acct_memory(h, 1);
580	}
581	}
582
583	/*
584	* Count and return the number of huge pages in the reserve map
585	* that intersect with the range [f, t).
586	*/
587	static long region_count(struct resv_map *resv, long f, long t)
588	{
589	struct list_head *head = &resv->regions;
590	struct file_region *rg;
591	long chg = 0;
592
593	spin_lock(&resv->lock);
594	/* Locate each segment we overlap with, and count that overlap. */
595	list_for_each_entry(rg, head, link) {
596	long seg_from;
597	long seg_to;
598
599	if (rg->to <= f)
600	continue;
601	if (rg->from >= t)
602	break;
603
604	seg_from = max(rg->from, f);
605	seg_to = min(rg->to, t);
606
607	chg += seg_to - seg_from;
608	}
609	spin_unlock(&resv->lock);
610
611	return chg;
612	}
613
614	/*
615	* Convert the address within this vma to the page offset within
616	* the mapping, in pagecache page units; huge pages here.
617	*/
618	static pgoff_t vma_hugecache_offset(struct hstate *h,
619	struct vm_area_struct *vma, unsigned long address)
620	{
621	return ((address - vma->vm_start) >> huge_page_shift(h)) +
622	(vma->vm_pgoff >> huge_page_order(h));
623	}
624
625	pgoff_t linear_hugepage_index(struct vm_area_struct *vma,
626	unsigned long address)
627	{
628	return vma_hugecache_offset(hstate_vma(vma), vma, address);
629	}
630	EXPORT_SYMBOL_GPL(linear_hugepage_index);
631
632	/*
633	* Return the size of the pages allocated when backing a VMA. In the majority
634	* cases this will be same size as used by the page table entries.
635	*/
636	unsigned long vma_kernel_pagesize(struct vm_area_struct *vma)
637	{
638	struct hstate *hstate;
639
640	if (!is_vm_hugetlb_page(vma))
641	return PAGE_SIZE;
642
643	hstate = hstate_vma(vma);
644
645	return 1UL << huge_page_shift(hstate);
646	}
647	EXPORT_SYMBOL_GPL(vma_kernel_pagesize);
648
649	/*
650	* Return the page size being used by the MMU to back a VMA. In the majority
651	* of cases, the page size used by the kernel matches the MMU size. On
652	* architectures where it differs, an architecture-specific version of this
653	* function is required.
654	*/
655	#ifndef vma_mmu_pagesize
656	unsigned long vma_mmu_pagesize(struct vm_area_struct *vma)
657	{
658	return vma_kernel_pagesize(vma);
659	}
660	#endif
661
662	/*
663	* Flags for MAP_PRIVATE reservations. These are stored in the bottom
664	* bits of the reservation map pointer, which are always clear due to
665	* alignment.
666	*/
667	#define HPAGE_RESV_OWNER (1UL << 0)
668	#define HPAGE_RESV_UNMAPPED (1UL << 1)
669	#define HPAGE_RESV_MASK (HPAGE_RESV_OWNER | HPAGE_RESV_UNMAPPED)
670
671	/*
672	* These helpers are used to track how many pages are reserved for
673	* faults in a MAP_PRIVATE mapping. Only the process that called mmap()
674	* is guaranteed to have their future faults succeed.
675	*
676	* With the exception of reset_vma_resv_huge_pages() which is called at fork(),
677	* the reserve counters are updated with the hugetlb_lock held. It is safe
678	* to reset the VMA at fork() time as it is not in use yet and there is no
679	* chance of the global counters getting corrupted as a result of the values.
680	*
681	* The private mapping reservation is represented in a subtly different
682	* manner to a shared mapping. A shared mapping has a region map associated
683	* with the underlying file, this region map represents the backing file
684	* pages which have ever had a reservation assigned which this persists even
685	* after the page is instantiated. A private mapping has a region map
686	* associated with the original mmap which is attached to all VMAs which
687	* reference it, this region map represents those offsets which have consumed
688	* reservation ie. where pages have been instantiated.
689	*/
690	static unsigned long get_vma_private_data(struct vm_area_struct *vma)
691	{
692	return (unsigned long)vma->vm_private_data;
693	}
694
695	static void set_vma_private_data(struct vm_area_struct *vma,
696	unsigned long value)
697	{
698	vma->vm_private_data = (void *)value;
699	}
700
701	struct resv_map *resv_map_alloc(void)
702	{
703	struct resv_map *resv_map = kmalloc(sizeof(*resv_map), GFP_KERNEL);
704	struct file_region *rg = kmalloc(sizeof(*rg), GFP_KERNEL);
705
706	if (!resv_map || !rg) {
707	kfree(resv_map);
708	kfree(rg);
709	return NULL;
710	}
711
712	kref_init(&resv_map->refs);
713	spin_lock_init(&resv_map->lock);
714	INIT_LIST_HEAD(&resv_map->regions);
715
716	resv_map->adds_in_progress = 0;
717
718	INIT_LIST_HEAD(&resv_map->region_cache);
719	list_add(&rg->link, &resv_map->region_cache);
720	resv_map->region_cache_count = 1;
721
722	return resv_map;
723	}
724
725	void resv_map_release(struct kref *ref)
726	{
727	struct resv_map *resv_map = container_of(ref, struct resv_map, refs);
728	struct list_head *head = &resv_map->region_cache;
729	struct file_region *rg, *trg;
730
731	/* Clear out any active regions before we release the map. */
732	region_del(resv_map, 0, LONG_MAX);
733
734	/* ... and any entries left in the cache */
735	list_for_each_entry_safe(rg, trg, head, link) {
736	list_del(&rg->link);
737	kfree(rg);
738	}
739
740	VM_BUG_ON(resv_map->adds_in_progress);
741
742	kfree(resv_map);
743	}
744
745	static inline struct resv_map *inode_resv_map(struct inode *inode)
746	{
747	return inode->i_mapping->private_data;
748	}
749
750	static struct resv_map *vma_resv_map(struct vm_area_struct *vma)
751	{
752	VM_BUG_ON_VMA(!is_vm_hugetlb_page(vma), vma);
753	if (vma->vm_flags & VM_MAYSHARE) {
754	struct address_space *mapping = vma->vm_file->f_mapping;
755	struct inode *inode = mapping->host;
756
757	return inode_resv_map(inode);
758
759	} else {
760	return (struct resv_map *)(get_vma_private_data(vma) &
761	~HPAGE_RESV_MASK);
762	}
763	}
764
765	static void set_vma_resv_map(struct vm_area_struct *vma, struct resv_map *map)
766	{
767	VM_BUG_ON_VMA(!is_vm_hugetlb_page(vma), vma);
768	VM_BUG_ON_VMA(vma->vm_flags & VM_MAYSHARE, vma);
769
770	set_vma_private_data(vma, (get_vma_private_data(vma) &
771	HPAGE_RESV_MASK) | (unsigned long)map);
772	}
773
774	static void set_vma_resv_flags(struct vm_area_struct *vma, unsigned long flags)
775	{
776	VM_BUG_ON_VMA(!is_vm_hugetlb_page(vma), vma);
777	VM_BUG_ON_VMA(vma->vm_flags & VM_MAYSHARE, vma);
778
779	set_vma_private_data(vma, get_vma_private_data(vma) | flags);
780	}
781
782	static int is_vma_resv_set(struct vm_area_struct *vma, unsigned long flag)
783	{
784	VM_BUG_ON_VMA(!is_vm_hugetlb_page(vma), vma);
785
786	return (get_vma_private_data(vma) & flag) != 0;
787	}
788
789	/* Reset counters to 0 and clear all HPAGE_RESV_* flags */
790	void reset_vma_resv_huge_pages(struct vm_area_struct *vma)
791	{
792	VM_BUG_ON_VMA(!is_vm_hugetlb_page(vma), vma);
793	if (!(vma->vm_flags & VM_MAYSHARE))
794	vma->vm_private_data = (void *)0;
795	}
796
797	/* Returns true if the VMA has associated reserve pages */
798	static bool vma_has_reserves(struct vm_area_struct *vma, long chg)
799	{
800	if (vma->vm_flags & VM_NORESERVE) {
801	/*
802	* This address is already reserved by other process(chg == 0),
803	* so, we should decrement reserved count. Without decrementing,
804	* reserve count remains after releasing inode, because this
805	* allocated page will go into page cache and is regarded as
806	* coming from reserved pool in releasing step. Currently, we
807	* don't have any other solution to deal with this situation
808	* properly, so add work-around here.
809	*/
810	if (vma->vm_flags & VM_MAYSHARE && chg == 0)
811	return true;
812	else
813	return false;
814	}
815
816	/* Shared mappings always use reserves */
817	if (vma->vm_flags & VM_MAYSHARE) {
818	/*
819	* We know VM_NORESERVE is not set. Therefore, there SHOULD
820	* be a region map for all pages. The only situation where
821	* there is no region map is if a hole was punched via
822	* fallocate. In this case, there really are no reverves to
823	* use. This situation is indicated if chg != 0.
824	*/
825	if (chg)
826	return false;
827	else
828	return true;
829	}
830
831	/*
832	* Only the process that called mmap() has reserves for
833	* private mappings.
834	*/
835	if (is_vma_resv_set(vma, HPAGE_RESV_OWNER)) {
836	/*
837	* Like the shared case above, a hole punch or truncate
838	* could have been performed on the private mapping.
839	* Examine the value of chg to determine if reserves
840	* actually exist or were previously consumed.
841	* Very Subtle - The value of chg comes from a previous
842	* call to vma_needs_reserves(). The reserve map for
843	* private mappings has different (opposite) semantics
844	* than that of shared mappings. vma_needs_reserves()
845	* has already taken this difference in semantics into
846	* account. Therefore, the meaning of chg is the same
847	* as in the shared case above. Code could easily be
848	* combined, but keeping it separate draws attention to
849	* subtle differences.
850	*/
851	if (chg)
852	return false;
853	else
854	return true;
855	}
856
857	return false;
858	}
859
860	static void enqueue_huge_page(struct hstate *h, struct page *page)
861	{
862	int nid = page_to_nid(page);
863	list_move(&page->lru, &h->hugepage_freelists[nid]);
864	h->free_huge_pages++;
865	h->free_huge_pages_node[nid]++;
866	}
867
868	static struct page *dequeue_huge_page_node(struct hstate *h, int nid)
869	{
870	struct page *page;
871
872	list_for_each_entry(page, &h->hugepage_freelists[nid], lru)
873	if (!is_migrate_isolate_page(page))
874	break;
875	/*
876	* if 'non-isolated free hugepage' not found on the list,
877	* the allocation fails.
878	*/
879	if (&h->hugepage_freelists[nid] == &page->lru)
880	return NULL;
881	list_move(&page->lru, &h->hugepage_activelist);
882	set_page_refcounted(page);
883	h->free_huge_pages--;
884	h->free_huge_pages_node[nid]--;
885	return page;
886	}
887
888	/* Movability of hugepages depends on migration support. */
889	static inline gfp_t htlb_alloc_mask(struct hstate *h)
890	{
891	if (hugepages_treat_as_movable || hugepage_migration_supported(h))
892	return GFP_HIGHUSER_MOVABLE;
893	else
894	return GFP_HIGHUSER;
895	}
896
897	static struct page *dequeue_huge_page_vma(struct hstate *h,
898	struct vm_area_struct *vma,
899	unsigned long address, int avoid_reserve,
900	long chg)
901	{
902	struct page *page = NULL;
903	struct mempolicy *mpol;
904	nodemask_t *nodemask;
905	struct zonelist *zonelist;
906	struct zone *zone;
907	struct zoneref *z;
908	unsigned int cpuset_mems_cookie;
909
910	/*
911	* A child process with MAP_PRIVATE mappings created by their parent
912	* have no page reserves. This check ensures that reservations are
913	* not "stolen". The child may still get SIGKILLed
914	*/
915	if (!vma_has_reserves(vma, chg) &&
916	h->free_huge_pages - h->resv_huge_pages == 0)
917	goto err;
918
919	/* If reserves cannot be used, ensure enough pages are in the pool */
920	if (avoid_reserve && h->free_huge_pages - h->resv_huge_pages == 0)
921	goto err;
922
923	retry_cpuset:
924	cpuset_mems_cookie = read_mems_allowed_begin();
925	zonelist = huge_zonelist(vma, address,
926	htlb_alloc_mask(h), &mpol, &nodemask);
927
928	for_each_zone_zonelist_nodemask(zone, z, zonelist,
929	MAX_NR_ZONES - 1, nodemask) {
930	if (cpuset_zone_allowed(zone, htlb_alloc_mask(h))) {
931	page = dequeue_huge_page_node(h, zone_to_nid(zone));
932	if (page) {
933	if (avoid_reserve)
934	break;
935	if (!vma_has_reserves(vma, chg))
936	break;
937
938	SetPagePrivate(page);
939	h->resv_huge_pages--;
940	break;
941	}
942	}
943	}
944
945	mpol_cond_put(mpol);
946	if (unlikely(!page && read_mems_allowed_retry(cpuset_mems_cookie)))
947	goto retry_cpuset;
948	return page;
949
950	err:
951	return NULL;
952	}
953
954	/*
955	* common helper functions for hstate_next_node_to_{alloc|free}.
956	* We may have allocated or freed a huge page based on a different
957	* nodes_allowed previously, so h->next_node_to_{alloc|free} might
958	* be outside of *nodes_allowed. Ensure that we use an allowed
959	* node for alloc or free.
960	*/
961	static int next_node_allowed(int nid, nodemask_t *nodes_allowed)
962	{
963	nid = next_node_in(nid, *nodes_allowed);
964	VM_BUG_ON(nid >= MAX_NUMNODES);
965
966	return nid;
967	}
968
969	static int get_valid_node_allowed(int nid, nodemask_t *nodes_allowed)
970	{
971	if (!node_isset(nid, *nodes_allowed))
972	nid = next_node_allowed(nid, nodes_allowed);
973	return nid;
974	}
975
976	/*
977	* returns the previously saved node ["this node"] from which to
978	* allocate a persistent huge page for the pool and advance the
979	* next node from which to allocate, handling wrap at end of node
980	* mask.
981	*/
982	static int hstate_next_node_to_alloc(struct hstate *h,
983	nodemask_t *nodes_allowed)
984	{
985	int nid;
986
987	VM_BUG_ON(!nodes_allowed);
988
989	nid = get_valid_node_allowed(h->next_nid_to_alloc, nodes_allowed);
990	h->next_nid_to_alloc = next_node_allowed(nid, nodes_allowed);
991
992	return nid;
993	}
994
995	/*
996	* helper for free_pool_huge_page() - return the previously saved
997	* node ["this node"] from which to free a huge page. Advance the
998	* next node id whether or not we find a free huge page to free so
999	* that the next attempt to free addresses the next node.
1000	*/
1001	static int hstate_next_node_to_free(struct hstate *h, nodemask_t *nodes_allowed)
1002	{
1003	int nid;
1004
1005	VM_BUG_ON(!nodes_allowed);
1006
1007	nid = get_valid_node_allowed(h->next_nid_to_free, nodes_allowed);
1008	h->next_nid_to_free = next_node_allowed(nid, nodes_allowed);
1009
1010	return nid;
1011	}
1012
1013	#define for_each_node_mask_to_alloc(hs, nr_nodes, node, mask) \
1014	for (nr_nodes = nodes_weight(*mask); \
1015	nr_nodes > 0 && \
1016	((node = hstate_next_node_to_alloc(hs, mask)) || 1); \
1017	nr_nodes--)
1018
1019	#define for_each_node_mask_to_free(hs, nr_nodes, node, mask) \
1020	for (nr_nodes = nodes_weight(*mask); \
1021	nr_nodes > 0 && \
1022	((node = hstate_next_node_to_free(hs, mask)) || 1); \
1023	nr_nodes--)
1024
1025	#if defined(CONFIG_ARCH_HAS_GIGANTIC_PAGE) && \
1026	((defined(CONFIG_MEMORY_ISOLATION) && defined(CONFIG_COMPACTION)) || \
1027	defined(CONFIG_CMA))
1028	static void destroy_compound_gigantic_page(struct page *page,
1029	unsigned int order)
1030	{
1031	int i;
1032	int nr_pages = 1 << order;
1033	struct page *p = page + 1;
1034
1035	atomic_set(compound_mapcount_ptr(page), 0);
1036	for (i = 1; i < nr_pages; i++, p = mem_map_next(p, page, i)) {
1037	clear_compound_head(p);
1038	set_page_refcounted(p);
1039	}
1040
1041	set_compound_order(page, 0);
1042	__ClearPageHead(page);
1043	}
1044
1045	static void free_gigantic_page(struct page *page, unsigned int order)
1046	{
1047	free_contig_range(page_to_pfn(page), 1 << order);
1048	}
1049
1050	static int __alloc_gigantic_page(unsigned long start_pfn,
1051	unsigned long nr_pages)
1052	{
1053	unsigned long end_pfn = start_pfn + nr_pages;
1054	return alloc_contig_range(start_pfn, end_pfn, MIGRATE_MOVABLE);
1055	}
1056
1057	static bool pfn_range_valid_gigantic(struct zone *z,
1058	unsigned long start_pfn, unsigned long nr_pages)
1059	{
1060	unsigned long i, end_pfn = start_pfn + nr_pages;
1061	struct page *page;
1062
1063	for (i = start_pfn; i < end_pfn; i++) {
1064	if (!pfn_valid(i))
1065	return false;
1066
1067	page = pfn_to_page(i);
1068
1069	if (page_zone(page) != z)
1070	return false;
1071
1072	if (PageReserved(page))
1073	return false;
1074
1075	if (page_count(page) > 0)
1076	return false;
1077
1078	if (PageHuge(page))
1079	return false;
1080	}
1081
1082	return true;
1083	}
1084
1085	static bool zone_spans_last_pfn(const struct zone *zone,
1086	unsigned long start_pfn, unsigned long nr_pages)
1087	{
1088	unsigned long last_pfn = start_pfn + nr_pages - 1;
1089	return zone_spans_pfn(zone, last_pfn);
1090	}
1091
1092	static struct page *alloc_gigantic_page(int nid, unsigned int order)
1093	{
1094	unsigned long nr_pages = 1 << order;
1095	unsigned long ret, pfn, flags;
1096	struct zone *z;
1097
1098	z = NODE_DATA(nid)->node_zones;
1099	for (; z - NODE_DATA(nid)->node_zones < MAX_NR_ZONES; z++) {
1100	spin_lock_irqsave(&z->lock, flags);
1101
1102	pfn = ALIGN(z->zone_start_pfn, nr_pages);
1103	while (zone_spans_last_pfn(z, pfn, nr_pages)) {
1104	if (pfn_range_valid_gigantic(z, pfn, nr_pages)) {
1105	/*
1106	* We release the zone lock here because
1107	* alloc_contig_range() will also lock the zone
1108	* at some point. If there's an allocation
1109	* spinning on this lock, it may win the race
1110	* and cause alloc_contig_range() to fail...
1111	*/
1112	spin_unlock_irqrestore(&z->lock, flags);
1113	ret = __alloc_gigantic_page(pfn, nr_pages);
1114	if (!ret)
1115	return pfn_to_page(pfn);
1116	spin_lock_irqsave(&z->lock, flags);
1117	}
1118	pfn += nr_pages;
1119	}
1120
1121	spin_unlock_irqrestore(&z->lock, flags);
1122	}
1123
1124	return NULL;
1125	}
1126
1127	static void prep_new_huge_page(struct hstate *h, struct page *page, int nid);
1128	static void prep_compound_gigantic_page(struct page *page, unsigned int order);
1129
1130	static struct page *alloc_fresh_gigantic_page_node(struct hstate *h, int nid)
1131	{
1132	struct page *page;
1133
1134	page = alloc_gigantic_page(nid, huge_page_order(h));
1135	if (page) {
1136	prep_compound_gigantic_page(page, huge_page_order(h));
1137	prep_new_huge_page(h, page, nid);
1138	}
1139
1140	return page;
1141	}
1142
1143	static int alloc_fresh_gigantic_page(struct hstate *h,
1144	nodemask_t *nodes_allowed)
1145	{
1146	struct page *page = NULL;
1147	int nr_nodes, node;
1148
1149	for_each_node_mask_to_alloc(h, nr_nodes, node, nodes_allowed) {
1150	page = alloc_fresh_gigantic_page_node(h, node);
1151	if (page)
1152	return 1;
1153	}
1154
1155	return 0;
1156	}
1157
1158	static inline bool gigantic_page_supported(void) { return true; }
1159	#else
1160	static inline bool gigantic_page_supported(void) { return false; }
1161	static inline void free_gigantic_page(struct page *page, unsigned int order) { }
1162	static inline void destroy_compound_gigantic_page(struct page *page,
1163	unsigned int order) { }
1164	static inline int alloc_fresh_gigantic_page(struct hstate *h,
1165	nodemask_t *nodes_allowed) { return 0; }
1166	#endif
1167
1168	static void update_and_free_page(struct hstate *h, struct page *page)
1169	{
1170	int i;
1171
1172	if (hstate_is_gigantic(h) && !gigantic_page_supported())
1173	return;
1174
1175	h->nr_huge_pages--;
1176	h->nr_huge_pages_node[page_to_nid(page)]--;
1177	for (i = 0; i < pages_per_huge_page(h); i++) {
1178	page[i].flags &= ~(1 << PG_locked | 1 << PG_error |
1179	1 << PG_referenced | 1 << PG_dirty |
1180	1 << PG_active | 1 << PG_private |
1181	1 << PG_writeback);
1182	}
1183	VM_BUG_ON_PAGE(hugetlb_cgroup_from_page(page), page);
1184	set_compound_page_dtor(page, NULL_COMPOUND_DTOR);
1185	set_page_refcounted(page);
1186	if (hstate_is_gigantic(h)) {
1187	destroy_compound_gigantic_page(page, huge_page_order(h));
1188	free_gigantic_page(page, huge_page_order(h));
1189	} else {
1190	__free_pages(page, huge_page_order(h));
1191	}
1192	}
1193
1194	struct hstate *size_to_hstate(unsigned long size)
1195	{
1196	struct hstate *h;
1197
1198	for_each_hstate(h) {
1199	if (huge_page_size(h) == size)
1200	return h;
1201	}
1202	return NULL;
1203	}
1204
1205	/*
1206	* Test to determine whether the hugepage is "active/in-use" (i.e. being linked
1207	* to hstate->hugepage_activelist.)
1208	*
1209	* This function can be called for tail pages, but never returns true for them.
1210	*/
1211	bool page_huge_active(struct page *page)
1212	{
1213	VM_BUG_ON_PAGE(!PageHuge(page), page);
1214	return PageHead(page) && PagePrivate(&page[1]);
1215	}
1216
1217	/* never called for tail page */
1218	static void set_page_huge_active(struct page *page)
1219	{
1220	VM_BUG_ON_PAGE(!PageHeadHuge(page), page);
1221	SetPagePrivate(&page[1]);
1222	}
1223
1224	static void clear_page_huge_active(struct page *page)
1225	{
1226	VM_BUG_ON_PAGE(!PageHeadHuge(page), page);
1227	ClearPagePrivate(&page[1]);
1228	}
1229
1230	void free_huge_page(struct page *page)
1231	{
1232	/*
1233	* Can't pass hstate in here because it is called from the
1234	* compound page destructor.
1235	*/
1236	struct hstate *h = page_hstate(page);
1237	int nid = page_to_nid(page);
1238	struct hugepage_subpool *spool =
1239	(struct hugepage_subpool *)page_private(page);
1240	bool restore_reserve;
1241
1242	set_page_private(page, 0);
1243	page->mapping = NULL;
1244	VM_BUG_ON_PAGE(page_count(page), page);
1245	VM_BUG_ON_PAGE(page_mapcount(page), page);
1246	restore_reserve = PagePrivate(page);
1247	ClearPagePrivate(page);
1248
1249	/*
1250	* A return code of zero implies that the subpool will be under its
1251	* minimum size if the reservation is not restored after page is free.
1252	* Therefore, force restore_reserve operation.
1253	*/
1254	if (hugepage_subpool_put_pages(spool, 1) == 0)
1255	restore_reserve = true;
1256
1257	spin_lock(&hugetlb_lock);
1258	clear_page_huge_active(page);
1259	hugetlb_cgroup_uncharge_page(hstate_index(h),
1260	pages_per_huge_page(h), page);
1261	if (restore_reserve)
1262	h->resv_huge_pages++;
1263
1264	if (h->surplus_huge_pages_node[nid]) {
1265	/* remove the page from active list */
1266	list_del(&page->lru);
1267	update_and_free_page(h, page);
1268	h->surplus_huge_pages--;
1269	h->surplus_huge_pages_node[nid]--;
1270	} else {
1271	arch_clear_hugepage_flags(page);
1272	enqueue_huge_page(h, page);
1273	}
1274	spin_unlock(&hugetlb_lock);
1275	}
1276
1277	static void prep_new_huge_page(struct hstate *h, struct page *page, int nid)
1278	{
1279	INIT_LIST_HEAD(&page->lru);
1280	set_compound_page_dtor(page, HUGETLB_PAGE_DTOR);
1281	spin_lock(&hugetlb_lock);
1282	set_hugetlb_cgroup(page, NULL);
1283	h->nr_huge_pages++;
1284	h->nr_huge_pages_node[nid]++;
1285	spin_unlock(&hugetlb_lock);
1286	put_page(page); /* free it into the hugepage allocator */
1287	}
1288
1289	static void prep_compound_gigantic_page(struct page *page, unsigned int order)
1290	{
1291	int i;
1292	int nr_pages = 1 << order;
1293	struct page *p = page + 1;
1294
1295	/* we rely on prep_new_huge_page to set the destructor */
1296	set_compound_order(page, order);
1297	__ClearPageReserved(page);
1298	__SetPageHead(page);
1299	for (i = 1; i < nr_pages; i++, p = mem_map_next(p, page, i)) {
1300	/*
1301	* For gigantic hugepages allocated through bootmem at
1302	* boot, it's safer to be consistent with the not-gigantic
1303	* hugepages and clear the PG_reserved bit from all tail pages
1304	* too. Otherwse drivers using get_user_pages() to access tail
1305	* pages may get the reference counting wrong if they see
1306	* PG_reserved set on a tail page (despite the head page not
1307	* having PG_reserved set). Enforcing this consistency between
1308	* head and tail pages allows drivers to optimize away a check
1309	* on the head page when they need know if put_page() is needed
1310	* after get_user_pages().
1311	*/
1312	__ClearPageReserved(p);
1313	set_page_count(p, 0);
1314	set_compound_head(p, page);
1315	}
1316	atomic_set(compound_mapcount_ptr(page), -1);
1317	}
1318
1319	/*
1320	* PageHuge() only returns true for hugetlbfs pages, but not for normal or
1321	* transparent huge pages. See the PageTransHuge() documentation for more
1322	* details.
1323	*/
1324	int PageHuge(struct page *page)
1325	{
1326	if (!PageCompound(page))
1327	return 0;
1328
1329	page = compound_head(page);
1330	return page[1].compound_dtor == HUGETLB_PAGE_DTOR;
1331	}
1332	EXPORT_SYMBOL_GPL(PageHuge);
1333
1334	/*
1335	* PageHeadHuge() only returns true for hugetlbfs head page, but not for
1336	* normal or transparent huge pages.
1337	*/
1338	int PageHeadHuge(struct page *page_head)
1339	{
1340	if (!PageHead(page_head))
1341	return 0;
1342
1343	return get_compound_page_dtor(page_head) == free_huge_page;
1344	}
1345
1346	pgoff_t __basepage_index(struct page *page)
1347	{
1348	struct page *page_head = compound_head(page);
1349	pgoff_t index = page_index(page_head);
1350	unsigned long compound_idx;
1351
1352	if (!PageHuge(page_head))
1353	return page_index(page);
1354
1355	if (compound_order(page_head) >= MAX_ORDER)
1356	compound_idx = page_to_pfn(page) - page_to_pfn(page_head);
1357	else
1358	compound_idx = page - page_head;
1359
1360	return (index << compound_order(page_head)) + compound_idx;
1361	}
1362
1363	static struct page *alloc_fresh_huge_page_node(struct hstate *h, int nid)
1364	{
1365	struct page *page;
1366
1367	page = __alloc_pages_node(nid,
1368	htlb_alloc_mask(h)|__GFP_COMP|__GFP_THISNODE|
1369	__GFP_REPEAT|__GFP_NOWARN,
1370	huge_page_order(h));
1371	if (page) {
1372	prep_new_huge_page(h, page, nid);
1373	}
1374
1375	return page;
1376	}
1377
1378	static int alloc_fresh_huge_page(struct hstate *h, nodemask_t *nodes_allowed)
1379	{
1380	struct page *page;
1381	int nr_nodes, node;
1382	int ret = 0;
1383
1384	for_each_node_mask_to_alloc(h, nr_nodes, node, nodes_allowed) {
1385	page = alloc_fresh_huge_page_node(h, node);
1386	if (page) {
1387	ret = 1;
1388	break;
1389	}
1390	}
1391
1392	if (ret)
1393	count_vm_event(HTLB_BUDDY_PGALLOC);
1394	else
1395	count_vm_event(HTLB_BUDDY_PGALLOC_FAIL);
1396
1397	return ret;
1398	}
1399
1400	/*
1401	* Free huge page from pool from next node to free.
1402	* Attempt to keep persistent huge pages more or less
1403	* balanced over allowed nodes.
1404	* Called with hugetlb_lock locked.
1405	*/
1406	static int free_pool_huge_page(struct hstate *h, nodemask_t *nodes_allowed,
1407	bool acct_surplus)
1408	{
1409	int nr_nodes, node;
1410	int ret = 0;
1411
1412	for_each_node_mask_to_free(h, nr_nodes, node, nodes_allowed) {
1413	/*
1414	* If we're returning unused surplus pages, only examine
1415	* nodes with surplus pages.
1416	*/
1417	if ((!acct_surplus || h->surplus_huge_pages_node[node]) &&
1418	!list_empty(&h->hugepage_freelists[node])) {
1419	struct page *page =
1420	list_entry(h->hugepage_freelists[node].next,
1421	struct page, lru);
1422	list_del(&page->lru);
1423	h->free_huge_pages--;
1424	h->free_huge_pages_node[node]--;
1425	if (acct_surplus) {
1426	h->surplus_huge_pages--;
1427	h->surplus_huge_pages_node[node]--;
1428	}
1429	update_and_free_page(h, page);
1430	ret = 1;
1431	break;
1432	}
1433	}
1434
1435	return ret;
1436	}
1437
1438	/*
1439	* Dissolve a given free hugepage into free buddy pages. This function does
1440	* nothing for in-use (including surplus) hugepages. Returns -EBUSY if the
1441	* number of free hugepages would be reduced below the number of reserved
1442	* hugepages.
1443	*/
1444	static int dissolve_free_huge_page(struct page *page)
1445	{
1446	int rc = 0;
1447
1448	spin_lock(&hugetlb_lock);
1449	if (PageHuge(page) && !page_count(page)) {
1450	struct page *head = compound_head(page);
1451	struct hstate *h = page_hstate(head);
1452	int nid = page_to_nid(head);
1453	if (h->free_huge_pages - h->resv_huge_pages == 0) {
1454	rc = -EBUSY;
1455	goto out;
1456	}
1457	list_del(&head->lru);
1458	h->free_huge_pages--;
1459	h->free_huge_pages_node[nid]--;
1460	h->max_huge_pages--;
1461	update_and_free_page(h, head);
1462	}
1463	out:
1464	spin_unlock(&hugetlb_lock);
1465	return rc;
1466	}
1467
1468	/*
1469	* Dissolve free hugepages in a given pfn range. Used by memory hotplug to
1470	* make specified memory blocks removable from the system.
1471	* Note that this will dissolve a free gigantic hugepage completely, if any
1472	* part of it lies within the given range.
1473	* Also note that if dissolve_free_huge_page() returns with an error, all
1474	* free hugepages that were dissolved before that error are lost.
1475	*/
1476	int dissolve_free_huge_pages(unsigned long start_pfn, unsigned long end_pfn)
1477	{
1478	unsigned long pfn;
1479	struct page *page;
1480	int rc = 0;
1481
1482	if (!hugepages_supported())
1483	return rc;
1484
1485	for (pfn = start_pfn; pfn < end_pfn; pfn += 1 << minimum_order) {
1486	page = pfn_to_page(pfn);
1487	if (PageHuge(page) && !page_count(page)) {
1488	rc = dissolve_free_huge_page(page);
1489	if (rc)
1490	break;
1491	}
1492	}
1493
1494	return rc;
1495	}
1496
1497	/*
1498	* There are 3 ways this can get called:
1499	* 1. With vma+addr: we use the VMA's memory policy
1500	* 2. With !vma, but nid=NUMA_NO_NODE: We try to allocate a huge
1501	* page from any node, and let the buddy allocator itself figure
1502	* it out.
1503	* 3. With !vma, but nid!=NUMA_NO_NODE. We allocate a huge page
1504	* strictly from 'nid'
1505	*/
1506	static struct page *__hugetlb_alloc_buddy_huge_page(struct hstate *h,
1507	struct vm_area_struct *vma, unsigned long addr, int nid)
1508	{
1509	int order = huge_page_order(h);
1510	gfp_t gfp = htlb_alloc_mask(h)|__GFP_COMP|__GFP_REPEAT|__GFP_NOWARN;
1511	unsigned int cpuset_mems_cookie;
1512
1513	/*
1514	* We need a VMA to get a memory policy. If we do not
1515	* have one, we use the 'nid' argument.
1516	*
1517	* The mempolicy stuff below has some non-inlined bits
1518	* and calls ->vm_ops. That makes it hard to optimize at
1519	* compile-time, even when NUMA is off and it does
1520	* nothing. This helps the compiler optimize it out.
1521	*/
1522	if (!IS_ENABLED(CONFIG_NUMA) || !vma) {
1523	/*
1524	* If a specific node is requested, make sure to
1525	* get memory from there, but only when a node
1526	* is explicitly specified.
1527	*/
1528	if (nid != NUMA_NO_NODE)
1529	gfp |= __GFP_THISNODE;
1530	/*
1531	* Make sure to call something that can handle
1532	* nid=NUMA_NO_NODE
1533	*/
1534	return alloc_pages_node(nid, gfp, order);
1535	}
1536
1537	/*
1538	* OK, so we have a VMA. Fetch the mempolicy and try to
1539	* allocate a huge page with it. We will only reach this
1540	* when CONFIG_NUMA=y.
1541	*/
1542	do {
1543	struct page *page;
1544	struct mempolicy *mpol;
1545	struct zonelist *zl;
1546	nodemask_t *nodemask;
1547
1548	cpuset_mems_cookie = read_mems_allowed_begin();
1549	zl = huge_zonelist(vma, addr, gfp, &mpol, &nodemask);
1550	mpol_cond_put(mpol);
1551	page = __alloc_pages_nodemask(gfp, order, zl, nodemask);
1552	if (page)
1553	return page;
1554	} while (read_mems_allowed_retry(cpuset_mems_cookie));
1555
1556	return NULL;
1557	}
1558
1559	/*
1560	* There are two ways to allocate a huge page:
1561	* 1. When you have a VMA and an address (like a fault)
1562	* 2. When you have no VMA (like when setting /proc/.../nr_hugepages)
1563	*
1564	* 'vma' and 'addr' are only for (1). 'nid' is always NUMA_NO_NODE in
1565	* this case which signifies that the allocation should be done with
1566	* respect for the VMA's memory policy.
1567	*
1568	* For (2), we ignore 'vma' and 'addr' and use 'nid' exclusively. This
1569	* implies that memory policies will not be taken in to account.
1570	*/
1571	static struct page *__alloc_buddy_huge_page(struct hstate *h,
1572	struct vm_area_struct *vma, unsigned long addr, int nid)
1573	{
1574	struct page *page;
1575	unsigned int r_nid;
1576
1577	if (hstate_is_gigantic(h))
1578	return NULL;
1579
1580	/*
1581	* Make sure that anyone specifying 'nid' is not also specifying a VMA.
1582	* This makes sure the caller is picking _one_ of the modes with which
1583	* we can call this function, not both.
1584	*/
1585	if (vma || (addr != -1)) {
1586	VM_WARN_ON_ONCE(addr == -1);
1587	VM_WARN_ON_ONCE(nid != NUMA_NO_NODE);
1588	}
1589	/*
1590	* Assume we will successfully allocate the surplus page to
1591	* prevent racing processes from causing the surplus to exceed
1592	* overcommit
1593	*
1594	* This however introduces a different race, where a process B
1595	* tries to grow the static hugepage pool while alloc_pages() is
1596	* called by process A. B will only examine the per-node
1597	* counters in determining if surplus huge pages can be
1598	* converted to normal huge pages in adjust_pool_surplus(). A
1599	* won't be able to increment the per-node counter, until the
1600	* lock is dropped by B, but B doesn't drop hugetlb_lock until
1601	* no more huge pages can be converted from surplus to normal
1602	* state (and doesn't try to convert again). Thus, we have a
1603	* case where a surplus huge page exists, the pool is grown, and
1604	* the surplus huge page still exists after, even though it
1605	* should just have been converted to a normal huge page. This
1606	* does not leak memory, though, as the hugepage will be freed
1607	* once it is out of use. It also does not allow the counters to
1608	* go out of whack in adjust_pool_surplus() as we don't modify
1609	* the node values until we've gotten the hugepage and only the
1610	* per-node value is checked there.
1611	*/
1612	spin_lock(&hugetlb_lock);
1613	if (h->surplus_huge_pages >= h->nr_overcommit_huge_pages) {
1614	spin_unlock(&hugetlb_lock);
1615	return NULL;
1616	} else {
1617	h->nr_huge_pages++;
1618	h->surplus_huge_pages++;
1619	}
1620	spin_unlock(&hugetlb_lock);
1621
1622	page = __hugetlb_alloc_buddy_huge_page(h, vma, addr, nid);
1623
1624	spin_lock(&hugetlb_lock);
1625	if (page) {
1626	INIT_LIST_HEAD(&page->lru);
1627	r_nid = page_to_nid(page);
1628	set_compound_page_dtor(page, HUGETLB_PAGE_DTOR);
1629	set_hugetlb_cgroup(page, NULL);
1630	/*
1631	* We incremented the global counters already
1632	*/
1633	h->nr_huge_pages_node[r_nid]++;
1634	h->surplus_huge_pages_node[r_nid]++;
1635	__count_vm_event(HTLB_BUDDY_PGALLOC);
1636	} else {
1637	h->nr_huge_pages--;
1638	h->surplus_huge_pages--;
1639	__count_vm_event(HTLB_BUDDY_PGALLOC_FAIL);
1640	}
1641	spin_unlock(&hugetlb_lock);
1642
1643	return page;
1644	}
1645
1646	/*
1647	* Allocate a huge page from 'nid'. Note, 'nid' may be
1648	* NUMA_NO_NODE, which means that it may be allocated
1649	* anywhere.
1650	*/
1651	static
1652	struct page *__alloc_buddy_huge_page_no_mpol(struct hstate *h, int nid)
1653	{
1654	unsigned long addr = -1;
1655
1656	return __alloc_buddy_huge_page(h, NULL, addr, nid);
1657	}
1658
1659	/*
1660	* Use the VMA's mpolicy to allocate a huge page from the buddy.
1661	*/
1662	static
1663	struct page *__alloc_buddy_huge_page_with_mpol(struct hstate *h,
1664	struct vm_area_struct *vma, unsigned long addr)
1665	{
1666	return __alloc_buddy_huge_page(h, vma, addr, NUMA_NO_NODE);
1667	}
1668
1669	/*
1670	* This allocation function is useful in the context where vma is irrelevant.
1671	* E.g. soft-offlining uses this function because it only cares physical
1672	* address of error page.
1673	*/
1674	struct page *alloc_huge_page_node(struct hstate *h, int nid)
1675	{
1676	struct page *page = NULL;
1677
1678	spin_lock(&hugetlb_lock);
1679	if (h->free_huge_pages - h->resv_huge_pages > 0)
1680	page = dequeue_huge_page_node(h, nid);
1681	spin_unlock(&hugetlb_lock);
1682
1683	if (!page)
1684	page = __alloc_buddy_huge_page_no_mpol(h, nid);
1685
1686	return page;
1687	}
1688
1689	/*
1690	* Increase the hugetlb pool such that it can accommodate a reservation
1691	* of size 'delta'.
1692	*/
1693	static int gather_surplus_pages(struct hstate *h, int delta)
1694	{
1695	struct list_head surplus_list;
1696	struct page *page, *tmp;
1697	int ret, i;
1698	int needed, allocated;
1699	bool alloc_ok = true;
1700
1701	needed = (h->resv_huge_pages + delta) - h->free_huge_pages;
1702	if (needed <= 0) {
1703	h->resv_huge_pages += delta;
1704	return 0;
1705	}
1706
1707	allocated = 0;
1708	INIT_LIST_HEAD(&surplus_list);
1709
1710	ret = -ENOMEM;
1711	retry:
1712	spin_unlock(&hugetlb_lock);
1713	for (i = 0; i < needed; i++) {
1714	page = __alloc_buddy_huge_page_no_mpol(h, NUMA_NO_NODE);
1715	if (!page) {
1716	alloc_ok = false;
1717	break;
1718	}
1719	list_add(&page->lru, &surplus_list);
1720	}
1721	allocated += i;
1722
1723	/*
1724	* After retaking hugetlb_lock, we need to recalculate 'needed'
1725	* because either resv_huge_pages or free_huge_pages may have changed.
1726	*/
1727	spin_lock(&hugetlb_lock);
1728	needed = (h->resv_huge_pages + delta) -
1729	(h->free_huge_pages + allocated);
1730	if (needed > 0) {
1731	if (alloc_ok)
1732	goto retry;
1733	/*
1734	* We were not able to allocate enough pages to
1735	* satisfy the entire reservation so we free what
1736	* we've allocated so far.
1737	*/
1738	goto free;
1739	}
1740	/*
1741	* The surplus_list now contains _at_least_ the number of extra pages
1742	* needed to accommodate the reservation. Add the appropriate number
1743	* of pages to the hugetlb pool and free the extras back to the buddy
1744	* allocator. Commit the entire reservation here to prevent another
1745	* process from stealing the pages as they are added to the pool but
1746	* before they are reserved.
1747	*/
1748	needed += allocated;
1749	h->resv_huge_pages += delta;
1750	ret = 0;
1751
1752	/* Free the needed pages to the hugetlb pool */
1753	list_for_each_entry_safe(page, tmp, &surplus_list, lru) {
1754	if ((--needed) < 0)
1755	break;
1756	/*
1757	* This page is now managed by the hugetlb allocator and has
1758	* no users -- drop the buddy allocator's reference.
1759	*/
1760	put_page_testzero(page);
1761	VM_BUG_ON_PAGE(page_count(page), page);
1762	enqueue_huge_page(h, page);
1763	}
1764	free:
1765	spin_unlock(&hugetlb_lock);
1766
1767	/* Free unnecessary surplus pages to the buddy allocator */
1768	list_for_each_entry_safe(page, tmp, &surplus_list, lru)
1769	put_page(page);
1770	spin_lock(&hugetlb_lock);
1771
1772	return ret;
1773	}
1774
1775	/*
1776	* This routine has two main purposes:
1777	* 1) Decrement the reservation count (resv_huge_pages) by the value passed
1778	* in unused_resv_pages. This corresponds to the prior adjustments made
1779	* to the associated reservation map.
1780	* 2) Free any unused surplus pages that may have been allocated to satisfy
1781	* the reservation. As many as unused_resv_pages may be freed.
1782	*
1783	* Called with hugetlb_lock held. However, the lock could be dropped (and
1784	* reacquired) during calls to cond_resched_lock. Whenever dropping the lock,
1785	* we must make sure nobody else can claim pages we are in the process of
1786	* freeing. Do this by ensuring resv_huge_page always is greater than the
1787	* number of huge pages we plan to free when dropping the lock.
1788	*/
1789	static void return_unused_surplus_pages(struct hstate *h,
1790	unsigned long unused_resv_pages)
1791	{
1792	unsigned long nr_pages;
1793
1794	/* Cannot return gigantic pages currently */
1795	if (hstate_is_gigantic(h))
1796	goto out;
1797
1798	/*
1799	* Part (or even all) of the reservation could have been backed
1800	* by pre-allocated pages. Only free surplus pages.
1801	*/
1802	nr_pages = min(unused_resv_pages, h->surplus_huge_pages);
1803
1804	/*
1805	* We want to release as many surplus pages as possible, spread
1806	* evenly across all nodes with memory. Iterate across these nodes
1807	* until we can no longer free unreserved surplus pages. This occurs
1808	* when the nodes with surplus pages have no free pages.
1809	* free_pool_huge_page() will balance the the freed pages across the
1810	* on-line nodes with memory and will handle the hstate accounting.
1811	*
1812	* Note that we decrement resv_huge_pages as we free the pages. If
1813	* we drop the lock, resv_huge_pages will still be sufficiently large
1814	* to cover subsequent pages we may free.
1815	*/
1816	while (nr_pages--) {
1817	h->resv_huge_pages--;
1818	unused_resv_pages--;
1819	if (!free_pool_huge_page(h, &node_states[N_MEMORY], 1))
1820	goto out;
1821	cond_resched_lock(&hugetlb_lock);
1822	}
1823
1824	out:
1825	/* Fully uncommit the reservation */
1826	h->resv_huge_pages -= unused_resv_pages;
1827	}
1828
1829
1830	/*
1831	* vma_needs_reservation, vma_commit_reservation and vma_end_reservation
1832	* are used by the huge page allocation routines to manage reservations.
1833	*
1834	* vma_needs_reservation is called to determine if the huge page at addr
1835	* within the vma has an associated reservation. If a reservation is
1836	* needed, the value 1 is returned. The caller is then responsible for
1837	* managing the global reservation and subpool usage counts. After
1838	* the huge page has been allocated, vma_commit_reservation is called
1839	* to add the page to the reservation map. If the page allocation fails,
1840	* the reservation must be ended instead of committed. vma_end_reservation
1841	* is called in such cases.
1842	*
1843	* In the normal case, vma_commit_reservation returns the same value
1844	* as the preceding vma_needs_reservation call. The only time this
1845	* is not the case is if a reserve map was changed between calls. It
1846	* is the responsibility of the caller to notice the difference and
1847	* take appropriate action.
1848	*
1849	* vma_add_reservation is used in error paths where a reservation must
1850	* be restored when a newly allocated huge page must be freed. It is
1851	* to be called after calling vma_needs_reservation to determine if a
1852	* reservation exists.
1853	*/
1854	enum vma_resv_mode {
1855	VMA_NEEDS_RESV,
1856	VMA_COMMIT_RESV,
1857	VMA_END_RESV,
1858	VMA_ADD_RESV,
1859	};
1860	static long __vma_reservation_common(struct hstate *h,
1861	struct vm_area_struct *vma, unsigned long addr,
1862	enum vma_resv_mode mode)
1863	{
1864	struct resv_map *resv;
1865	pgoff_t idx;
1866	long ret;
1867
1868	resv = vma_resv_map(vma);
1869	if (!resv)
1870	return 1;
1871
1872	idx = vma_hugecache_offset(h, vma, addr);
1873	switch (mode) {
1874	case VMA_NEEDS_RESV:
1875	ret = region_chg(resv, idx, idx + 1);
1876	break;
1877	case VMA_COMMIT_RESV:
1878	ret = region_add(resv, idx, idx + 1);
1879	break;
1880	case VMA_END_RESV:
1881	region_abort(resv, idx, idx + 1);
1882	ret = 0;
1883	break;
1884	case VMA_ADD_RESV:
1885	if (vma->vm_flags & VM_MAYSHARE)
1886	ret = region_add(resv, idx, idx + 1);
1887	else {
1888	region_abort(resv, idx, idx + 1);
1889	ret = region_del(resv, idx, idx + 1);
1890	}
1891	break;
1892	default:
1893	BUG();
1894	}
1895
1896	if (vma->vm_flags & VM_MAYSHARE)
1897	return ret;
1898	else if (is_vma_resv_set(vma, HPAGE_RESV_OWNER) && ret >= 0) {
1899	/*
1900	* In most cases, reserves always exist for private mappings.
1901	* However, a file associated with mapping could have been
1902	* hole punched or truncated after reserves were consumed.
1903	* As subsequent fault on such a range will not use reserves.
1904	* Subtle - The reserve map for private mappings has the
1905	* opposite meaning than that of shared mappings. If NO
1906	* entry is in the reserve map, it means a reservation exists.
1907	* If an entry exists in the reserve map, it means the
1908	* reservation has already been consumed. As a result, the
1909	* return value of this routine is the opposite of the
1910	* value returned from reserve map manipulation routines above.
1911	*/
1912	if (ret)
1913	return 0;
1914	else
1915	return 1;
1916	}
1917	else
1918	return ret < 0 ? ret : 0;
1919	}
1920
1921	static long vma_needs_reservation(struct hstate *h,
1922	struct vm_area_struct *vma, unsigned long addr)
1923	{
1924	return __vma_reservation_common(h, vma, addr, VMA_NEEDS_RESV);
1925	}
1926
1927	static long vma_commit_reservation(struct hstate *h,
1928	struct vm_area_struct *vma, unsigned long addr)
1929	{
1930	return __vma_reservation_common(h, vma, addr, VMA_COMMIT_RESV);
1931	}
1932
1933	static void vma_end_reservation(struct hstate *h,
1934	struct vm_area_struct *vma, unsigned long addr)
1935	{
1936	(void)__vma_reservation_common(h, vma, addr, VMA_END_RESV);
1937	}
1938
1939	static long vma_add_reservation(struct hstate *h,
1940	struct vm_area_struct *vma, unsigned long addr)
1941	{
1942	return __vma_reservation_common(h, vma, addr, VMA_ADD_RESV);
1943	}
1944
1945	/*
1946	* This routine is called to restore a reservation on error paths. In the
1947	* specific error paths, a huge page was allocated (via alloc_huge_page)
1948	* and is about to be freed. If a reservation for the page existed,
1949	* alloc_huge_page would have consumed the reservation and set PagePrivate
1950	* in the newly allocated page. When the page is freed via free_huge_page,
1951	* the global reservation count will be incremented if PagePrivate is set.
1952	* However, free_huge_page can not adjust the reserve map. Adjust the
1953	* reserve map here to be consistent with global reserve count adjustments
1954	* to be made by free_huge_page.
1955	*/
1956	static void restore_reserve_on_error(struct hstate *h,
1957	struct vm_area_struct *vma, unsigned long address,
1958	struct page *page)
1959	{
1960	if (unlikely(PagePrivate(page))) {
1961	long rc = vma_needs_reservation(h, vma, address);
1962
1963	if (unlikely(rc < 0)) {
1964	/*
1965	* Rare out of memory condition in reserve map
1966	* manipulation. Clear PagePrivate so that
1967	* global reserve count will not be incremented
1968	* by free_huge_page. This will make it appear
1969	* as though the reservation for this page was
1970	* consumed. This may prevent the task from
1971	* faulting in the page at a later time. This
1972	* is better than inconsistent global huge page
1973	* accounting of reserve counts.
1974	*/
1975	ClearPagePrivate(page);
1976	} else if (rc) {
1977	rc = vma_add_reservation(h, vma, address);
1978	if (unlikely(rc < 0))
1979	/*
1980	* See above comment about rare out of
1981	* memory condition.
1982	*/
1983	ClearPagePrivate(page);
1984	} else
1985	vma_end_reservation(h, vma, address);
1986	}
1987	}
1988
1989	struct page *alloc_huge_page(struct vm_area_struct *vma,
1990	unsigned long addr, int avoid_reserve)
1991	{
1992	struct hugepage_subpool *spool = subpool_vma(vma);
1993	struct hstate *h = hstate_vma(vma);
1994	struct page *page;
1995	long map_chg, map_commit;
1996	long gbl_chg;
1997	int ret, idx;
1998	struct hugetlb_cgroup *h_cg;
1999
2000	idx = hstate_index(h);
2001	/*
2002	* Examine the region/reserve map to determine if the process
2003	* has a reservation for the page to be allocated. A return
2004	* code of zero indicates a reservation exists (no change).
2005	*/
2006	map_chg = gbl_chg = vma_needs_reservation(h, vma, addr);
2007	if (map_chg < 0)
2008	return ERR_PTR(-ENOMEM);
2009
2010	/*
2011	* Processes that did not create the mapping will have no
2012	* reserves as indicated by the region/reserve map. Check
2013	* that the allocation will not exceed the subpool limit.
2014	* Allocations for MAP_NORESERVE mappings also need to be
2015	* checked against any subpool limit.
2016	*/
2017	if (map_chg || avoid_reserve) {
2018	gbl_chg = hugepage_subpool_get_pages(spool, 1);
2019	if (gbl_chg < 0) {
2020	vma_end_reservation(h, vma, addr);
2021	return ERR_PTR(-ENOSPC);
2022	}
2023
2024	/*
2025	* Even though there was no reservation in the region/reserve
2026	* map, there could be reservations associated with the
2027	* subpool that can be used. This would be indicated if the
2028	* return value of hugepage_subpool_get_pages() is zero.
2029	* However, if avoid_reserve is specified we still avoid even
2030	* the subpool reservations.
2031	*/
2032	if (avoid_reserve)
2033	gbl_chg = 1;
2034	}
2035
2036	ret = hugetlb_cgroup_charge_cgroup(idx, pages_per_huge_page(h), &h_cg);
2037	if (ret)
2038	goto out_subpool_put;
2039
2040	spin_lock(&hugetlb_lock);
2041	/*
2042	* glb_chg is passed to indicate whether or not a page must be taken
2043	* from the global free pool (global change). gbl_chg == 0 indicates
2044	* a reservation exists for the allocation.
2045	*/
2046	page = dequeue_huge_page_vma(h, vma, addr, avoid_reserve, gbl_chg);
2047	if (!page) {
2048	spin_unlock(&hugetlb_lock);
2049	page = __alloc_buddy_huge_page_with_mpol(h, vma, addr);
2050	if (!page)
2051	goto out_uncharge_cgroup;
2052	if (!avoid_reserve && vma_has_reserves(vma, gbl_chg)) {
2053	SetPagePrivate(page);
2054	h->resv_huge_pages--;
2055	}
2056	spin_lock(&hugetlb_lock);
2057	list_move(&page->lru, &h->hugepage_activelist);
2058	/* Fall through */
2059	}
2060	hugetlb_cgroup_commit_charge(idx, pages_per_huge_page(h), h_cg, page);
2061	spin_unlock(&hugetlb_lock);
2062
2063	set_page_private(page, (unsigned long)spool);
2064
2065	map_commit = vma_commit_reservation(h, vma, addr);
2066	if (unlikely(map_chg > map_commit)) {
2067	/*
2068	* The page was added to the reservation map between
2069	* vma_needs_reservation and vma_commit_reservation.
2070	* This indicates a race with hugetlb_reserve_pages.
2071	* Adjust for the subpool count incremented above AND
2072	* in hugetlb_reserve_pages for the same page. Also,
2073	* the reservation count added in hugetlb_reserve_pages
2074	* no longer applies.
2075	*/
2076	long rsv_adjust;
2077
2078	rsv_adjust = hugepage_subpool_put_pages(spool, 1);
2079	hugetlb_acct_memory(h, -rsv_adjust);
2080	}
2081	return page;
2082
2083	out_uncharge_cgroup:
2084	hugetlb_cgroup_uncharge_cgroup(idx, pages_per_huge_page(h), h_cg);
2085	out_subpool_put:
2086	if (map_chg || avoid_reserve)
2087	hugepage_subpool_put_pages(spool, 1);
2088	vma_end_reservation(h, vma, addr);
2089	return ERR_PTR(-ENOSPC);
2090	}
2091
2092	/*
2093	* alloc_huge_page()'s wrapper which simply returns the page if allocation
2094	* succeeds, otherwise NULL. This function is called from new_vma_page(),
2095	* where no ERR_VALUE is expected to be returned.
2096	*/
2097	struct page *alloc_huge_page_noerr(struct vm_area_struct *vma,
2098	unsigned long addr, int avoid_reserve)
2099	{
2100	struct page *page = alloc_huge_page(vma, addr, avoid_reserve);
2101	if (IS_ERR(page))
2102	page = NULL;
2103	return page;
2104	}
2105
2106	int __weak alloc_bootmem_huge_page(struct hstate *h)
2107	{
2108	struct huge_bootmem_page *m;
2109	int nr_nodes, node;
2110
2111	for_each_node_mask_to_alloc(h, nr_nodes, node, &node_states[N_MEMORY]) {
2112	void *addr;
2113
2114	addr = memblock_virt_alloc_try_nid_nopanic(
2115	huge_page_size(h), huge_page_size(h),
2116	0, BOOTMEM_ALLOC_ACCESSIBLE, node);
2117	if (addr) {
2118	/*
2119	* Use the beginning of the huge page to store the
2120	* huge_bootmem_page struct (until gather_bootmem
2121	* puts them into the mem_map).
2122	*/
2123	m = addr;
2124	goto found;
2125	}
2126	}
2127	return 0;
2128
2129	found:
2130	BUG_ON(!IS_ALIGNED(virt_to_phys(m), huge_page_size(h)));
2131	/* Put them into a private list first because mem_map is not up yet */
2132	list_add(&m->list, &huge_boot_pages);
2133	m->hstate = h;
2134	return 1;
2135	}
2136
2137	static void __init prep_compound_huge_page(struct page *page,
2138	unsigned int order)
2139	{
2140	if (unlikely(order > (MAX_ORDER - 1)))
2141	prep_compound_gigantic_page(page, order);
2142	else
2143	prep_compound_page(page, order);
2144	}
2145
2146	/* Put bootmem huge pages into the standard lists after mem_map is up */
2147	static void __init gather_bootmem_prealloc(void)
2148	{
2149	struct huge_bootmem_page *m;
2150
2151	list_for_each_entry(m, &huge_boot_pages, list) {
2152	struct hstate *h = m->hstate;
2153	struct page *page;
2154
2155	#ifdef CONFIG_HIGHMEM
2156	page = pfn_to_page(m->phys >> PAGE_SHIFT);
2157	memblock_free_late(__pa(m),
2158	sizeof(struct huge_bootmem_page));
2159	#else
2160	page = virt_to_page(m);
2161	#endif
2162	WARN_ON(page_count(page) != 1);
2163	prep_compound_huge_page(page, h->order);
2164	WARN_ON(PageReserved(page));
2165	prep_new_huge_page(h, page, page_to_nid(page));
2166	/*
2167	* If we had gigantic hugepages allocated at boot time, we need
2168	* to restore the 'stolen' pages to totalram_pages in order to
2169	* fix confusing memory reports from free(1) and another
2170	* side-effects, like CommitLimit going negative.
2171	*/
2172	if (hstate_is_gigantic(h))
2173	adjust_managed_page_count(page, 1 << h->order);
2174	cond_resched();
2175	}
2176	}
2177
2178	static void __init hugetlb_hstate_alloc_pages(struct hstate *h)
2179	{
2180	unsigned long i;
2181
2182	for (i = 0; i < h->max_huge_pages; ++i) {
2183	if (hstate_is_gigantic(h)) {
2184	if (!alloc_bootmem_huge_page(h))
2185	break;
2186	} else if (!alloc_fresh_huge_page(h,
2187	&node_states[N_MEMORY]))
2188	break;
2189	}
2190	h->max_huge_pages = i;
2191	}
2192
2193	static void __init hugetlb_init_hstates(void)
2194	{
2195	struct hstate *h;
2196
2197	for_each_hstate(h) {
2198	if (minimum_order > huge_page_order(h))
2199	minimum_order = huge_page_order(h);
2200
2201	/* oversize hugepages were init'ed in early boot */
2202	if (!hstate_is_gigantic(h))
2203	hugetlb_hstate_alloc_pages(h);
2204	}
2205	VM_BUG_ON(minimum_order == UINT_MAX);
2206	}
2207
2208	static char * __init memfmt(char *buf, unsigned long n)
2209	{
2210	if (n >= (1UL << 30))
2211	sprintf(buf, "%lu GB", n >> 30);
2212	else if (n >= (1UL << 20))
2213	sprintf(buf, "%lu MB", n >> 20);
2214	else
2215	sprintf(buf, "%lu KB", n >> 10);
2216	return buf;
2217	}
2218
2219	static void __init report_hugepages(void)
2220	{
2221	struct hstate *h;
2222
2223	for_each_hstate(h) {
2224	char buf[32];
2225	pr_info("HugeTLB registered %s page size, pre-allocated %ld pages\n",
2226	memfmt(buf, huge_page_size(h)),
2227	h->free_huge_pages);
2228	}
2229	}
2230
2231	#ifdef CONFIG_HIGHMEM
2232	static void try_to_free_low(struct hstate *h, unsigned long count,
2233	nodemask_t *nodes_allowed)
2234	{
2235	int i;
2236
2237	if (hstate_is_gigantic(h))
2238	return;
2239
2240	for_each_node_mask(i, *nodes_allowed) {
2241	struct page *page, *next;
2242	struct list_head *freel = &h->hugepage_freelists[i];
2243	list_for_each_entry_safe(page, next, freel, lru) {
2244	if (count >= h->nr_huge_pages)
2245	return;
2246	if (PageHighMem(page))
2247	continue;
2248	list_del(&page->lru);
2249	update_and_free_page(h, page);
2250	h->free_huge_pages--;
2251	h->free_huge_pages_node[page_to_nid(page)]--;
2252	}
2253	}
2254	}
2255	#else
2256	static inline void try_to_free_low(struct hstate *h, unsigned long count,
2257	nodemask_t *nodes_allowed)
2258	{
2259	}
2260	#endif
2261
2262	/*
2263	* Increment or decrement surplus_huge_pages. Keep node-specific counters
2264	* balanced by operating on them in a round-robin fashion.
2265	* Returns 1 if an adjustment was made.
2266	*/
2267	static int adjust_pool_surplus(struct hstate *h, nodemask_t *nodes_allowed,
2268	int delta)
2269	{
2270	int nr_nodes, node;
2271
2272	VM_BUG_ON(delta != -1 && delta != 1);
2273
2274	if (delta < 0) {
2275	for_each_node_mask_to_alloc(h, nr_nodes, node, nodes_allowed) {
2276	if (h->surplus_huge_pages_node[node])
2277	goto found;
2278	}
2279	} else {
2280	for_each_node_mask_to_free(h, nr_nodes, node, nodes_allowed) {
2281	if (h->surplus_huge_pages_node[node] <
2282	h->nr_huge_pages_node[node])
2283	goto found;
2284	}
2285	}
2286	return 0;
2287
2288	found:
2289	h->surplus_huge_pages += delta;
2290	h->surplus_huge_pages_node[node] += delta;
2291	return 1;
2292	}
2293
2294	#define persistent_huge_pages(h) (h->nr_huge_pages - h->surplus_huge_pages)
2295	static unsigned long set_max_huge_pages(struct hstate *h, unsigned long count,
2296	nodemask_t *nodes_allowed)
2297	{
2298	unsigned long min_count, ret;
2299
2300	if (hstate_is_gigantic(h) && !gigantic_page_supported())
2301	return h->max_huge_pages;
2302
2303	/*
2304	* Increase the pool size
2305	* First take pages out of surplus state. Then make up the
2306	* remaining difference by allocating fresh huge pages.
2307	*
2308	* We might race with __alloc_buddy_huge_page() here and be unable
2309	* to convert a surplus huge page to a normal huge page. That is
2310	* not critical, though, it just means the overall size of the
2311	* pool might be one hugepage larger than it needs to be, but
2312	* within all the constraints specified by the sysctls.
2313	*/
2314	spin_lock(&hugetlb_lock);
2315	while (h->surplus_huge_pages && count > persistent_huge_pages(h)) {
2316	if (!adjust_pool_surplus(h, nodes_allowed, -1))
2317	break;
2318	}
2319
2320	while (count > persistent_huge_pages(h)) {
2321	/*
2322	* If this allocation races such that we no longer need the
2323	* page, free_huge_page will handle it by freeing the page
2324	* and reducing the surplus.
2325	*/
2326	spin_unlock(&hugetlb_lock);
2327
2328	/* yield cpu to avoid soft lockup */
2329	cond_resched();
2330
2331	if (hstate_is_gigantic(h))
2332	ret = alloc_fresh_gigantic_page(h, nodes_allowed);
2333	else
2334	ret = alloc_fresh_huge_page(h, nodes_allowed);
2335	spin_lock(&hugetlb_lock);
2336	if (!ret)
2337	goto out;
2338
2339	/* Bail for signals. Probably ctrl-c from user */
2340	if (signal_pending(current))
2341	goto out;
2342	}
2343
2344	/*
2345	* Decrease the pool size
2346	* First return free pages to the buddy allocator (being careful
2347	* to keep enough around to satisfy reservations). Then place
2348	* pages into surplus state as needed so the pool will shrink
2349	* to the desired size as pages become free.
2350	*
2351	* By placing pages into the surplus state independent of the
2352	* overcommit value, we are allowing the surplus pool size to
2353	* exceed overcommit. There are few sane options here. Since
2354	* __alloc_buddy_huge_page() is checking the global counter,
2355	* though, we'll note that we're not allowed to exceed surplus
2356	* and won't grow the pool anywhere else. Not until one of the
2357	* sysctls are changed, or the surplus pages go out of use.
2358	*/
2359	min_count = h->resv_huge_pages + h->nr_huge_pages - h->free_huge_pages;
2360	min_count = max(count, min_count);
2361	try_to_free_low(h, min_count, nodes_allowed);
2362	while (min_count < persistent_huge_pages(h)) {
2363	if (!free_pool_huge_page(h, nodes_allowed, 0))
2364	break;
2365	cond_resched_lock(&hugetlb_lock);
2366	}
2367	while (count < persistent_huge_pages(h)) {
2368	if (!adjust_pool_surplus(h, nodes_allowed, 1))
2369	break;
2370	}
2371	out:
2372	ret = persistent_huge_pages(h);
2373	spin_unlock(&hugetlb_lock);
2374	return ret;
2375	}
2376
2377	#define HSTATE_ATTR_RO(_name) \
2378	static struct kobj_attribute _name##_attr = __ATTR_RO(_name)
2379
2380	#define HSTATE_ATTR(_name) \
2381	static struct kobj_attribute _name##_attr = \
2382	__ATTR(_name, 0644, _name##_show, _name##_store)
2383
2384	static struct kobject *hugepages_kobj;
2385	static struct kobject *hstate_kobjs[HUGE_MAX_HSTATE];
2386
2387	static struct hstate *kobj_to_node_hstate(struct kobject *kobj, int *nidp);
2388
2389	static struct hstate *kobj_to_hstate(struct kobject *kobj, int *nidp)
2390	{
2391	int i;
2392
2393	for (i = 0; i < HUGE_MAX_HSTATE; i++)
2394	if (hstate_kobjs[i] == kobj) {
2395	if (nidp)
2396	*nidp = NUMA_NO_NODE;
2397	return &hstates[i];
2398	}
2399
2400	return kobj_to_node_hstate(kobj, nidp);
2401	}
2402
2403	static ssize_t nr_hugepages_show_common(struct kobject *kobj,
2404	struct kobj_attribute *attr, char *buf)
2405	{
2406	struct hstate *h;
2407	unsigned long nr_huge_pages;
2408	int nid;
2409
2410	h = kobj_to_hstate(kobj, &nid);
2411	if (nid == NUMA_NO_NODE)
2412	nr_huge_pages = h->nr_huge_pages;
2413	else
2414	nr_huge_pages = h->nr_huge_pages_node[nid];
2415
2416	return sprintf(buf, "%lu\n", nr_huge_pages);
2417	}
2418
2419	static ssize_t __nr_hugepages_store_common(bool obey_mempolicy,
2420	struct hstate *h, int nid,
2421	unsigned long count, size_t len)
2422	{
2423	int err;
2424	NODEMASK_ALLOC(nodemask_t, nodes_allowed, GFP_KERNEL | __GFP_NORETRY);
2425
2426	if (hstate_is_gigantic(h) && !gigantic_page_supported()) {
2427	err = -EINVAL;
2428	goto out;
2429	}
2430
2431	if (nid == NUMA_NO_NODE) {
2432	/*
2433	* global hstate attribute
2434	*/
2435	if (!(obey_mempolicy &&
2436	init_nodemask_of_mempolicy(nodes_allowed))) {
2437	NODEMASK_FREE(nodes_allowed);
2438	nodes_allowed = &node_states[N_MEMORY];
2439	}
2440	} else if (nodes_allowed) {
2441	/*
2442	* per node hstate attribute: adjust count to global,
2443	* but restrict alloc/free to the specified node.
2444	*/
2445	count += h->nr_huge_pages - h->nr_huge_pages_node[nid];
2446	init_nodemask_of_node(nodes_allowed, nid);
2447	} else
2448	nodes_allowed = &node_states[N_MEMORY];
2449
2450	h->max_huge_pages = set_max_huge_pages(h, count, nodes_allowed);
2451
2452	if (nodes_allowed != &node_states[N_MEMORY])
2453	NODEMASK_FREE(nodes_allowed);
2454
2455	return len;
2456	out:
2457	NODEMASK_FREE(nodes_allowed);
2458	return err;
2459	}
2460
2461	static ssize_t nr_hugepages_store_common(bool obey_mempolicy,
2462	struct kobject *kobj, const char *buf,
2463	size_t len)
2464	{
2465	struct hstate *h;
2466	unsigned long count;
2467	int nid;
2468	int err;
2469
2470	err = kstrtoul(buf, 10, &count);
2471	if (err)
2472	return err;
2473
2474	h = kobj_to_hstate(kobj, &nid);
2475	return __nr_hugepages_store_common(obey_mempolicy, h, nid, count, len);
2476	}
2477
2478	static ssize_t nr_hugepages_show(struct kobject *kobj,
2479	struct kobj_attribute *attr, char *buf)
2480	{
2481	return nr_hugepages_show_common(kobj, attr, buf);
2482	}
2483
2484	static ssize_t nr_hugepages_store(struct kobject *kobj,
2485	struct kobj_attribute *attr, const char *buf, size_t len)
2486	{
2487	return nr_hugepages_store_common(false, kobj, buf, len);
2488	}
2489	HSTATE_ATTR(nr_hugepages);
2490
2491	#ifdef CONFIG_NUMA
2492
2493	/*
2494	* hstate attribute for optionally mempolicy-based constraint on persistent
2495	* huge page alloc/free.
2496	*/
2497	static ssize_t nr_hugepages_mempolicy_show(struct kobject *kobj,
2498	struct kobj_attribute *attr, char *buf)
2499	{
2500	return nr_hugepages_show_common(kobj, attr, buf);
2501	}
2502
2503	static ssize_t nr_hugepages_mempolicy_store(struct kobject *kobj,
2504	struct kobj_attribute *attr, const char *buf, size_t len)
2505	{
2506	return nr_hugepages_store_common(true, kobj, buf, len);
2507	}
2508	HSTATE_ATTR(nr_hugepages_mempolicy);
2509	#endif
2510
2511
2512	static ssize_t nr_overcommit_hugepages_show(struct kobject *kobj,
2513	struct kobj_attribute *attr, char *buf)
2514	{
2515	struct hstate *h = kobj_to_hstate(kobj, NULL);
2516	return sprintf(buf, "%lu\n", h->nr_overcommit_huge_pages);
2517	}
2518
2519	static ssize_t nr_overcommit_hugepages_store(struct kobject *kobj,
2520	struct kobj_attribute *attr, const char *buf, size_t count)
2521	{
2522	int err;
2523	unsigned long input;
2524	struct hstate *h = kobj_to_hstate(kobj, NULL);
2525
2526	if (hstate_is_gigantic(h))
2527	return -EINVAL;
2528
2529	err = kstrtoul(buf, 10, &input);
2530	if (err)
2531	return err;
2532
2533	spin_lock(&hugetlb_lock);
2534	h->nr_overcommit_huge_pages = input;
2535	spin_unlock(&hugetlb_lock);
2536
2537	return count;
2538	}
2539	HSTATE_ATTR(nr_overcommit_hugepages);
2540
2541	static ssize_t free_hugepages_show(struct kobject *kobj,
2542	struct kobj_attribute *attr, char *buf)
2543	{
2544	struct hstate *h;
2545	unsigned long free_huge_pages;
2546	int nid;
2547
2548	h = kobj_to_hstate(kobj, &nid);
2549	if (nid == NUMA_NO_NODE)
2550	free_huge_pages = h->free_huge_pages;
2551	else
2552	free_huge_pages = h->free_huge_pages_node[nid];
2553
2554	return sprintf(buf, "%lu\n", free_huge_pages);
2555	}
2556	HSTATE_ATTR_RO(free_hugepages);
2557
2558	static ssize_t resv_hugepages_show(struct kobject *kobj,
2559	struct kobj_attribute *attr, char *buf)
2560	{
2561	struct hstate *h = kobj_to_hstate(kobj, NULL);
2562	return sprintf(buf, "%lu\n", h->resv_huge_pages);
2563	}
2564	HSTATE_ATTR_RO(resv_hugepages);
2565
2566	static ssize_t surplus_hugepages_show(struct kobject *kobj,
2567	struct kobj_attribute *attr, char *buf)
2568	{
2569	struct hstate *h;
2570	unsigned long surplus_huge_pages;
2571	int nid;
2572
2573	h = kobj_to_hstate(kobj, &nid);
2574	if (nid == NUMA_NO_NODE)
2575	surplus_huge_pages = h->surplus_huge_pages;
2576	else
2577	surplus_huge_pages = h->surplus_huge_pages_node[nid];
2578
2579	return sprintf(buf, "%lu\n", surplus_huge_pages);
2580	}
2581	HSTATE_ATTR_RO(surplus_hugepages);
2582
2583	static struct attribute *hstate_attrs[] = {
2584	&nr_hugepages_attr.attr,
2585	&nr_overcommit_hugepages_attr.attr,
2586	&free_hugepages_attr.attr,
2587	&resv_hugepages_attr.attr,
2588	&surplus_hugepages_attr.attr,
2589	#ifdef CONFIG_NUMA
2590	&nr_hugepages_mempolicy_attr.attr,
2591	#endif
2592	NULL,
2593	};
2594
2595	static struct attribute_group hstate_attr_group = {
2596	.attrs = hstate_attrs,
2597	};
2598
2599	static int hugetlb_sysfs_add_hstate(struct hstate *h, struct kobject *parent,
2600	struct kobject **hstate_kobjs,
2601	struct attribute_group *hstate_attr_group)
2602	{
2603	int retval;
2604	int hi = hstate_index(h);
2605
2606	hstate_kobjs[hi] = kobject_create_and_add(h->name, parent);
2607	if (!hstate_kobjs[hi])
2608	return -ENOMEM;
2609
2610	retval = sysfs_create_group(hstate_kobjs[hi], hstate_attr_group);
2611	if (retval)
2612	kobject_put(hstate_kobjs[hi]);
2613
2614	return retval;
2615	}
2616
2617	static void __init hugetlb_sysfs_init(void)
2618	{
2619	struct hstate *h;
2620	int err;
2621
2622	hugepages_kobj = kobject_create_and_add("hugepages", mm_kobj);
2623	if (!hugepages_kobj)
2624	return;
2625
2626	for_each_hstate(h) {
2627	err = hugetlb_sysfs_add_hstate(h, hugepages_kobj,
2628	hstate_kobjs, &hstate_attr_group);
2629	if (err)
2630	pr_err("Hugetlb: Unable to add hstate %s", h->name);
2631	}
2632	}
2633
2634	#ifdef CONFIG_NUMA
2635
2636	/*
2637	* node_hstate/s - associate per node hstate attributes, via their kobjects,
2638	* with node devices in node_devices[] using a parallel array. The array
2639	* index of a node device or _hstate == node id.
2640	* This is here to avoid any static dependency of the node device driver, in
2641	* the base kernel, on the hugetlb module.
2642	*/
2643	struct node_hstate {
2644	struct kobject *hugepages_kobj;
2645	struct kobject *hstate_kobjs[HUGE_MAX_HSTATE];
2646	};
2647	static struct node_hstate node_hstates[MAX_NUMNODES];
2648
2649	/*
2650	* A subset of global hstate attributes for node devices
2651	*/
2652	static struct attribute *per_node_hstate_attrs[] = {
2653	&nr_hugepages_attr.attr,
2654	&free_hugepages_attr.attr,
2655	&surplus_hugepages_attr.attr,
2656	NULL,
2657	};
2658
2659	static struct attribute_group per_node_hstate_attr_group = {
2660	.attrs = per_node_hstate_attrs,
2661	};
2662
2663	/*
2664	* kobj_to_node_hstate - lookup global hstate for node device hstate attr kobj.
2665	* Returns node id via non-NULL nidp.
2666	*/
2667	static struct hstate *kobj_to_node_hstate(struct kobject *kobj, int *nidp)
2668	{
2669	int nid;
2670
2671	for (nid = 0; nid < nr_node_ids; nid++) {
2672	struct node_hstate *nhs = &node_hstates[nid];
2673	int i;
2674	for (i = 0; i < HUGE_MAX_HSTATE; i++)
2675	if (nhs->hstate_kobjs[i] == kobj) {
2676	if (nidp)
2677	*nidp = nid;
2678	return &hstates[i];
2679	}
2680	}
2681
2682	BUG();
2683	return NULL;
2684	}
2685
2686	/*
2687	* Unregister hstate attributes from a single node device.
2688	* No-op if no hstate attributes attached.
2689	*/
2690	static void hugetlb_unregister_node(struct node *node)
2691	{
2692	struct hstate *h;
2693	struct node_hstate *nhs = &node_hstates[node->dev.id];
2694
2695	if (!nhs->hugepages_kobj)
2696	return; /* no hstate attributes */
2697
2698	for_each_hstate(h) {
2699	int idx = hstate_index(h);
2700	if (nhs->hstate_kobjs[idx]) {
2701	kobject_put(nhs->hstate_kobjs[idx]);
2702	nhs->hstate_kobjs[idx] = NULL;
2703	}
2704	}
2705
2706	kobject_put(nhs->hugepages_kobj);
2707	nhs->hugepages_kobj = NULL;
2708	}
2709
2710
2711	/*
2712	* Register hstate attributes for a single node device.
2713	* No-op if attributes already registered.
2714	*/
2715	static void hugetlb_register_node(struct node *node)
2716	{
2717	struct hstate *h;
2718	struct node_hstate *nhs = &node_hstates[node->dev.id];
2719	int err;
2720
2721	if (nhs->hugepages_kobj)
2722	return; /* already allocated */
2723
2724	nhs->hugepages_kobj = kobject_create_and_add("hugepages",
2725	&node->dev.kobj);
2726	if (!nhs->hugepages_kobj)
2727	return;
2728
2729	for_each_hstate(h) {
2730	err = hugetlb_sysfs_add_hstate(h, nhs->hugepages_kobj,
2731	nhs->hstate_kobjs,
2732	&per_node_hstate_attr_group);
2733	if (err) {
2734	pr_err("Hugetlb: Unable to add hstate %s for node %d\n",
2735	h->name, node->dev.id);
2736	hugetlb_unregister_node(node);
2737	break;
2738	}
2739	}
2740	}
2741
2742	/*
2743	* hugetlb init time: register hstate attributes for all registered node
2744	* devices of nodes that have memory. All on-line nodes should have
2745	* registered their associated device by this time.
2746	*/
2747	static void __init hugetlb_register_all_nodes(void)
2748	{
2749	int nid;
2750
2751	for_each_node_state(nid, N_MEMORY) {
2752	struct node *node = node_devices[nid];
2753	if (node->dev.id == nid)
2754	hugetlb_register_node(node);
2755	}
2756
2757	/*
2758	* Let the node device driver know we're here so it can
2759	* [un]register hstate attributes on node hotplug.
2760	*/
2761	register_hugetlbfs_with_node(hugetlb_register_node,
2762	hugetlb_unregister_node);
2763	}
2764	#else /* !CONFIG_NUMA */
2765
2766	static struct hstate *kobj_to_node_hstate(struct kobject *kobj, int *nidp)
2767	{
2768	BUG();
2769	if (nidp)
2770	*nidp = -1;
2771	return NULL;
2772	}
2773
2774	static void hugetlb_register_all_nodes(void) { }
2775
2776	#endif
2777
2778	static int __init hugetlb_init(void)
2779	{
2780	int i;
2781
2782	if (!hugepages_supported())
2783	return 0;
2784
2785	if (!size_to_hstate(default_hstate_size)) {
2786	default_hstate_size = HPAGE_SIZE;
2787	if (!size_to_hstate(default_hstate_size))
2788	hugetlb_add_hstate(HUGETLB_PAGE_ORDER);
2789	}
2790	default_hstate_idx = hstate_index(size_to_hstate(default_hstate_size));
2791	if (default_hstate_max_huge_pages) {
2792	if (!default_hstate.max_huge_pages)
2793	default_hstate.max_huge_pages = default_hstate_max_huge_pages;
2794	}
2795
2796	hugetlb_init_hstates();
2797	gather_bootmem_prealloc();
2798	report_hugepages();
2799
2800	hugetlb_sysfs_init();
2801	hugetlb_register_all_nodes();
2802	hugetlb_cgroup_file_init();
2803
2804	#ifdef CONFIG_SMP
2805	num_fault_mutexes = roundup_pow_of_two(8 * num_possible_cpus());
2806	#else
2807	num_fault_mutexes = 1;
2808	#endif
2809	hugetlb_fault_mutex_table =
2810	kmalloc(sizeof(struct mutex) * num_fault_mutexes, GFP_KERNEL);
2811	BUG_ON(!hugetlb_fault_mutex_table);
2812
2813	for (i = 0; i < num_fault_mutexes; i++)
2814	mutex_init(&hugetlb_fault_mutex_table[i]);
2815	return 0;
2816	}
2817	subsys_initcall(hugetlb_init);
2818
2819	/* Should be called on processing a hugepagesz=... option */
2820	void __init hugetlb_bad_size(void)
2821	{
2822	parsed_valid_hugepagesz = false;
2823	}
2824
2825	void __init hugetlb_add_hstate(unsigned int order)
2826	{
2827	struct hstate *h;
2828	unsigned long i;
2829
2830	if (size_to_hstate(PAGE_SIZE << order)) {
2831	pr_warn("hugepagesz= specified twice, ignoring\n");
2832	return;
2833	}
2834	BUG_ON(hugetlb_max_hstate >= HUGE_MAX_HSTATE);
2835	BUG_ON(order == 0);
2836	h = &hstates[hugetlb_max_hstate++];
2837	h->order = order;
2838	h->mask = ~((1ULL << (order + PAGE_SHIFT)) - 1);
2839	h->nr_huge_pages = 0;
2840	h->free_huge_pages = 0;
2841	for (i = 0; i < MAX_NUMNODES; ++i)
2842	INIT_LIST_HEAD(&h->hugepage_freelists[i]);
2843	INIT_LIST_HEAD(&h->hugepage_activelist);
2844	h->next_nid_to_alloc = first_memory_node;
2845	h->next_nid_to_free = first_memory_node;
2846	snprintf(h->name, HSTATE_NAME_LEN, "hugepages-%lukB",
2847	huge_page_size(h)/1024);
2848
2849	parsed_hstate = h;
2850	}
2851
2852	static int __init hugetlb_nrpages_setup(char *s)
2853	{
2854	unsigned long *mhp;
2855	static unsigned long *last_mhp;
2856
2857	if (!parsed_valid_hugepagesz) {
2858	pr_warn("hugepages = %s preceded by "
2859	"an unsupported hugepagesz, ignoring\n", s);
2860	parsed_valid_hugepagesz = true;
2861	return 1;
2862	}
2863	/*
2864	* !hugetlb_max_hstate means we haven't parsed a hugepagesz= parameter yet,
2865	* so this hugepages= parameter goes to the "default hstate".
2866	*/
2867	else if (!hugetlb_max_hstate)
2868	mhp = &default_hstate_max_huge_pages;
2869	else
2870	mhp = &parsed_hstate->max_huge_pages;
2871
2872	if (mhp == last_mhp) {
2873	pr_warn("hugepages= specified twice without interleaving hugepagesz=, ignoring\n");
2874	return 1;
2875	}
2876
2877	if (sscanf(s, "%lu", mhp) <= 0)
2878	*mhp = 0;
2879
2880	/*
2881	* Global state is always initialized later in hugetlb_init.
2882	* But we need to allocate >= MAX_ORDER hstates here early to still
2883	* use the bootmem allocator.
2884	*/
2885	if (hugetlb_max_hstate && parsed_hstate->order >= MAX_ORDER)
2886	hugetlb_hstate_alloc_pages(parsed_hstate);
2887
2888	last_mhp = mhp;
2889
2890	return 1;
2891	}
2892	__setup("hugepages=", hugetlb_nrpages_setup);
2893
2894	static int __init hugetlb_default_setup(char *s)
2895	{
2896	default_hstate_size = memparse(s, &s);
2897	return 1;
2898	}
2899	__setup("default_hugepagesz=", hugetlb_default_setup);
2900
2901	static unsigned int cpuset_mems_nr(unsigned int *array)
2902	{
2903	int node;
2904	unsigned int nr = 0;
2905
2906	for_each_node_mask(node, cpuset_current_mems_allowed)
2907	nr += array[node];
2908
2909	return nr;
2910	}
2911
2912	#ifdef CONFIG_SYSCTL
2913	static int hugetlb_sysctl_handler_common(bool obey_mempolicy,
2914	struct ctl_table *table, int write,
2915	void __user *buffer, size_t *length, loff_t *ppos)
2916	{
2917	struct hstate *h = &default_hstate;
2918	unsigned long tmp = h->max_huge_pages;
2919	int ret;
2920
2921	if (!hugepages_supported())
2922	return -EOPNOTSUPP;
2923
2924	table->data = &tmp;
2925	table->maxlen = sizeof(unsigned long);
2926	ret = proc_doulongvec_minmax(table, write, buffer, length, ppos);
2927	if (ret)
2928	goto out;
2929
2930	if (write)
2931	ret = __nr_hugepages_store_common(obey_mempolicy, h,
2932	NUMA_NO_NODE, tmp, *length);
2933	out:
2934	return ret;
2935	}
2936
2937	int hugetlb_sysctl_handler(struct ctl_table *table, int write,
2938	void __user *buffer, size_t *length, loff_t *ppos)
2939	{
2940
2941	return hugetlb_sysctl_handler_common(false, table, write,
2942	buffer, length, ppos);
2943	}
2944
2945	#ifdef CONFIG_NUMA
2946	int hugetlb_mempolicy_sysctl_handler(struct ctl_table *table, int write,
2947	void __user *buffer, size_t *length, loff_t *ppos)
2948	{
2949	return hugetlb_sysctl_handler_common(true, table, write,
2950	buffer, length, ppos);
2951	}
2952	#endif /* CONFIG_NUMA */
2953
2954	int hugetlb_overcommit_handler(struct ctl_table *table, int write,
2955	void __user *buffer,
2956	size_t *length, loff_t *ppos)
2957	{
2958	struct hstate *h = &default_hstate;
2959	unsigned long tmp;
2960	int ret;
2961
2962	if (!hugepages_supported())
2963	return -EOPNOTSUPP;
2964
2965	tmp = h->nr_overcommit_huge_pages;
2966
2967	if (write && hstate_is_gigantic(h))
2968	return -EINVAL;
2969
2970	table->data = &tmp;
2971	table->maxlen = sizeof(unsigned long);
2972	ret = proc_doulongvec_minmax(table, write, buffer, length, ppos);
2973	if (ret)
2974	goto out;
2975
2976	if (write) {
2977	spin_lock(&hugetlb_lock);
2978	h->nr_overcommit_huge_pages = tmp;
2979	spin_unlock(&hugetlb_lock);
2980	}
2981	out:
2982	return ret;
2983	}
2984
2985	#endif /* CONFIG_SYSCTL */
2986
2987	void hugetlb_report_meminfo(struct seq_file *m)
2988	{
2989	struct hstate *h = &default_hstate;
2990	if (!hugepages_supported())
2991	return;
2992	seq_printf(m,
2993	"HugePages_Total: %5lu\n"
2994	"HugePages_Free: %5lu\n"
2995	"HugePages_Rsvd: %5lu\n"
2996	"HugePages_Surp: %5lu\n"
2997	"Hugepagesize: %8lu kB\n",
2998	h->nr_huge_pages,
2999	h->free_huge_pages,
3000	h->resv_huge_pages,
3001	h->surplus_huge_pages,
3002	1UL << (huge_page_order(h) + PAGE_SHIFT - 10));
3003	}
3004
3005	int hugetlb_report_node_meminfo(int nid, char *buf)
3006	{
3007	struct hstate *h = &default_hstate;
3008	if (!hugepages_supported())
3009	return 0;
3010	return sprintf(buf,
3011	"Node %d HugePages_Total: %5u\n"
3012	"Node %d HugePages_Free: %5u\n"
3013	"Node %d HugePages_Surp: %5u\n",
3014	nid, h->nr_huge_pages_node[nid],
3015	nid, h->free_huge_pages_node[nid],
3016	nid, h->surplus_huge_pages_node[nid]);
3017	}
3018
3019	void hugetlb_show_meminfo(void)
3020	{
3021	struct hstate *h;
3022	int nid;
3023
3024	if (!hugepages_supported())
3025	return;
3026
3027	for_each_node_state(nid, N_MEMORY)
3028	for_each_hstate(h)
3029	pr_info("Node %d hugepages_total=%u hugepages_free=%u hugepages_surp=%u hugepages_size=%lukB\n",
3030	nid,
3031	h->nr_huge_pages_node[nid],
3032	h->free_huge_pages_node[nid],
3033	h->surplus_huge_pages_node[nid],
3034	1UL << (huge_page_order(h) + PAGE_SHIFT - 10));
3035	}
3036
3037	void hugetlb_report_usage(struct seq_file *m, struct mm_struct *mm)
3038	{
3039	seq_printf(m, "HugetlbPages:\t%8lu kB\n",
3040	atomic_long_read(&mm->hugetlb_usage) << (PAGE_SHIFT - 10));
3041	}
3042
3043	/* Return the number pages of memory we physically have, in PAGE_SIZE units. */
3044	unsigned long hugetlb_total_pages(void)
3045	{
3046	struct hstate *h;
3047	unsigned long nr_total_pages = 0;
3048
3049	for_each_hstate(h)
3050	nr_total_pages += h->nr_huge_pages * pages_per_huge_page(h);
3051	return nr_total_pages;
3052	}
3053
3054	static int hugetlb_acct_memory(struct hstate *h, long delta)
3055	{
3056	int ret = -ENOMEM;
3057
3058	spin_lock(&hugetlb_lock);
3059	/*
3060	* When cpuset is configured, it breaks the strict hugetlb page
3061	* reservation as the accounting is done on a global variable. Such
3062	* reservation is completely rubbish in the presence of cpuset because
3063	* the reservation is not checked against page availability for the
3064	* current cpuset. Application can still potentially OOM'ed by kernel
3065	* with lack of free htlb page in cpuset that the task is in.
3066	* Attempt to enforce strict accounting with cpuset is almost
3067	* impossible (or too ugly) because cpuset is too fluid that
3068	* task or memory node can be dynamically moved between cpusets.
3069	*
3070	* The change of semantics for shared hugetlb mapping with cpuset is
3071	* undesirable. However, in order to preserve some of the semantics,
3072	* we fall back to check against current free page availability as
3073	* a best attempt and hopefully to minimize the impact of changing
3074	* semantics that cpuset has.
3075	*/
3076	if (delta > 0) {
3077	if (gather_surplus_pages(h, delta) < 0)
3078	goto out;
3079
3080	if (delta > cpuset_mems_nr(h->free_huge_pages_node)) {
3081	return_unused_surplus_pages(h, delta);
3082	goto out;
3083	}
3084	}
3085
3086	ret = 0;
3087	if (delta < 0)
3088	return_unused_surplus_pages(h, (unsigned long) -delta);
3089
3090	out:
3091	spin_unlock(&hugetlb_lock);
3092	return ret;
3093	}
3094
3095	static void hugetlb_vm_op_open(struct vm_area_struct *vma)
3096	{
3097	struct resv_map *resv = vma_resv_map(vma);
3098
3099	/*
3100	* This new VMA should share its siblings reservation map if present.
3101	* The VMA will only ever have a valid reservation map pointer where
3102	* it is being copied for another still existing VMA. As that VMA
3103	* has a reference to the reservation map it cannot disappear until
3104	* after this open call completes. It is therefore safe to take a
3105	* new reference here without additional locking.
3106	*/
3107	if (resv && is_vma_resv_set(vma, HPAGE_RESV_OWNER))
3108	kref_get(&resv->refs);
3109	}
3110
3111	static void hugetlb_vm_op_close(struct vm_area_struct *vma)
3112	{
3113	struct hstate *h = hstate_vma(vma);
3114	struct resv_map *resv = vma_resv_map(vma);
3115	struct hugepage_subpool *spool = subpool_vma(vma);
3116	unsigned long reserve, start, end;
3117	long gbl_reserve;
3118
3119	if (!resv || !is_vma_resv_set(vma, HPAGE_RESV_OWNER))
3120	return;
3121
3122	start = vma_hugecache_offset(h, vma, vma->vm_start);
3123	end = vma_hugecache_offset(h, vma, vma->vm_end);
3124
3125	reserve = (end - start) - region_count(resv, start, end);
3126
3127	kref_put(&resv->refs, resv_map_release);
3128
3129	if (reserve) {
3130	/*
3131	* Decrement reserve counts. The global reserve count may be
3132	* adjusted if the subpool has a minimum size.
3133	*/
3134	gbl_reserve = hugepage_subpool_put_pages(spool, reserve);
3135	hugetlb_acct_memory(h, -gbl_reserve);
3136	}
3137	}
3138
3139	static int hugetlb_vm_op_split(struct vm_area_struct *vma, unsigned long addr)
3140	{
3141	if (addr & ~(huge_page_mask(hstate_vma(vma))))
3142	return -EINVAL;
3143	return 0;
3144	}
3145
3146	/*
3147	* We cannot handle pagefaults against hugetlb pages at all. They cause
3148	* handle_mm_fault() to try to instantiate regular-sized pages in the
3149	* hugegpage VMA. do_page_fault() is supposed to trap this, so BUG is we get
3150	* this far.
3151	*/
3152	static int hugetlb_vm_op_fault(struct vm_area_struct *vma, struct vm_fault *vmf)
3153	{
3154	BUG();
3155	return 0;
3156	}
3157
3158	const struct vm_operations_struct hugetlb_vm_ops = {
3159	.fault = hugetlb_vm_op_fault,
3160	.open = hugetlb_vm_op_open,
3161	.close = hugetlb_vm_op_close,
3162	.split = hugetlb_vm_op_split,
3163	};
3164
3165	static pte_t make_huge_pte(struct vm_area_struct *vma, struct page *page,
3166	int writable)
3167	{
3168	pte_t entry;
3169
3170	if (writable) {
3171	entry = huge_pte_mkwrite(huge_pte_mkdirty(mk_huge_pte(page,
3172	vma->vm_page_prot)));
3173	} else {
3174	entry = huge_pte_wrprotect(mk_huge_pte(page,
3175	vma->vm_page_prot));
3176	}
3177	entry = pte_mkyoung(entry);
3178	entry = pte_mkhuge(entry);
3179	entry = arch_make_huge_pte(entry, vma, page, writable);
3180
3181	return entry;
3182	}
3183
3184	static void set_huge_ptep_writable(struct vm_area_struct *vma,
3185	unsigned long address, pte_t *ptep)
3186	{
3187	pte_t entry;
3188
3189	entry = huge_pte_mkwrite(huge_pte_mkdirty(huge_ptep_get(ptep)));
3190	if (huge_ptep_set_access_flags(vma, address, ptep, entry, 1))
3191	update_mmu_cache(vma, address, ptep);
3192	}
3193
3194	static int is_hugetlb_entry_migration(pte_t pte)
3195	{
3196	swp_entry_t swp;
3197
3198	if (huge_pte_none(pte) || pte_present(pte))
3199	return 0;
3200	swp = pte_to_swp_entry(pte);
3201	if (non_swap_entry(swp) && is_migration_entry(swp))
3202	return 1;
3203	else
3204	return 0;
3205	}
3206
3207	static int is_hugetlb_entry_hwpoisoned(pte_t pte)
3208	{
3209	swp_entry_t swp;
3210
3211	if (huge_pte_none(pte) || pte_present(pte))
3212	return 0;
3213	swp = pte_to_swp_entry(pte);
3214	if (non_swap_entry(swp) && is_hwpoison_entry(swp))
3215	return 1;
3216	else
3217	return 0;
3218	}
3219
3220	int copy_hugetlb_page_range(struct mm_struct *dst, struct mm_struct *src,
3221	struct vm_area_struct *vma)
3222	{
3223	pte_t *src_pte, *dst_pte, entry, dst_entry;
3224	struct page *ptepage;
3225	unsigned long addr;
3226	int cow;
3227	struct hstate *h = hstate_vma(vma);
3228	unsigned long sz = huge_page_size(h);
3229	unsigned long mmun_start; /* For mmu_notifiers */
3230	unsigned long mmun_end; /* For mmu_notifiers */
3231	int ret = 0;
3232
3233	cow = (vma->vm_flags & (VM_SHARED | VM_MAYWRITE)) == VM_MAYWRITE;
3234
3235	mmun_start = vma->vm_start;
3236	mmun_end = vma->vm_end;
3237	if (cow)
3238	mmu_notifier_invalidate_range_start(src, mmun_start, mmun_end);
3239
3240	for (addr = vma->vm_start; addr < vma->vm_end; addr += sz) {
3241	spinlock_t *src_ptl, *dst_ptl;
3242	src_pte = huge_pte_offset(src, addr);
3243	if (!src_pte)
3244	continue;
3245	dst_pte = huge_pte_alloc(dst, addr, sz);
3246	if (!dst_pte) {
3247	ret = -ENOMEM;
3248	break;
3249	}
3250
3251	/*
3252	* If the pagetables are shared don't copy or take references.
3253	* dst_pte == src_pte is the common case of src/dest sharing.
3254	*
3255	* However, src could have 'unshared' and dst shares with
3256	* another vma. If dst_pte !none, this implies sharing.
3257	* Check here before taking page table lock, and once again
3258	* after taking the lock below.
3259	*/
3260	dst_entry = huge_ptep_get(dst_pte);
3261	if ((dst_pte == src_pte) || !huge_pte_none(dst_entry))
3262	continue;
3263
3264	dst_ptl = huge_pte_lock(h, dst, dst_pte);
3265	src_ptl = huge_pte_lockptr(h, src, src_pte);
3266	spin_lock_nested(src_ptl, SINGLE_DEPTH_NESTING);
3267	entry = huge_ptep_get(src_pte);
3268	dst_entry = huge_ptep_get(dst_pte);
3269	if (huge_pte_none(entry) || !huge_pte_none(dst_entry)) {
3270	/*
3271	* Skip if src entry none. Also, skip in the
3272	* unlikely case dst entry !none as this implies
3273	* sharing with another vma.
3274	*/
3275	;
3276	} else if (unlikely(is_hugetlb_entry_migration(entry) ||
3277	is_hugetlb_entry_hwpoisoned(entry))) {
3278	swp_entry_t swp_entry = pte_to_swp_entry(entry);
3279
3280	if (is_write_migration_entry(swp_entry) && cow) {
3281	/*
3282	* COW mappings require pages in both
3283	* parent and child to be set to read.
3284	*/
3285	make_migration_entry_read(&swp_entry);
3286	entry = swp_entry_to_pte(swp_entry);
3287	set_huge_pte_at(src, addr, src_pte, entry);
3288	}
3289	set_huge_pte_at(dst, addr, dst_pte, entry);
3290	} else {
3291	if (cow) {
3292	huge_ptep_set_wrprotect(src, addr, src_pte);
3293	mmu_notifier_invalidate_range(src, mmun_start,
3294	mmun_end);
3295	}
3296	entry = huge_ptep_get(src_pte);
3297	ptepage = pte_page(entry);
3298	get_page(ptepage);
3299	page_dup_rmap(ptepage, true);
3300	set_huge_pte_at(dst, addr, dst_pte, entry);
3301	hugetlb_count_add(pages_per_huge_page(h), dst);
3302	}
3303	spin_unlock(src_ptl);
3304	spin_unlock(dst_ptl);
3305	}
3306
3307	if (cow)
3308	mmu_notifier_invalidate_range_end(src, mmun_start, mmun_end);
3309
3310	return ret;
3311	}
3312
3313	void __unmap_hugepage_range(struct mmu_gather *tlb, struct vm_area_struct *vma,
3314	unsigned long start, unsigned long end,
3315	struct page *ref_page)
3316	{
3317	struct mm_struct *mm = vma->vm_mm;
3318	unsigned long address;
3319	pte_t *ptep;
3320	pte_t pte;
3321	spinlock_t *ptl;
3322	struct page *page;
3323	struct hstate *h = hstate_vma(vma);
3324	unsigned long sz = huge_page_size(h);
3325	const unsigned long mmun_start = start; /* For mmu_notifiers */
3326	const unsigned long mmun_end = end; /* For mmu_notifiers */
3327
3328	WARN_ON(!is_vm_hugetlb_page(vma));
3329	BUG_ON(start & ~huge_page_mask(h));
3330	BUG_ON(end & ~huge_page_mask(h));
3331
3332	tlb_start_vma(tlb, vma);
3333	mmu_notifier_invalidate_range_start(mm, mmun_start, mmun_end);
3334	address = start;
3335	for (; address < end; address += sz) {
3336	ptep = huge_pte_offset(mm, address);
3337	if (!ptep)
3338	continue;
3339
3340	ptl = huge_pte_lock(h, mm, ptep);
3341	if (huge_pmd_unshare(mm, &address, ptep)) {
3342	spin_unlock(ptl);
3343	continue;
3344	}
3345
3346	pte = huge_ptep_get(ptep);
3347	if (huge_pte_none(pte)) {
3348	spin_unlock(ptl);
3349	continue;
3350	}
3351
3352	/*
3353	* Migrating hugepage or HWPoisoned hugepage is already
3354	* unmapped and its refcount is dropped, so just clear pte here.
3355	*/
3356	if (unlikely(!pte_present(pte))) {
3357	huge_pte_clear(mm, address, ptep);
3358	spin_unlock(ptl);
3359	continue;
3360	}
3361
3362	page = pte_page(pte);
3363	/*
3364	* If a reference page is supplied, it is because a specific
3365	* page is being unmapped, not a range. Ensure the page we
3366	* are about to unmap is the actual page of interest.
3367	*/
3368	if (ref_page) {
3369	if (page != ref_page) {
3370	spin_unlock(ptl);
3371	continue;
3372	}
3373	/*
3374	* Mark the VMA as having unmapped its page so that
3375	* future faults in this VMA will fail rather than
3376	* looking like data was lost
3377	*/
3378	set_vma_resv_flags(vma, HPAGE_RESV_UNMAPPED);
3379	}
3380
3381	pte = huge_ptep_get_and_clear(mm, address, ptep);
3382	tlb_remove_tlb_entry(tlb, ptep, address);
3383	if (huge_pte_dirty(pte))
3384	set_page_dirty(page);
3385
3386	hugetlb_count_sub(pages_per_huge_page(h), mm);
3387	page_remove_rmap(page, true);
3388
3389	spin_unlock(ptl);
3390	tlb_remove_page_size(tlb, page, huge_page_size(h));
3391	/*
3392	* Bail out after unmapping reference page if supplied
3393	*/
3394	if (ref_page)
3395	break;
3396	}
3397	mmu_notifier_invalidate_range_end(mm, mmun_start, mmun_end);
3398	tlb_end_vma(tlb, vma);
3399	}
3400
3401	void __unmap_hugepage_range_final(struct mmu_gather *tlb,
3402	struct vm_area_struct *vma, unsigned long start,
3403	unsigned long end, struct page *ref_page)
3404	{
3405	__unmap_hugepage_range(tlb, vma, start, end, ref_page);
3406
3407	/*
3408	* Clear this flag so that x86's huge_pmd_share page_table_shareable
3409	* test will fail on a vma being torn down, and not grab a page table
3410	* on its way out. We're lucky that the flag has such an appropriate
3411	* name, and can in fact be safely cleared here. We could clear it
3412	* before the __unmap_hugepage_range above, but all that's necessary
3413	* is to clear it before releasing the i_mmap_rwsem. This works
3414	* because in the context this is called, the VMA is about to be
3415	* destroyed and the i_mmap_rwsem is held.
3416	*/
3417	vma->vm_flags &= ~VM_MAYSHARE;
3418	}
3419
3420	void unmap_hugepage_range(struct vm_area_struct *vma, unsigned long start,
3421	unsigned long end, struct page *ref_page)
3422	{
3423	struct mm_struct *mm;
3424	struct mmu_gather tlb;
3425
3426	mm = vma->vm_mm;
3427
3428	tlb_gather_mmu(&tlb, mm, start, end);
3429	__unmap_hugepage_range(&tlb, vma, start, end, ref_page);
3430	tlb_finish_mmu(&tlb, start, end);
3431	}
3432
3433	/*
3434	* This is called when the original mapper is failing to COW a MAP_PRIVATE
3435	* mappping it owns the reserve page for. The intention is to unmap the page
3436	* from other VMAs and let the children be SIGKILLed if they are faulting the
3437	* same region.
3438	*/
3439	static void unmap_ref_private(struct mm_struct *mm, struct vm_area_struct *vma,
3440	struct page *page, unsigned long address)
3441	{
3442	struct hstate *h = hstate_vma(vma);
3443	struct vm_area_struct *iter_vma;
3444	struct address_space *mapping;
3445	pgoff_t pgoff;
3446
3447	/*
3448	* vm_pgoff is in PAGE_SIZE units, hence the different calculation
3449	* from page cache lookup which is in HPAGE_SIZE units.
3450	*/
3451	address = address & huge_page_mask(h);
3452	pgoff = ((address - vma->vm_start) >> PAGE_SHIFT) +
3453	vma->vm_pgoff;
3454	mapping = vma->vm_file->f_mapping;
3455
3456	/*
3457	* Take the mapping lock for the duration of the table walk. As
3458	* this mapping should be shared between all the VMAs,
3459	* __unmap_hugepage_range() is called as the lock is already held
3460	*/
3461	i_mmap_lock_write(mapping);
3462	vma_interval_tree_foreach(iter_vma, &mapping->i_mmap, pgoff, pgoff) {
3463	/* Do not unmap the current VMA */
3464	if (iter_vma == vma)
3465	continue;
3466
3467	/*
3468	* Shared VMAs have their own reserves and do not affect
3469	* MAP_PRIVATE accounting but it is possible that a shared
3470	* VMA is using the same page so check and skip such VMAs.
3471	*/
3472	if (iter_vma->vm_flags & VM_MAYSHARE)
3473	continue;
3474
3475	/*
3476	* Unmap the page from other VMAs without their own reserves.
3477	* They get marked to be SIGKILLed if they fault in these
3478	* areas. This is because a future no-page fault on this VMA
3479	* could insert a zeroed page instead of the data existing
3480	* from the time of fork. This would look like data corruption
3481	*/
3482	if (!is_vma_resv_set(iter_vma, HPAGE_RESV_OWNER))
3483	unmap_hugepage_range(iter_vma, address,
3484	address + huge_page_size(h), page);
3485	}
3486	i_mmap_unlock_write(mapping);
3487	}
3488
3489	/*
3490	* Hugetlb_cow() should be called with page lock of the original hugepage held.
3491	* Called with hugetlb_instantiation_mutex held and pte_page locked so we
3492	* cannot race with other handlers or page migration.
3493	* Keep the pte_same checks anyway to make transition from the mutex easier.
3494	*/
3495	static int hugetlb_cow(struct mm_struct *mm, struct vm_area_struct *vma,
3496	unsigned long address, pte_t *ptep,
3497	struct page *pagecache_page, spinlock_t *ptl)
3498	{
3499	pte_t pte;
3500	struct hstate *h = hstate_vma(vma);
3501	struct page *old_page, *new_page;
3502	int ret = 0, outside_reserve = 0;
3503	unsigned long mmun_start; /* For mmu_notifiers */
3504	unsigned long mmun_end; /* For mmu_notifiers */
3505
3506	pte = huge_ptep_get(ptep);
3507	old_page = pte_page(pte);
3508
3509	retry_avoidcopy:
3510	/* If no-one else is actually using this page, avoid the copy
3511	* and just make the page writable */
3512	if (page_mapcount(old_page) == 1 && PageAnon(old_page)) {
3513	page_move_anon_rmap(old_page, vma);
3514	set_huge_ptep_writable(vma, address, ptep);
3515	return 0;
3516	}
3517
3518	/*
3519	* If the process that created a MAP_PRIVATE mapping is about to
3520	* perform a COW due to a shared page count, attempt to satisfy
3521	* the allocation without using the existing reserves. The pagecache
3522	* page is used to determine if the reserve at this address was
3523	* consumed or not. If reserves were used, a partial faulted mapping
3524	* at the time of fork() could consume its reserves on COW instead
3525	* of the full address range.
3526	*/
3527	if (is_vma_resv_set(vma, HPAGE_RESV_OWNER) &&
3528	old_page != pagecache_page)
3529	outside_reserve = 1;
3530
3531	get_page(old_page);
3532
3533	/*
3534	* Drop page table lock as buddy allocator may be called. It will
3535	* be acquired again before returning to the caller, as expected.
3536	*/
3537	spin_unlock(ptl);
3538	new_page = alloc_huge_page(vma, address, outside_reserve);
3539
3540	if (IS_ERR(new_page)) {
3541	/*
3542	* If a process owning a MAP_PRIVATE mapping fails to COW,
3543	* it is due to references held by a child and an insufficient
3544	* huge page pool. To guarantee the original mappers
3545	* reliability, unmap the page from child processes. The child
3546	* may get SIGKILLed if it later faults.
3547	*/
3548	if (outside_reserve) {
3549	put_page(old_page);
3550	BUG_ON(huge_pte_none(pte));
3551	unmap_ref_private(mm, vma, old_page, address);
3552	BUG_ON(huge_pte_none(pte));
3553	spin_lock(ptl);
3554	ptep = huge_pte_offset(mm, address & huge_page_mask(h));
3555	if (likely(ptep &&
3556	pte_same(huge_ptep_get(ptep), pte)))
3557	goto retry_avoidcopy;
3558	/*
3559	* race occurs while re-acquiring page table
3560	* lock, and our job is done.
3561	*/
3562	return 0;
3563	}
3564
3565	ret = (PTR_ERR(new_page) == -ENOMEM) ?
3566	VM_FAULT_OOM : VM_FAULT_SIGBUS;
3567	goto out_release_old;
3568	}
3569
3570	/*
3571	* When the original hugepage is shared one, it does not have
3572	* anon_vma prepared.
3573	*/
3574	if (unlikely(anon_vma_prepare(vma))) {
3575	ret = VM_FAULT_OOM;
3576	goto out_release_all;
3577	}
3578
3579	copy_user_huge_page(new_page, old_page, address, vma,
3580	pages_per_huge_page(h));
3581	__SetPageUptodate(new_page);
3582
3583	mmun_start = address & huge_page_mask(h);
3584	mmun_end = mmun_start + huge_page_size(h);
3585	mmu_notifier_invalidate_range_start(mm, mmun_start, mmun_end);
3586
3587	/*
3588	* Retake the page table lock to check for racing updates
3589	* before the page tables are altered
3590	*/
3591	spin_lock(ptl);
3592	ptep = huge_pte_offset(mm, address & huge_page_mask(h));
3593	if (likely(ptep && pte_same(huge_ptep_get(ptep), pte))) {
3594	ClearPagePrivate(new_page);
3595
3596	/* Break COW */
3597	huge_ptep_clear_flush(vma, address, ptep);
3598	mmu_notifier_invalidate_range(mm, mmun_start, mmun_end);
3599	set_huge_pte_at(mm, address, ptep,
3600	make_huge_pte(vma, new_page, 1));
3601	page_remove_rmap(old_page, true);
3602	hugepage_add_new_anon_rmap(new_page, vma, address);
3603	set_page_huge_active(new_page);
3604	/* Make the old page be freed below */
3605	new_page = old_page;
3606	}
3607	spin_unlock(ptl);
3608	mmu_notifier_invalidate_range_end(mm, mmun_start, mmun_end);
3609	out_release_all:
3610	restore_reserve_on_error(h, vma, address, new_page);
3611	put_page(new_page);
3612	out_release_old:
3613	put_page(old_page);
3614
3615	spin_lock(ptl); /* Caller expects lock to be held */
3616	return ret;
3617	}
3618
3619	/* Return the pagecache page at a given address within a VMA */
3620	static struct page *hugetlbfs_pagecache_page(struct hstate *h,
3621	struct vm_area_struct *vma, unsigned long address)
3622	{
3623	struct address_space *mapping;
3624	pgoff_t idx;
3625
3626	mapping = vma->vm_file->f_mapping;
3627	idx = vma_hugecache_offset(h, vma, address);
3628
3629	return find_lock_page(mapping, idx);
3630	}
3631
3632	/*
3633	* Return whether there is a pagecache page to back given address within VMA.
3634	* Caller follow_hugetlb_page() holds page_table_lock so we cannot lock_page.
3635	*/
3636	static bool hugetlbfs_pagecache_present(struct hstate *h,
3637	struct vm_area_struct *vma, unsigned long address)
3638	{
3639	struct address_space *mapping;
3640	pgoff_t idx;
3641	struct page *page;
3642
3643	mapping = vma->vm_file->f_mapping;
3644	idx = vma_hugecache_offset(h, vma, address);
3645
3646	page = find_get_page(mapping, idx);
3647	if (page)
3648	put_page(page);
3649	return page != NULL;
3650	}
3651
3652	int huge_add_to_page_cache(struct page *page, struct address_space *mapping,
3653	pgoff_t idx)
3654	{
3655	struct inode *inode = mapping->host;
3656	struct hstate *h = hstate_inode(inode);
3657	int err = add_to_page_cache(page, mapping, idx, GFP_KERNEL);
3658
3659	if (err)
3660	return err;
3661	ClearPagePrivate(page);
3662
3663	/*
3664	* set page dirty so that it will not be removed from cache/file
3665	* by non-hugetlbfs specific code paths.
3666	*/
3667	set_page_dirty(page);
3668
3669	spin_lock(&inode->i_lock);
3670	inode->i_blocks += blocks_per_huge_page(h);
3671	spin_unlock(&inode->i_lock);
3672	return 0;
3673	}
3674
3675	static int hugetlb_no_page(struct mm_struct *mm, struct vm_area_struct *vma,
3676	struct address_space *mapping, pgoff_t idx,
3677	unsigned long address, pte_t *ptep, unsigned int flags)
3678	{
3679	struct hstate *h = hstate_vma(vma);
3680	int ret = VM_FAULT_SIGBUS;
3681	int anon_rmap = 0;
3682	unsigned long size;
3683	struct page *page;
3684	pte_t new_pte;
3685	spinlock_t *ptl;
3686	bool new_page = false;
3687
3688	/*
3689	* Currently, we are forced to kill the process in the event the
3690	* original mapper has unmapped pages from the child due to a failed
3691	* COW. Warn that such a situation has occurred as it may not be obvious
3692	*/
3693	if (is_vma_resv_set(vma, HPAGE_RESV_UNMAPPED)) {
3694	pr_warn_ratelimited("PID %d killed due to inadequate hugepage pool\n",
3695	current->pid);
3696	return ret;
3697	}
3698
3699	/*
3700	* Use page lock to guard against racing truncation
3701	* before we get page_table_lock.
3702	*/
3703	retry:
3704	page = find_lock_page(mapping, idx);
3705	if (!page) {
3706	size = i_size_read(mapping->host) >> huge_page_shift(h);
3707	if (idx >= size)
3708	goto out;
3709	page = alloc_huge_page(vma, address, 0);
3710	if (IS_ERR(page)) {
3711	ret = PTR_ERR(page);
3712	if (ret == -ENOMEM)
3713	ret = VM_FAULT_OOM;
3714	else
3715	ret = VM_FAULT_SIGBUS;
3716	goto out;
3717	}
3718	clear_huge_page(page, address, pages_per_huge_page(h));
3719	__SetPageUptodate(page);
3720	new_page = true;
3721
3722	if (vma->vm_flags & VM_MAYSHARE) {
3723	int err = huge_add_to_page_cache(page, mapping, idx);
3724	if (err) {
3725	put_page(page);
3726	if (err == -EEXIST)
3727	goto retry;
3728	goto out;
3729	}
3730	} else {
3731	lock_page(page);
3732	if (unlikely(anon_vma_prepare(vma))) {
3733	ret = VM_FAULT_OOM;
3734	goto backout_unlocked;
3735	}
3736	anon_rmap = 1;
3737	}
3738	} else {
3739	/*
3740	* If memory error occurs between mmap() and fault, some process
3741	* don't have hwpoisoned swap entry for errored virtual address.
3742	* So we need to block hugepage fault by PG_hwpoison bit check.
3743	*/
3744	if (unlikely(PageHWPoison(page))) {
3745	ret = VM_FAULT_HWPOISON |
3746	VM_FAULT_SET_HINDEX(hstate_index(h));
3747	goto backout_unlocked;
3748	}
3749	}
3750
3751	/*
3752	* If we are going to COW a private mapping later, we examine the
3753	* pending reservations for this page now. This will ensure that
3754	* any allocations necessary to record that reservation occur outside
3755	* the spinlock.
3756	*/
3757	if ((flags & FAULT_FLAG_WRITE) && !(vma->vm_flags & VM_SHARED)) {
3758	if (vma_needs_reservation(h, vma, address) < 0) {
3759	ret = VM_FAULT_OOM;
3760	goto backout_unlocked;
3761	}
3762	/* Just decrements count, does not deallocate */
3763	vma_end_reservation(h, vma, address);
3764	}
3765
3766	ptl = huge_pte_lockptr(h, mm, ptep);
3767	spin_lock(ptl);
3768	size = i_size_read(mapping->host) >> huge_page_shift(h);
3769	if (idx >= size)
3770	goto backout;
3771
3772	ret = 0;
3773	if (!huge_pte_none(huge_ptep_get(ptep)))
3774	goto backout;
3775
3776	if (anon_rmap) {
3777	ClearPagePrivate(page);
3778	hugepage_add_new_anon_rmap(page, vma, address);
3779	} else
3780	page_dup_rmap(page, true);
3781	new_pte = make_huge_pte(vma, page, ((vma->vm_flags & VM_WRITE)
3782	&& (vma->vm_flags & VM_SHARED)));
3783	set_huge_pte_at(mm, address, ptep, new_pte);
3784
3785	hugetlb_count_add(pages_per_huge_page(h), mm);
3786	if ((flags & FAULT_FLAG_WRITE) && !(vma->vm_flags & VM_SHARED)) {
3787	/* Optimization, do the COW without a second fault */
3788	ret = hugetlb_cow(mm, vma, address, ptep, page, ptl);
3789	}
3790
3791	spin_unlock(ptl);
3792
3793	/*
3794	* Only make newly allocated pages active. Existing pages found
3795	* in the pagecache could be !page_huge_active() if they have been
3796	* isolated for migration.
3797	*/
3798	if (new_page)
3799	set_page_huge_active(page);
3800
3801	unlock_page(page);
3802	out:
3803	return ret;
3804
3805	backout:
3806	spin_unlock(ptl);
3807	backout_unlocked:
3808	unlock_page(page);
3809	restore_reserve_on_error(h, vma, address, page);
3810	put_page(page);
3811	goto out;
3812	}
3813
3814	#ifdef CONFIG_SMP
3815	u32 hugetlb_fault_mutex_hash(struct hstate *h, struct address_space *mapping,
3816	pgoff_t idx, unsigned long address)
3817	{
3818	unsigned long key[2];
3819	u32 hash;
3820
3821	key[0] = (unsigned long) mapping;
3822	key[1] = idx;
3823
3824	hash = jhash2((u32 *)&key, sizeof(key)/sizeof(u32), 0);
3825
3826	return hash & (num_fault_mutexes - 1);
3827	}
3828	#else
3829	/*
3830	* For uniprocesor systems we always use a single mutex, so just
3831	* return 0 and avoid the hashing overhead.
3832	*/
3833	u32 hugetlb_fault_mutex_hash(struct hstate *h, struct address_space *mapping,
3834	pgoff_t idx, unsigned long address)
3835	{
3836	return 0;
3837	}
3838	#endif
3839
3840	int hugetlb_fault(struct mm_struct *mm, struct vm_area_struct *vma,
3841	unsigned long address, unsigned int flags)
3842	{
3843	pte_t *ptep, entry;
3844	spinlock_t *ptl;
3845	int ret;
3846	u32 hash;
3847	pgoff_t idx;
3848	struct page *page = NULL;
3849	struct page *pagecache_page = NULL;
3850	struct hstate *h = hstate_vma(vma);
3851	struct address_space *mapping;
3852	int need_wait_lock = 0;
3853
3854	address &= huge_page_mask(h);
3855
3856	ptep = huge_pte_offset(mm, address);
3857	if (ptep) {
3858	entry = huge_ptep_get(ptep);
3859	if (unlikely(is_hugetlb_entry_migration(entry))) {
3860	migration_entry_wait_huge(vma, mm, ptep);
3861	return 0;
3862	} else if (unlikely(is_hugetlb_entry_hwpoisoned(entry)))
3863	return VM_FAULT_HWPOISON_LARGE |
3864	VM_FAULT_SET_HINDEX(hstate_index(h));
3865	} else {
3866	ptep = huge_pte_alloc(mm, address, huge_page_size(h));
3867	if (!ptep)
3868	return VM_FAULT_OOM;
3869	}
3870
3871	mapping = vma->vm_file->f_mapping;
3872	idx = vma_hugecache_offset(h, vma, address);
3873
3874	/*
3875	* Serialize hugepage allocation and instantiation, so that we don't
3876	* get spurious allocation failures if two CPUs race to instantiate
3877	* the same page in the page cache.
3878	*/
3879	hash = hugetlb_fault_mutex_hash(h, mapping, idx, address);
3880	mutex_lock(&hugetlb_fault_mutex_table[hash]);
3881
3882	entry = huge_ptep_get(ptep);
3883	if (huge_pte_none(entry)) {
3884	ret = hugetlb_no_page(mm, vma, mapping, idx, address, ptep, flags);
3885	goto out_mutex;
3886	}
3887
3888	ret = 0;
3889
3890	/*
3891	* entry could be a migration/hwpoison entry at this point, so this
3892	* check prevents the kernel from going below assuming that we have
3893	* a active hugepage in pagecache. This goto expects the 2nd page fault,
3894	* and is_hugetlb_entry_(migration|hwpoisoned) check will properly
3895	* handle it.
3896	*/
3897	if (!pte_present(entry))
3898	goto out_mutex;
3899
3900	/*
3901	* If we are going to COW the mapping later, we examine the pending
3902	* reservations for this page now. This will ensure that any
3903	* allocations necessary to record that reservation occur outside the
3904	* spinlock. For private mappings, we also lookup the pagecache
3905	* page now as it is used to determine if a reservation has been
3906	* consumed.
3907	*/
3908	if ((flags & FAULT_FLAG_WRITE) && !huge_pte_write(entry)) {
3909	if (vma_needs_reservation(h, vma, address) < 0) {
3910	ret = VM_FAULT_OOM;
3911	goto out_mutex;
3912	}
3913	/* Just decrements count, does not deallocate */
3914	vma_end_reservation(h, vma, address);
3915
3916	if (!(vma->vm_flags & VM_MAYSHARE))
3917	pagecache_page = hugetlbfs_pagecache_page(h,
3918	vma, address);
3919	}
3920
3921	ptl = huge_pte_lock(h, mm, ptep);
3922
3923	/* Check for a racing update before calling hugetlb_cow */
3924	if (unlikely(!pte_same(entry, huge_ptep_get(ptep))))
3925	goto out_ptl;
3926
3927	/*
3928	* hugetlb_cow() requires page locks of pte_page(entry) and
3929	* pagecache_page, so here we need take the former one
3930	* when page != pagecache_page or !pagecache_page.
3931	*/
3932	page = pte_page(entry);
3933	if (page != pagecache_page)
3934	if (!trylock_page(page)) {
3935	need_wait_lock = 1;
3936	goto out_ptl;
3937	}
3938
3939	get_page(page);
3940
3941	if (flags & FAULT_FLAG_WRITE) {
3942	if (!huge_pte_write(entry)) {
3943	ret = hugetlb_cow(mm, vma, address, ptep,
3944	pagecache_page, ptl);
3945	goto out_put_page;
3946	}
3947	entry = huge_pte_mkdirty(entry);
3948	}
3949	entry = pte_mkyoung(entry);
3950	if (huge_ptep_set_access_flags(vma, address, ptep, entry,
3951	flags & FAULT_FLAG_WRITE))
3952	update_mmu_cache(vma, address, ptep);
3953	out_put_page:
3954	if (page != pagecache_page)
3955	unlock_page(page);
3956	put_page(page);
3957	out_ptl:
3958	spin_unlock(ptl);
3959
3960	if (pagecache_page) {
3961	unlock_page(pagecache_page);
3962	put_page(pagecache_page);
3963	}
3964	out_mutex:
3965	mutex_unlock(&hugetlb_fault_mutex_table[hash]);
3966	/*
3967	* Generally it's safe to hold refcount during waiting page lock. But
3968	* here we just wait to defer the next page fault to avoid busy loop and
3969	* the page is not used after unlocked before returning from the current
3970	* page fault. So we are safe from accessing freed page, even if we wait
3971	* here without taking refcount.
3972	*/
3973	if (need_wait_lock)
3974	wait_on_page_locked(page);
3975	return ret;
3976	}
3977
3978	long follow_hugetlb_page(struct mm_struct *mm, struct vm_area_struct *vma,
3979	struct page **pages, struct vm_area_struct **vmas,
3980	unsigned long *position, unsigned long *nr_pages,
3981	long i, unsigned int flags)
3982	{
3983	unsigned long pfn_offset;
3984	unsigned long vaddr = *position;
3985	unsigned long remainder = *nr_pages;
3986	struct hstate *h = hstate_vma(vma);
3987
3988	while (vaddr < vma->vm_end && remainder) {
3989	pte_t *pte;
3990	spinlock_t *ptl = NULL;
3991	int absent;
3992	struct page *page;
3993
3994	/*
3995	* If we have a pending SIGKILL, don't keep faulting pages and
3996	* potentially allocating memory.
3997	*/
3998	if (unlikely(fatal_signal_pending(current))) {
3999	remainder = 0;
4000	break;
4001	}
4002
4003	/*
4004	* Some archs (sparc64, sh*) have multiple pte_ts to
4005	* each hugepage. We have to make sure we get the
4006	* first, for the page indexing below to work.
4007	*
4008	* Note that page table lock is not held when pte is null.
4009	*/
4010	pte = huge_pte_offset(mm, vaddr & huge_page_mask(h));
4011	if (pte)
4012	ptl = huge_pte_lock(h, mm, pte);
4013	absent = !pte || huge_pte_none(huge_ptep_get(pte));
4014
4015	/*
4016	* When coredumping, it suits get_dump_page if we just return
4017	* an error where there's an empty slot with no huge pagecache
4018	* to back it. This way, we avoid allocating a hugepage, and
4019	* the sparse dumpfile avoids allocating disk blocks, but its
4020	* huge holes still show up with zeroes where they need to be.
4021	*/
4022	if (absent && (flags & FOLL_DUMP) &&
4023	!hugetlbfs_pagecache_present(h, vma, vaddr)) {
4024	if (pte)
4025	spin_unlock(ptl);
4026	remainder = 0;
4027	break;
4028	}
4029
4030	/*
4031	* We need call hugetlb_fault for both hugepages under migration
4032	* (in which case hugetlb_fault waits for the migration,) and
4033	* hwpoisoned hugepages (in which case we need to prevent the
4034	* caller from accessing to them.) In order to do this, we use
4035	* here is_swap_pte instead of is_hugetlb_entry_migration and
4036	* is_hugetlb_entry_hwpoisoned. This is because it simply covers
4037	* both cases, and because we can't follow correct pages
4038	* directly from any kind of swap entries.
4039	*/
4040	if (absent || is_swap_pte(huge_ptep_get(pte)) ||
4041	((flags & FOLL_WRITE) &&
4042	!huge_pte_write(huge_ptep_get(pte)))) {
4043	int ret;
4044
4045	if (pte)
4046	spin_unlock(ptl);
4047	ret = hugetlb_fault(mm, vma, vaddr,
4048	(flags & FOLL_WRITE) ? FAULT_FLAG_WRITE : 0);
4049	if (!(ret & VM_FAULT_ERROR))
4050	continue;
4051
4052	remainder = 0;
4053	break;
4054	}
4055
4056	pfn_offset = (vaddr & ~huge_page_mask(h)) >> PAGE_SHIFT;
4057	page = pte_page(huge_ptep_get(pte));
4058	same_page:
4059	if (pages) {
4060	pages[i] = mem_map_offset(page, pfn_offset);
4061	get_page(pages[i]);
4062	}
4063
4064	if (vmas)
4065	vmas[i] = vma;
4066
4067	vaddr += PAGE_SIZE;
4068	++pfn_offset;
4069	--remainder;
4070	++i;
4071	if (vaddr < vma->vm_end && remainder &&
4072	pfn_offset < pages_per_huge_page(h)) {
4073	/*
4074	* We use pfn_offset to avoid touching the pageframes
4075	* of this compound page.
4076	*/
4077	goto same_page;
4078	}
4079	spin_unlock(ptl);
4080	}
4081	*nr_pages = remainder;
4082	*position = vaddr;
4083
4084	return i ? i : -EFAULT;
4085	}
4086
4087	#ifndef __HAVE_ARCH_FLUSH_HUGETLB_TLB_RANGE
4088	/*
4089	* ARCHes with special requirements for evicting HUGETLB backing TLB entries can
4090	* implement this.
4091	*/
4092	#define flush_hugetlb_tlb_range(vma, addr, end) flush_tlb_range(vma, addr, end)
4093	#endif
4094
4095	unsigned long hugetlb_change_protection(struct vm_area_struct *vma,
4096	unsigned long address, unsigned long end, pgprot_t newprot)
4097	{
4098	struct mm_struct *mm = vma->vm_mm;
4099	unsigned long start = address;
4100	pte_t *ptep;
4101	pte_t pte;
4102	struct hstate *h = hstate_vma(vma);
4103	unsigned long pages = 0;
4104
4105	BUG_ON(address >= end);
4106	flush_cache_range(vma, address, end);
4107
4108	mmu_notifier_invalidate_range_start(mm, start, end);
4109	i_mmap_lock_write(vma->vm_file->f_mapping);
4110	for (; address < end; address += huge_page_size(h)) {
4111	spinlock_t *ptl;
4112	ptep = huge_pte_offset(mm, address);
4113	if (!ptep)
4114	continue;
4115	ptl = huge_pte_lock(h, mm, ptep);
4116	if (huge_pmd_unshare(mm, &address, ptep)) {
4117	pages++;
4118	spin_unlock(ptl);
4119	continue;
4120	}
4121	pte = huge_ptep_get(ptep);
4122	if (unlikely(is_hugetlb_entry_hwpoisoned(pte))) {
4123	spin_unlock(ptl);
4124	continue;
4125	}
4126	if (unlikely(is_hugetlb_entry_migration(pte))) {
4127	swp_entry_t entry = pte_to_swp_entry(pte);
4128
4129	if (is_write_migration_entry(entry)) {
4130	pte_t newpte;
4131
4132	make_migration_entry_read(&entry);
4133	newpte = swp_entry_to_pte(entry);
4134	set_huge_pte_at(mm, address, ptep, newpte);
4135	pages++;
4136	}
4137	spin_unlock(ptl);
4138	continue;
4139	}
4140	if (!huge_pte_none(pte)) {
4141	pte = huge_ptep_get_and_clear(mm, address, ptep);
4142	pte = pte_mkhuge(huge_pte_modify(pte, newprot));
4143	pte = arch_make_huge_pte(pte, vma, NULL, 0);
4144	set_huge_pte_at(mm, address, ptep, pte);
4145	pages++;
4146	}
4147	spin_unlock(ptl);
4148	}
4149	/*
4150	* Must flush TLB before releasing i_mmap_rwsem: x86's huge_pmd_unshare
4151	* may have cleared our pud entry and done put_page on the page table:
4152	* once we release i_mmap_rwsem, another task can do the final put_page
4153	* and that page table be reused and filled with junk.
4154	*/
4155	flush_hugetlb_tlb_range(vma, start, end);
4156	mmu_notifier_invalidate_range(mm, start, end);
4157	i_mmap_unlock_write(vma->vm_file->f_mapping);
4158	mmu_notifier_invalidate_range_end(mm, start, end);
4159
4160	return pages << h->order;
4161	}
4162
4163	int hugetlb_reserve_pages(struct inode *inode,
4164	long from, long to,
4165	struct vm_area_struct *vma,
4166	vm_flags_t vm_flags)
4167	{
4168	long ret, chg;
4169	struct hstate *h = hstate_inode(inode);
4170	struct hugepage_subpool *spool = subpool_inode(inode);
4171	struct resv_map *resv_map;
4172	long gbl_reserve;
4173
4174	/* This should never happen */
4175	if (from > to) {
4176	VM_WARN(1, "%s called with a negative range\n", __func__);
4177	return -EINVAL;
4178	}
4179
4180	/*
4181	* Only apply hugepage reservation if asked. At fault time, an
4182	* attempt will be made for VM_NORESERVE to allocate a page
4183	* without using reserves
4184	*/
4185	if (vm_flags & VM_NORESERVE)
4186	return 0;
4187
4188	/*
4189	* Shared mappings base their reservation on the number of pages that
4190	* are already allocated on behalf of the file. Private mappings need
4191	* to reserve the full area even if read-only as mprotect() may be
4192	* called to make the mapping read-write. Assume !vma is a shm mapping
4193	*/
4194	if (!vma || vma->vm_flags & VM_MAYSHARE) {
4195	resv_map = inode_resv_map(inode);
4196
4197	chg = region_chg(resv_map, from, to);
4198
4199	} else {
4200	resv_map = resv_map_alloc();
4201	if (!resv_map)
4202	return -ENOMEM;
4203
4204	chg = to - from;
4205
4206	set_vma_resv_map(vma, resv_map);
4207	set_vma_resv_flags(vma, HPAGE_RESV_OWNER);
4208	}
4209
4210	if (chg < 0) {
4211	ret = chg;
4212	goto out_err;
4213	}
4214
4215	/*
4216	* There must be enough pages in the subpool for the mapping. If
4217	* the subpool has a minimum size, there may be some global
4218	* reservations already in place (gbl_reserve).
4219	*/
4220	gbl_reserve = hugepage_subpool_get_pages(spool, chg);
4221	if (gbl_reserve < 0) {
4222	ret = -ENOSPC;
4223	goto out_err;
4224	}
4225
4226	/*
4227	* Check enough hugepages are available for the reservation.
4228	* Hand the pages back to the subpool if there are not
4229	*/
4230	ret = hugetlb_acct_memory(h, gbl_reserve);
4231	if (ret < 0) {
4232	/* put back original number of pages, chg */
4233	(void)hugepage_subpool_put_pages(spool, chg);
4234	goto out_err;
4235	}
4236
4237	/*
4238	* Account for the reservations made. Shared mappings record regions
4239	* that have reservations as they are shared by multiple VMAs.
4240	* When the last VMA disappears, the region map says how much
4241	* the reservation was and the page cache tells how much of
4242	* the reservation was consumed. Private mappings are per-VMA and
4243	* only the consumed reservations are tracked. When the VMA
4244	* disappears, the original reservation is the VMA size and the
4245	* consumed reservations are stored in the map. Hence, nothing
4246	* else has to be done for private mappings here
4247	*/
4248	if (!vma || vma->vm_flags & VM_MAYSHARE) {
4249	long add = region_add(resv_map, from, to);
4250
4251	if (unlikely(chg > add)) {
4252	/*
4253	* pages in this range were added to the reserve
4254	* map between region_chg and region_add. This
4255	* indicates a race with alloc_huge_page. Adjust
4256	* the subpool and reserve counts modified above
4257	* based on the difference.
4258	*/
4259	long rsv_adjust;
4260
4261	rsv_adjust = hugepage_subpool_put_pages(spool,
4262	chg - add);
4263	hugetlb_acct_memory(h, -rsv_adjust);
4264	}
4265	}
4266	return 0;
4267	out_err:
4268	if (!vma || vma->vm_flags & VM_MAYSHARE)
4269	/* Don't call region_abort if region_chg failed */
4270	if (chg >= 0)
4271	region_abort(resv_map, from, to);
4272	if (vma && is_vma_resv_set(vma, HPAGE_RESV_OWNER))
4273	kref_put(&resv_map->refs, resv_map_release);
4274	return ret;
4275	}
4276
4277	long hugetlb_unreserve_pages(struct inode *inode, long start, long end,
4278	long freed)
4279	{
4280	struct hstate *h = hstate_inode(inode);
4281	struct resv_map *resv_map = inode_resv_map(inode);
4282	long chg = 0;
4283	struct hugepage_subpool *spool = subpool_inode(inode);
4284	long gbl_reserve;
4285
4286	if (resv_map) {
4287	chg = region_del(resv_map, start, end);
4288	/*
4289	* region_del() can fail in the rare case where a region
4290	* must be split and another region descriptor can not be
4291	* allocated. If end == LONG_MAX, it will not fail.
4292	*/
4293	if (chg < 0)
4294	return chg;
4295	}
4296
4297	spin_lock(&inode->i_lock);
4298	inode->i_blocks -= (blocks_per_huge_page(h) * freed);
4299	spin_unlock(&inode->i_lock);
4300
4301	/*
4302	* If the subpool has a minimum size, the number of global
4303	* reservations to be released may be adjusted.
4304	*/
4305	gbl_reserve = hugepage_subpool_put_pages(spool, (chg - freed));
4306	hugetlb_acct_memory(h, -gbl_reserve);
4307
4308	return 0;
4309	}
4310
4311	#ifdef CONFIG_ARCH_WANT_HUGE_PMD_SHARE
4312	static unsigned long page_table_shareable(struct vm_area_struct *svma,
4313	struct vm_area_struct *vma,
4314	unsigned long addr, pgoff_t idx)
4315	{
4316	unsigned long saddr = ((idx - svma->vm_pgoff) << PAGE_SHIFT) +
4317	svma->vm_start;
4318	unsigned long sbase = saddr & PUD_MASK;
4319	unsigned long s_end = sbase + PUD_SIZE;
4320
4321	/* Allow segments to share if only one is marked locked */
4322	unsigned long vm_flags = vma->vm_flags & VM_LOCKED_CLEAR_MASK;
4323	unsigned long svm_flags = svma->vm_flags & VM_LOCKED_CLEAR_MASK;
4324
4325	/*
4326	* match the virtual addresses, permission and the alignment of the
4327	* page table page.
4328	*/
4329	if (pmd_index(addr) != pmd_index(saddr) ||
4330	vm_flags != svm_flags ||
4331	sbase < svma->vm_start || svma->vm_end < s_end)
4332	return 0;
4333
4334	return saddr;
4335	}
4336
4337	static bool vma_shareable(struct vm_area_struct *vma, unsigned long addr)
4338	{
4339	unsigned long base = addr & PUD_MASK;
4340	unsigned long end = base + PUD_SIZE;
4341
4342	/*
4343	* check on proper vm_flags and page table alignment
4344	*/
4345	if (vma->vm_flags & VM_MAYSHARE && range_in_vma(vma, base, end))
4346	return true;
4347	return false;
4348	}
4349
4350	/*
4351	* Determine if start,end range within vma could be mapped by shared pmd.
4352	* If yes, adjust start and end to cover range associated with possible
4353	* shared pmd mappings.
4354	*/
4355	void adjust_range_if_pmd_sharing_possible(struct vm_area_struct *vma,
4356	unsigned long *start, unsigned long *end)
4357	{
4358	unsigned long check_addr = *start;
4359
4360	if (!(vma->vm_flags & VM_MAYSHARE))
4361	return;
4362
4363	for (check_addr = *start; check_addr < *end; check_addr += PUD_SIZE) {
4364	unsigned long a_start = check_addr & PUD_MASK;
4365	unsigned long a_end = a_start + PUD_SIZE;
4366
4367	/*
4368	* If sharing is possible, adjust start/end if necessary.
4369	*/
4370	if (range_in_vma(vma, a_start, a_end)) {
4371	if (a_start < *start)
4372	*start = a_start;
4373	if (a_end > *end)
4374	*end = a_end;
4375	}
4376	}
4377	}
4378
4379	/*
4380	* Search for a shareable pmd page for hugetlb. In any case calls pmd_alloc()
4381	* and returns the corresponding pte. While this is not necessary for the
4382	* !shared pmd case because we can allocate the pmd later as well, it makes the
4383	* code much cleaner. pmd allocation is essential for the shared case because
4384	* pud has to be populated inside the same i_mmap_rwsem section - otherwise
4385	* racing tasks could either miss the sharing (see huge_pte_offset) or select a
4386	* bad pmd for sharing.
4387	*/
4388	pte_t *huge_pmd_share(struct mm_struct *mm, unsigned long addr, pud_t *pud)
4389	{
4390	struct vm_area_struct *vma = find_vma(mm, addr);
4391	struct address_space *mapping = vma->vm_file->f_mapping;
4392	pgoff_t idx = ((addr - vma->vm_start) >> PAGE_SHIFT) +
4393	vma->vm_pgoff;
4394	struct vm_area_struct *svma;
4395	unsigned long saddr;
4396	pte_t *spte = NULL;
4397	pte_t *pte;
4398	spinlock_t *ptl;
4399
4400	if (!vma_shareable(vma, addr))
4401	return (pte_t *)pmd_alloc(mm, pud, addr);
4402
4403	i_mmap_lock_write(mapping);
4404	vma_interval_tree_foreach(svma, &mapping->i_mmap, idx, idx) {
4405	if (svma == vma)
4406	continue;
4407
4408	saddr = page_table_shareable(svma, vma, addr, idx);
4409	if (saddr) {
4410	spte = huge_pte_offset(svma->vm_mm, saddr);
4411	if (spte) {
4412	get_page(virt_to_page(spte));
4413	break;
4414	}
4415	}
4416	}
4417
4418	if (!spte)
4419	goto out;
4420
4421	ptl = huge_pte_lockptr(hstate_vma(vma), mm, spte);
4422	spin_lock(ptl);
4423	if (pud_none(*pud)) {
4424	pud_populate(mm, pud,
4425	(pmd_t *)((unsigned long)spte & PAGE_MASK));
4426	mm_inc_nr_pmds(mm);
4427	} else {
4428	put_page(virt_to_page(spte));
4429	}
4430	spin_unlock(ptl);
4431	out:
4432	pte = (pte_t *)pmd_alloc(mm, pud, addr);
4433	i_mmap_unlock_write(mapping);
4434	return pte;
4435	}
4436
4437	/*
4438	* unmap huge page backed by shared pte.
4439	*
4440	* Hugetlb pte page is ref counted at the time of mapping. If pte is shared
4441	* indicated by page_count > 1, unmap is achieved by clearing pud and
4442	* decrementing the ref count. If count == 1, the pte page is not shared.
4443	*
4444	* called with page table lock held.
4445	*
4446	* returns: 1 successfully unmapped a shared pte page
4447	* 0 the underlying pte page is not shared, or it is the last user
4448	*/
4449	int huge_pmd_unshare(struct mm_struct *mm, unsigned long *addr, pte_t *ptep)
4450	{
4451	pgd_t *pgd = pgd_offset(mm, *addr);
4452	pud_t *pud = pud_offset(pgd, *addr);
4453
4454	BUG_ON(page_count(virt_to_page(ptep)) == 0);
4455	if (page_count(virt_to_page(ptep)) == 1)
4456	return 0;
4457
4458	pud_clear(pud);
4459	put_page(virt_to_page(ptep));
4460	mm_dec_nr_pmds(mm);
4461	*addr = ALIGN(*addr, HPAGE_SIZE * PTRS_PER_PTE) - HPAGE_SIZE;
4462	return 1;
4463	}
4464	#define want_pmd_share() (1)
4465	#else /* !CONFIG_ARCH_WANT_HUGE_PMD_SHARE */
4466	pte_t *huge_pmd_share(struct mm_struct *mm, unsigned long addr, pud_t *pud)
4467	{
4468	return NULL;
4469	}
4470
4471	int huge_pmd_unshare(struct mm_struct *mm, unsigned long *addr, pte_t *ptep)
4472	{
4473	return 0;
4474	}
4475
4476	void adjust_range_if_pmd_sharing_possible(struct vm_area_struct *vma,
4477	unsigned long *start, unsigned long *end)
4478	{
4479	}
4480	#define want_pmd_share() (0)
4481	#endif /* CONFIG_ARCH_WANT_HUGE_PMD_SHARE */
4482
4483	#ifdef CONFIG_ARCH_WANT_GENERAL_HUGETLB
4484	pte_t *huge_pte_alloc(struct mm_struct *mm,
4485	unsigned long addr, unsigned long sz)
4486	{
4487	pgd_t *pgd;
4488	pud_t *pud;
4489	pte_t *pte = NULL;
4490
4491	pgd = pgd_offset(mm, addr);
4492	pud = pud_alloc(mm, pgd, addr);
4493	if (pud) {
4494	if (sz == PUD_SIZE) {
4495	pte = (pte_t *)pud;
4496	} else {
4497	BUG_ON(sz != PMD_SIZE);
4498	if (want_pmd_share() && pud_none(*pud))
4499	pte = huge_pmd_share(mm, addr, pud);
4500	else
4501	pte = (pte_t *)pmd_alloc(mm, pud, addr);
4502	}
4503	}
4504	BUG_ON(pte && pte_present(*pte) && !pte_huge(*pte));
4505
4506	return pte;
4507	}
4508
4509	pte_t *huge_pte_offset(struct mm_struct *mm, unsigned long addr)
4510	{
4511	pgd_t *pgd;
4512	pud_t *pud;
4513	pmd_t *pmd = NULL;
4514
4515	pgd = pgd_offset(mm, addr);
4516	if (pgd_present(*pgd)) {
4517	pud = pud_offset(pgd, addr);
4518	if (pud_present(*pud)) {
4519	if (pud_huge(*pud))
4520	return (pte_t *)pud;
4521	pmd = pmd_offset(pud, addr);
4522	}
4523	}
4524	return (pte_t *) pmd;
4525	}
4526
4527	#endif /* CONFIG_ARCH_WANT_GENERAL_HUGETLB */
4528
4529	/*
4530	* These functions are overwritable if your architecture needs its own
4531	* behavior.
4532	*/
4533	struct page * __weak
4534	follow_huge_addr(struct mm_struct *mm, unsigned long address,
4535	int write)
4536	{
4537	return ERR_PTR(-EINVAL);
4538	}
4539
4540	struct page * __weak
4541	follow_huge_pmd(struct mm_struct *mm, unsigned long address,
4542	pmd_t *pmd, int flags)
4543	{
4544	struct page *page = NULL;
4545	spinlock_t *ptl;
4546	pte_t pte;
4547	retry:
4548	ptl = pmd_lockptr(mm, pmd);
4549	spin_lock(ptl);
4550	/*
4551	* make sure that the address range covered by this pmd is not
4552	* unmapped from other threads.
4553	*/
4554	if (!pmd_huge(*pmd))
4555	goto out;
4556	pte = huge_ptep_get((pte_t *)pmd);
4557	if (pte_present(pte)) {
4558	page = pmd_page(*pmd) + ((address & ~PMD_MASK) >> PAGE_SHIFT);
4559	if (flags & FOLL_GET)
4560	get_page(page);
4561	} else {
4562	if (is_hugetlb_entry_migration(pte)) {
4563	spin_unlock(ptl);
4564	__migration_entry_wait(mm, (pte_t *)pmd, ptl);
4565	goto retry;
4566	}
4567	/*
4568	* hwpoisoned entry is treated as no_page_table in
4569	* follow_page_mask().
4570	*/
4571	}
4572	out:
4573	spin_unlock(ptl);
4574	return page;
4575	}
4576
4577	struct page * __weak
4578	follow_huge_pud(struct mm_struct *mm, unsigned long address,
4579	pud_t *pud, int flags)
4580	{
4581	if (flags & FOLL_GET)
4582	return NULL;
4583
4584	return pte_page(*(pte_t *)pud) + ((address & ~PUD_MASK) >> PAGE_SHIFT);
4585	}
4586
4587	#ifdef CONFIG_MEMORY_FAILURE
4588
4589	/*
4590	* This function is called from memory failure code.
4591	*/
4592	int dequeue_hwpoisoned_huge_page(struct page *hpage)
4593	{
4594	struct hstate *h = page_hstate(hpage);
4595	int nid = page_to_nid(hpage);
4596	int ret = -EBUSY;
4597
4598	spin_lock(&hugetlb_lock);
4599	/*
4600	* Just checking !page_huge_active is not enough, because that could be
4601	* an isolated/hwpoisoned hugepage (which have >0 refcount).
4602	*/
4603	if (!page_huge_active(hpage) && !page_count(hpage)) {
4604	/*
4605	* Hwpoisoned hugepage isn't linked to activelist or freelist,
4606	* but dangling hpage->lru can trigger list-debug warnings
4607	* (this happens when we call unpoison_memory() on it),
4608	* so let it point to itself with list_del_init().
4609	*/
4610	list_del_init(&hpage->lru);
4611	set_page_refcounted(hpage);
4612	h->free_huge_pages--;
4613	h->free_huge_pages_node[nid]--;
4614	ret = 0;
4615	}
4616	spin_unlock(&hugetlb_lock);
4617	return ret;
4618	}
4619	#endif
4620
4621	bool isolate_huge_page(struct page *page, struct list_head *list)
4622	{
4623	bool ret = true;
4624
4625	VM_BUG_ON_PAGE(!PageHead(page), page);
4626	spin_lock(&hugetlb_lock);
4627	if (!page_huge_active(page) || !get_page_unless_zero(page)) {
4628	ret = false;
4629	goto unlock;
4630	}
4631	clear_page_huge_active(page);
4632	list_move_tail(&page->lru, list);
4633	unlock:
4634	spin_unlock(&hugetlb_lock);
4635	return ret;
4636	}
4637
4638	void putback_active_hugepage(struct page *page)
4639	{
4640	VM_BUG_ON_PAGE(!PageHead(page), page);
4641	spin_lock(&hugetlb_lock);
4642	set_page_huge_active(page);
4643	list_move_tail(&page->lru, &(page_hstate(page))->hugepage_activelist);
4644	spin_unlock(&hugetlb_lock);
4645	put_page(page);
4646	}
4647