path: root/mm/memory.c (plain)
blob: 546e72d68abdc3e2a1894ebb4e1d33cebdd5aadb

1	/*
2	* linux/mm/memory.c
3	*
4	* Copyright (C) 1991, 1992, 1993, 1994 Linus Torvalds
5	*/
6
7	/*
8	* demand-loading started 01.12.91 - seems it is high on the list of
9	* things wanted, and it should be easy to implement. - Linus
10	*/
11
12	/*
13	* Ok, demand-loading was easy, shared pages a little bit tricker. Shared
14	* pages started 02.12.91, seems to work. - Linus.
15	*
16	* Tested sharing by executing about 30 /bin/sh: under the old kernel it
17	* would have taken more than the 6M I have free, but it worked well as
18	* far as I could see.
19	*
20	* Also corrected some "invalidate()"s - I wasn't doing enough of them.
21	*/
22
23	/*
24	* Real VM (paging to/from disk) started 18.12.91. Much more work and
25	* thought has to go into this. Oh, well..
26	* 19.12.91 - works, somewhat. Sometimes I get faults, don't know why.
27	* Found it. Everything seems to work now.
28	* 20.12.91 - Ok, making the swap-device changeable like the root.
29	*/
30
31	/*
32	* 05.04.94 - Multi-page memory management added for v1.1.
33	* Idea by Alex Bligh (alex@cconcepts.co.uk)
34	*
35	* 16.07.99 - Support of BIGMEM added by Gerhard Wichert, Siemens AG
36	* (Gerhard.Wichert@pdb.siemens.de)
37	*
38	* Aug/Sep 2004 Changed to four level page tables (Andi Kleen)
39	*/
40
41	#include <linux/kernel_stat.h>
42	#include <linux/mm.h>
43	#include <linux/hugetlb.h>
44	#include <linux/mman.h>
45	#include <linux/swap.h>
46	#include <linux/highmem.h>
47	#include <linux/pagemap.h>
48	#include <linux/ksm.h>
49	#include <linux/rmap.h>
50	#include <linux/export.h>
51	#include <linux/delayacct.h>
52	#include <linux/init.h>
53	#include <linux/pfn_t.h>
54	#include <linux/writeback.h>
55	#include <linux/memcontrol.h>
56	#include <linux/mmu_notifier.h>
57	#include <linux/kallsyms.h>
58	#include <linux/swapops.h>
59	#include <linux/elf.h>
60	#include <linux/gfp.h>
61	#include <linux/migrate.h>
62	#include <linux/string.h>
63	#include <linux/dma-debug.h>
64	#include <linux/debugfs.h>
65	#include <linux/userfaultfd_k.h>
66	#include <linux/dax.h>
67
68	#include <asm/io.h>
69	#include <asm/mmu_context.h>
70	#include <asm/pgalloc.h>
71	#include <asm/uaccess.h>
72	#include <asm/tlb.h>
73	#include <asm/tlbflush.h>
74	#include <asm/pgtable.h>
75
76	#include "internal.h"
77
78	#if defined(LAST_CPUPID_NOT_IN_PAGE_FLAGS) && !defined(CONFIG_COMPILE_TEST)
79	#warning Unfortunate NUMA and NUMA Balancing config, growing page-frame for last_cpupid.
80	#endif
81
82	#ifndef CONFIG_NEED_MULTIPLE_NODES
83	/* use the per-pgdat data instead for discontigmem - mbligh */
84	unsigned long max_mapnr;
85	struct page *mem_map;
86
87	EXPORT_SYMBOL(max_mapnr);
88	EXPORT_SYMBOL(mem_map);
89	#endif
90
91	/*
92	* A number of key systems in x86 including ioremap() rely on the assumption
93	* that high_memory defines the upper bound on direct map memory, then end
94	* of ZONE_NORMAL. Under CONFIG_DISCONTIG this means that max_low_pfn and
95	* highstart_pfn must be the same; there must be no gap between ZONE_NORMAL
96	* and ZONE_HIGHMEM.
97	*/
98	void * high_memory;
99
100	EXPORT_SYMBOL(high_memory);
101
102	/*
103	* Randomize the address space (stacks, mmaps, brk, etc.).
104	*
105	* ( When CONFIG_COMPAT_BRK=y we exclude brk from randomization,
106	* as ancient (libc5 based) binaries can segfault. )
107	*/
108	int randomize_va_space __read_mostly =
109	#ifdef CONFIG_COMPAT_BRK
110	1;
111	#else
112	2;
113	#endif
114
115	static int __init disable_randmaps(char *s)
116	{
117	randomize_va_space = 0;
118	return 1;
119	}
120	__setup("norandmaps", disable_randmaps);
121
122	unsigned long zero_pfn __read_mostly;
123	unsigned long highest_memmap_pfn __read_mostly;
124
125	EXPORT_SYMBOL(zero_pfn);
126
127	/*
128	* CONFIG_MMU architectures set up ZERO_PAGE in their paging_init()
129	*/
130	static int __init init_zero_pfn(void)
131	{
132	zero_pfn = page_to_pfn(ZERO_PAGE(0));
133	return 0;
134	}
135	core_initcall(init_zero_pfn);
136
137
138	#if defined(SPLIT_RSS_COUNTING)
139
140	void sync_mm_rss(struct mm_struct *mm)
141	{
142	int i;
143
144	for (i = 0; i < NR_MM_COUNTERS; i++) {
145	if (current->rss_stat.count[i]) {
146	add_mm_counter(mm, i, current->rss_stat.count[i]);
147	current->rss_stat.count[i] = 0;
148	}
149	}
150	current->rss_stat.events = 0;
151	}
152
153	static void add_mm_counter_fast(struct mm_struct *mm, int member, int val)
154	{
155	struct task_struct *task = current;
156
157	if (likely(task->mm == mm))
158	task->rss_stat.count[member] += val;
159	else
160	add_mm_counter(mm, member, val);
161	}
162	#define inc_mm_counter_fast(mm, member) add_mm_counter_fast(mm, member, 1)
163	#define dec_mm_counter_fast(mm, member) add_mm_counter_fast(mm, member, -1)
164
165	/* sync counter once per 64 page faults */
166	#define TASK_RSS_EVENTS_THRESH (64)
167	static void check_sync_rss_stat(struct task_struct *task)
168	{
169	if (unlikely(task != current))
170	return;
171	if (unlikely(task->rss_stat.events++ > TASK_RSS_EVENTS_THRESH))
172	sync_mm_rss(task->mm);
173	}
174	#else /* SPLIT_RSS_COUNTING */
175
176	#define inc_mm_counter_fast(mm, member) inc_mm_counter(mm, member)
177	#define dec_mm_counter_fast(mm, member) dec_mm_counter(mm, member)
178
179	static void check_sync_rss_stat(struct task_struct *task)
180	{
181	}
182
183	#endif /* SPLIT_RSS_COUNTING */
184
185	#ifdef HAVE_GENERIC_MMU_GATHER
186
187	static bool tlb_next_batch(struct mmu_gather *tlb)
188	{
189	struct mmu_gather_batch *batch;
190
191	batch = tlb->active;
192	if (batch->next) {
193	tlb->active = batch->next;
194	return true;
195	}
196
197	if (tlb->batch_count == MAX_GATHER_BATCH_COUNT)
198	return false;
199
200	batch = (void *)__get_free_pages(GFP_NOWAIT | __GFP_NOWARN, 0);
201	if (!batch)
202	return false;
203
204	tlb->batch_count++;
205	batch->next = NULL;
206	batch->nr = 0;
207	batch->max = MAX_GATHER_BATCH;
208
209	tlb->active->next = batch;
210	tlb->active = batch;
211
212	return true;
213	}
214
215	/* tlb_gather_mmu
216	* Called to initialize an (on-stack) mmu_gather structure for page-table
217	* tear-down from @mm. The @fullmm argument is used when @mm is without
218	* users and we're going to destroy the full address space (exit/execve).
219	*/
220	void tlb_gather_mmu(struct mmu_gather *tlb, struct mm_struct *mm, unsigned long start, unsigned long end)
221	{
222	tlb->mm = mm;
223
224	/* Is it from 0 to ~0? */
225	tlb->fullmm = !(start | (end+1));
226	tlb->need_flush_all = 0;
227	tlb->local.next = NULL;
228	tlb->local.nr = 0;
229	tlb->local.max = ARRAY_SIZE(tlb->__pages);
230	tlb->active = &tlb->local;
231	tlb->batch_count = 0;
232
233	#ifdef CONFIG_HAVE_RCU_TABLE_FREE
234	tlb->batch = NULL;
235	#endif
236	tlb->page_size = 0;
237
238	__tlb_reset_range(tlb);
239	}
240
241	static void tlb_flush_mmu_tlbonly(struct mmu_gather *tlb)
242	{
243	if (!tlb->end)
244	return;
245
246	tlb_flush(tlb);
247	mmu_notifier_invalidate_range(tlb->mm, tlb->start, tlb->end);
248	#ifdef CONFIG_HAVE_RCU_TABLE_FREE
249	tlb_table_flush(tlb);
250	#endif
251	__tlb_reset_range(tlb);
252	}
253
254	static void tlb_flush_mmu_free(struct mmu_gather *tlb)
255	{
256	struct mmu_gather_batch *batch;
257
258	for (batch = &tlb->local; batch && batch->nr; batch = batch->next) {
259	free_pages_and_swap_cache(batch->pages, batch->nr);
260	batch->nr = 0;
261	}
262	tlb->active = &tlb->local;
263	}
264
265	void tlb_flush_mmu(struct mmu_gather *tlb)
266	{
267	tlb_flush_mmu_tlbonly(tlb);
268	tlb_flush_mmu_free(tlb);
269	}
270
271	/* tlb_finish_mmu
272	* Called at the end of the shootdown operation to free up any resources
273	* that were required.
274	*/
275	void tlb_finish_mmu(struct mmu_gather *tlb, unsigned long start, unsigned long end)
276	{
277	struct mmu_gather_batch *batch, *next;
278
279	tlb_flush_mmu(tlb);
280
281	/* keep the page table cache within bounds */
282	check_pgt_cache();
283
284	for (batch = tlb->local.next; batch; batch = next) {
285	next = batch->next;
286	free_pages((unsigned long)batch, 0);
287	}
288	tlb->local.next = NULL;
289	}
290
291	/* __tlb_remove_page
292	* Must perform the equivalent to __free_pte(pte_get_and_clear(ptep)), while
293	* handling the additional races in SMP caused by other CPUs caching valid
294	* mappings in their TLBs. Returns the number of free page slots left.
295	* When out of page slots we must call tlb_flush_mmu().
296	*returns true if the caller should flush.
297	*/
298	bool __tlb_remove_page_size(struct mmu_gather *tlb, struct page *page, int page_size)
299	{
300	struct mmu_gather_batch *batch;
301
302	VM_BUG_ON(!tlb->end);
303
304	if (!tlb->page_size)
305	tlb->page_size = page_size;
306	else {
307	if (page_size != tlb->page_size)
308	return true;
309	}
310
311	batch = tlb->active;
312	if (batch->nr == batch->max) {
313	if (!tlb_next_batch(tlb))
314	return true;
315	batch = tlb->active;
316	}
317	VM_BUG_ON_PAGE(batch->nr > batch->max, page);
318
319	batch->pages[batch->nr++] = page;
320	return false;
321	}
322
323	#endif /* HAVE_GENERIC_MMU_GATHER */
324
325	#ifdef CONFIG_HAVE_RCU_TABLE_FREE
326
327	/*
328	* See the comment near struct mmu_table_batch.
329	*/
330
331	static void tlb_remove_table_smp_sync(void *arg)
332	{
333	/* Simply deliver the interrupt */
334	}
335
336	static void tlb_remove_table_one(void *table)
337	{
338	/*
339	* This isn't an RCU grace period and hence the page-tables cannot be
340	* assumed to be actually RCU-freed.
341	*
342	* It is however sufficient for software page-table walkers that rely on
343	* IRQ disabling. See the comment near struct mmu_table_batch.
344	*/
345	smp_call_function(tlb_remove_table_smp_sync, NULL, 1);
346	__tlb_remove_table(table);
347	}
348
349	static void tlb_remove_table_rcu(struct rcu_head *head)
350	{
351	struct mmu_table_batch *batch;
352	int i;
353
354	batch = container_of(head, struct mmu_table_batch, rcu);
355
356	for (i = 0; i < batch->nr; i++)
357	__tlb_remove_table(batch->tables[i]);
358
359	free_page((unsigned long)batch);
360	}
361
362	void tlb_table_flush(struct mmu_gather *tlb)
363	{
364	struct mmu_table_batch **batch = &tlb->batch;
365
366	if (*batch) {
367	call_rcu_sched(&(*batch)->rcu, tlb_remove_table_rcu);
368	*batch = NULL;
369	}
370	}
371
372	void tlb_remove_table(struct mmu_gather *tlb, void *table)
373	{
374	struct mmu_table_batch **batch = &tlb->batch;
375
376	if (*batch == NULL) {
377	*batch = (struct mmu_table_batch *)__get_free_page(GFP_NOWAIT | __GFP_NOWARN);
378	if (*batch == NULL) {
379	tlb_remove_table_one(table);
380	return;
381	}
382	(*batch)->nr = 0;
383	}
384	(*batch)->tables[(*batch)->nr++] = table;
385	if ((*batch)->nr == MAX_TABLE_BATCH)
386	tlb_table_flush(tlb);
387	}
388
389	#endif /* CONFIG_HAVE_RCU_TABLE_FREE */
390
391	/*
392	* Note: this doesn't free the actual pages themselves. That
393	* has been handled earlier when unmapping all the memory regions.
394	*/
395	static void free_pte_range(struct mmu_gather *tlb, pmd_t *pmd,
396	unsigned long addr)
397	{
398	pgtable_t token = pmd_pgtable(*pmd);
399	pmd_clear(pmd);
400	pte_free_tlb(tlb, token, addr);
401	atomic_long_dec(&tlb->mm->nr_ptes);
402	}
403
404	static inline void free_pmd_range(struct mmu_gather *tlb, pud_t *pud,
405	unsigned long addr, unsigned long end,
406	unsigned long floor, unsigned long ceiling)
407	{
408	pmd_t *pmd;
409	unsigned long next;
410	unsigned long start;
411
412	start = addr;
413	pmd = pmd_offset(pud, addr);
414	do {
415	next = pmd_addr_end(addr, end);
416	if (pmd_none_or_clear_bad(pmd))
417	continue;
418	free_pte_range(tlb, pmd, addr);
419	} while (pmd++, addr = next, addr != end);
420
421	start &= PUD_MASK;
422	if (start < floor)
423	return;
424	if (ceiling) {
425	ceiling &= PUD_MASK;
426	if (!ceiling)
427	return;
428	}
429	if (end - 1 > ceiling - 1)
430	return;
431
432	pmd = pmd_offset(pud, start);
433	pud_clear(pud);
434	pmd_free_tlb(tlb, pmd, start);
435	mm_dec_nr_pmds(tlb->mm);
436	}
437
438	static inline void free_pud_range(struct mmu_gather *tlb, pgd_t *pgd,
439	unsigned long addr, unsigned long end,
440	unsigned long floor, unsigned long ceiling)
441	{
442	pud_t *pud;
443	unsigned long next;
444	unsigned long start;
445
446	start = addr;
447	pud = pud_offset(pgd, addr);
448	do {
449	next = pud_addr_end(addr, end);
450	if (pud_none_or_clear_bad(pud))
451	continue;
452	free_pmd_range(tlb, pud, addr, next, floor, ceiling);
453	} while (pud++, addr = next, addr != end);
454
455	start &= PGDIR_MASK;
456	if (start < floor)
457	return;
458	if (ceiling) {
459	ceiling &= PGDIR_MASK;
460	if (!ceiling)
461	return;
462	}
463	if (end - 1 > ceiling - 1)
464	return;
465
466	pud = pud_offset(pgd, start);
467	pgd_clear(pgd);
468	pud_free_tlb(tlb, pud, start);
469	}
470
471	/*
472	* This function frees user-level page tables of a process.
473	*/
474	void free_pgd_range(struct mmu_gather *tlb,
475	unsigned long addr, unsigned long end,
476	unsigned long floor, unsigned long ceiling)
477	{
478	pgd_t *pgd;
479	unsigned long next;
480
481	/*
482	* The next few lines have given us lots of grief...
483	*
484	* Why are we testing PMD* at this top level? Because often
485	* there will be no work to do at all, and we'd prefer not to
486	* go all the way down to the bottom just to discover that.
487	*
488	* Why all these "- 1"s? Because 0 represents both the bottom
489	* of the address space and the top of it (using -1 for the
490	* top wouldn't help much: the masks would do the wrong thing).
491	* The rule is that addr 0 and floor 0 refer to the bottom of
492	* the address space, but end 0 and ceiling 0 refer to the top
493	* Comparisons need to use "end - 1" and "ceiling - 1" (though
494	* that end 0 case should be mythical).
495	*
496	* Wherever addr is brought up or ceiling brought down, we must
497	* be careful to reject "the opposite 0" before it confuses the
498	* subsequent tests. But what about where end is brought down
499	* by PMD_SIZE below? no, end can't go down to 0 there.
500	*
501	* Whereas we round start (addr) and ceiling down, by different
502	* masks at different levels, in order to test whether a table
503	* now has no other vmas using it, so can be freed, we don't
504	* bother to round floor or end up - the tests don't need that.
505	*/
506
507	addr &= PMD_MASK;
508	if (addr < floor) {
509	addr += PMD_SIZE;
510	if (!addr)
511	return;
512	}
513	if (ceiling) {
514	ceiling &= PMD_MASK;
515	if (!ceiling)
516	return;
517	}
518	if (end - 1 > ceiling - 1)
519	end -= PMD_SIZE;
520	if (addr > end - 1)
521	return;
522
523	pgd = pgd_offset(tlb->mm, addr);
524	do {
525	next = pgd_addr_end(addr, end);
526	if (pgd_none_or_clear_bad(pgd))
527	continue;
528	free_pud_range(tlb, pgd, addr, next, floor, ceiling);
529	} while (pgd++, addr = next, addr != end);
530	}
531
532	void free_pgtables(struct mmu_gather *tlb, struct vm_area_struct *vma,
533	unsigned long floor, unsigned long ceiling)
534	{
535	while (vma) {
536	struct vm_area_struct *next = vma->vm_next;
537	unsigned long addr = vma->vm_start;
538
539	/*
540	* Hide vma from rmap and truncate_pagecache before freeing
541	* pgtables
542	*/
543	unlink_anon_vmas(vma);
544	unlink_file_vma(vma);
545
546	if (is_vm_hugetlb_page(vma)) {
547	hugetlb_free_pgd_range(tlb, addr, vma->vm_end,
548	floor, next? next->vm_start: ceiling);
549	} else {
550	/*
551	* Optimization: gather nearby vmas into one call down
552	*/
553	while (next && next->vm_start <= vma->vm_end + PMD_SIZE
554	&& !is_vm_hugetlb_page(next)) {
555	vma = next;
556	next = vma->vm_next;
557	unlink_anon_vmas(vma);
558	unlink_file_vma(vma);
559	}
560	free_pgd_range(tlb, addr, vma->vm_end,
561	floor, next? next->vm_start: ceiling);
562	}
563	vma = next;
564	}
565	}
566
567	int __pte_alloc(struct mm_struct *mm, pmd_t *pmd, unsigned long address)
568	{
569	spinlock_t *ptl;
570	pgtable_t new = pte_alloc_one(mm, address);
571	if (!new)
572	return -ENOMEM;
573
574	/*
575	* Ensure all pte setup (eg. pte page lock and page clearing) are
576	* visible before the pte is made visible to other CPUs by being
577	* put into page tables.
578	*
579	* The other side of the story is the pointer chasing in the page
580	* table walking code (when walking the page table without locking;
581	* ie. most of the time). Fortunately, these data accesses consist
582	* of a chain of data-dependent loads, meaning most CPUs (alpha
583	* being the notable exception) will already guarantee loads are
584	* seen in-order. See the alpha page table accessors for the
585	* smp_read_barrier_depends() barriers in page table walking code.
586	*/
587	smp_wmb(); /* Could be smp_wmb__xxx(before|after)_spin_lock */
588
589	ptl = pmd_lock(mm, pmd);
590	if (likely(pmd_none(*pmd))) { /* Has another populated it ? */
591	atomic_long_inc(&mm->nr_ptes);
592	pmd_populate(mm, pmd, new);
593	new = NULL;
594	}
595	spin_unlock(ptl);
596	if (new)
597	pte_free(mm, new);
598	return 0;
599	}
600
601	int __pte_alloc_kernel(pmd_t *pmd, unsigned long address)
602	{
603	pte_t *new = pte_alloc_one_kernel(&init_mm, address);
604	if (!new)
605	return -ENOMEM;
606
607	smp_wmb(); /* See comment in __pte_alloc */
608
609	spin_lock(&init_mm.page_table_lock);
610	if (likely(pmd_none(*pmd))) { /* Has another populated it ? */
611	pmd_populate_kernel(&init_mm, pmd, new);
612	new = NULL;
613	}
614	spin_unlock(&init_mm.page_table_lock);
615	if (new)
616	pte_free_kernel(&init_mm, new);
617	return 0;
618	}
619
620	static inline void init_rss_vec(int *rss)
621	{
622	memset(rss, 0, sizeof(int) * NR_MM_COUNTERS);
623	}
624
625	static inline void add_mm_rss_vec(struct mm_struct *mm, int *rss)
626	{
627	int i;
628
629	if (current->mm == mm)
630	sync_mm_rss(mm);
631	for (i = 0; i < NR_MM_COUNTERS; i++)
632	if (rss[i])
633	add_mm_counter(mm, i, rss[i]);
634	}
635
636	/*
637	* This function is called to print an error when a bad pte
638	* is found. For example, we might have a PFN-mapped pte in
639	* a region that doesn't allow it.
640	*
641	* The calling function must still handle the error.
642	*/
643	static void print_bad_pte(struct vm_area_struct *vma, unsigned long addr,
644	pte_t pte, struct page *page)
645	{
646	pgd_t *pgd = pgd_offset(vma->vm_mm, addr);
647	pud_t *pud = pud_offset(pgd, addr);
648	pmd_t *pmd = pmd_offset(pud, addr);
649	struct address_space *mapping;
650	pgoff_t index;
651	static unsigned long resume;
652	static unsigned long nr_shown;
653	static unsigned long nr_unshown;
654
655	/*
656	* Allow a burst of 60 reports, then keep quiet for that minute;
657	* or allow a steady drip of one report per second.
658	*/
659	if (nr_shown == 60) {
660	if (time_before(jiffies, resume)) {
661	nr_unshown++;
662	return;
663	}
664	if (nr_unshown) {
665	pr_alert("BUG: Bad page map: %lu messages suppressed\n",
666	nr_unshown);
667	nr_unshown = 0;
668	}
669	nr_shown = 0;
670	}
671	if (nr_shown++ == 0)
672	resume = jiffies + 60 * HZ;
673
674	mapping = vma->vm_file ? vma->vm_file->f_mapping : NULL;
675	index = linear_page_index(vma, addr);
676
677	pr_alert("BUG: Bad page map in process %s pte:%08llx pmd:%08llx\n",
678	current->comm,
679	(long long)pte_val(pte), (long long)pmd_val(*pmd));
680	if (page)
681	dump_page(page, "bad pte");
682	pr_alert("addr:%p vm_flags:%08lx anon_vma:%p mapping:%p index:%lx\n",
683	(void *)addr, vma->vm_flags, vma->anon_vma, mapping, index);
684	/*
685	* Choose text because data symbols depend on CONFIG_KALLSYMS_ALL=y
686	*/
687	pr_alert("file:%pD fault:%pf mmap:%pf readpage:%pf\n",
688	vma->vm_file,
689	vma->vm_ops ? vma->vm_ops->fault : NULL,
690	vma->vm_file ? vma->vm_file->f_op->mmap : NULL,
691	mapping ? mapping->a_ops->readpage : NULL);
692	dump_stack();
693	add_taint(TAINT_BAD_PAGE, LOCKDEP_NOW_UNRELIABLE);
694	}
695
696	/*
697	* vm_normal_page -- This function gets the "struct page" associated with a pte.
698	*
699	* "Special" mappings do not wish to be associated with a "struct page" (either
700	* it doesn't exist, or it exists but they don't want to touch it). In this
701	* case, NULL is returned here. "Normal" mappings do have a struct page.
702	*
703	* There are 2 broad cases. Firstly, an architecture may define a pte_special()
704	* pte bit, in which case this function is trivial. Secondly, an architecture
705	* may not have a spare pte bit, which requires a more complicated scheme,
706	* described below.
707	*
708	* A raw VM_PFNMAP mapping (ie. one that is not COWed) is always considered a
709	* special mapping (even if there are underlying and valid "struct pages").
710	* COWed pages of a VM_PFNMAP are always normal.
711	*
712	* The way we recognize COWed pages within VM_PFNMAP mappings is through the
713	* rules set up by "remap_pfn_range()": the vma will have the VM_PFNMAP bit
714	* set, and the vm_pgoff will point to the first PFN mapped: thus every special
715	* mapping will always honor the rule
716	*
717	* pfn_of_page == vma->vm_pgoff + ((addr - vma->vm_start) >> PAGE_SHIFT)
718	*
719	* And for normal mappings this is false.
720	*
721	* This restricts such mappings to be a linear translation from virtual address
722	* to pfn. To get around this restriction, we allow arbitrary mappings so long
723	* as the vma is not a COW mapping; in that case, we know that all ptes are
724	* special (because none can have been COWed).
725	*
726	*
727	* In order to support COW of arbitrary special mappings, we have VM_MIXEDMAP.
728	*
729	* VM_MIXEDMAP mappings can likewise contain memory with or without "struct
730	* page" backing, however the difference is that _all_ pages with a struct
731	* page (that is, those where pfn_valid is true) are refcounted and considered
732	* normal pages by the VM. The disadvantage is that pages are refcounted
733	* (which can be slower and simply not an option for some PFNMAP users). The
734	* advantage is that we don't have to follow the strict linearity rule of
735	* PFNMAP mappings in order to support COWable mappings.
736	*
737	*/
738	#ifdef __HAVE_ARCH_PTE_SPECIAL
739	# define HAVE_PTE_SPECIAL 1
740	#else
741	# define HAVE_PTE_SPECIAL 0
742	#endif
743	struct page *vm_normal_page(struct vm_area_struct *vma, unsigned long addr,
744	pte_t pte)
745	{
746	unsigned long pfn = pte_pfn(pte);
747
748	if (HAVE_PTE_SPECIAL) {
749	if (likely(!pte_special(pte)))
750	goto check_pfn;
751	if (vma->vm_ops && vma->vm_ops->find_special_page)
752	return vma->vm_ops->find_special_page(vma, addr);
753	if (vma->vm_flags & (VM_PFNMAP | VM_MIXEDMAP))
754	return NULL;
755	if (!is_zero_pfn(pfn))
756	print_bad_pte(vma, addr, pte, NULL);
757	return NULL;
758	}
759
760	/* !HAVE_PTE_SPECIAL case follows: */
761
762	if (unlikely(vma->vm_flags & (VM_PFNMAP|VM_MIXEDMAP))) {
763	if (vma->vm_flags & VM_MIXEDMAP) {
764	if (!pfn_valid(pfn))
765	return NULL;
766	goto out;
767	} else {
768	unsigned long off;
769	off = (addr - vma->vm_start) >> PAGE_SHIFT;
770	if (pfn == vma->vm_pgoff + off)
771	return NULL;
772	if (!is_cow_mapping(vma->vm_flags))
773	return NULL;
774	}
775	}
776
777	if (is_zero_pfn(pfn))
778	return NULL;
779	check_pfn:
780	if (unlikely(pfn > highest_memmap_pfn)) {
781	print_bad_pte(vma, addr, pte, NULL);
782	return NULL;
783	}
784
785	/*
786	* NOTE! We still have PageReserved() pages in the page tables.
787	* eg. VDSO mappings can cause them to exist.
788	*/
789	out:
790	return pfn_to_page(pfn);
791	}
792
793	#ifdef CONFIG_TRANSPARENT_HUGEPAGE
794	struct page *vm_normal_page_pmd(struct vm_area_struct *vma, unsigned long addr,
795	pmd_t pmd)
796	{
797	unsigned long pfn = pmd_pfn(pmd);
798
799	/*
800	* There is no pmd_special() but there may be special pmds, e.g.
801	* in a direct-access (dax) mapping, so let's just replicate the
802	* !HAVE_PTE_SPECIAL case from vm_normal_page() here.
803	*/
804	if (unlikely(vma->vm_flags & (VM_PFNMAP|VM_MIXEDMAP))) {
805	if (vma->vm_flags & VM_MIXEDMAP) {
806	if (!pfn_valid(pfn))
807	return NULL;
808	goto out;
809	} else {
810	unsigned long off;
811	off = (addr - vma->vm_start) >> PAGE_SHIFT;
812	if (pfn == vma->vm_pgoff + off)
813	return NULL;
814	if (!is_cow_mapping(vma->vm_flags))
815	return NULL;
816	}
817	}
818
819	if (is_zero_pfn(pfn))
820	return NULL;
821	if (unlikely(pfn > highest_memmap_pfn))
822	return NULL;
823
824	/*
825	* NOTE! We still have PageReserved() pages in the page tables.
826	* eg. VDSO mappings can cause them to exist.
827	*/
828	out:
829	return pfn_to_page(pfn);
830	}
831	#endif
832
833	/*
834	* copy one vm_area from one task to the other. Assumes the page tables
835	* already present in the new task to be cleared in the whole range
836	* covered by this vma.
837	*/
838
839	static inline unsigned long
840	copy_one_pte(struct mm_struct *dst_mm, struct mm_struct *src_mm,
841	pte_t *dst_pte, pte_t *src_pte, struct vm_area_struct *vma,
842	unsigned long addr, int *rss)
843	{
844	unsigned long vm_flags = vma->vm_flags;
845	pte_t pte = *src_pte;
846	struct page *page;
847
848	/* pte contains position in swap or file, so copy. */
849	if (unlikely(!pte_present(pte))) {
850	swp_entry_t entry = pte_to_swp_entry(pte);
851
852	if (likely(!non_swap_entry(entry))) {
853	if (swap_duplicate(entry) < 0)
854	return entry.val;
855
856	/* make sure dst_mm is on swapoff's mmlist. */
857	if (unlikely(list_empty(&dst_mm->mmlist))) {
858	spin_lock(&mmlist_lock);
859	if (list_empty(&dst_mm->mmlist))
860	list_add(&dst_mm->mmlist,
861	&src_mm->mmlist);
862	spin_unlock(&mmlist_lock);
863	}
864	rss[MM_SWAPENTS]++;
865	} else if (is_migration_entry(entry)) {
866	page = migration_entry_to_page(entry);
867
868	rss[mm_counter(page)]++;
869
870	if (is_write_migration_entry(entry) &&
871	is_cow_mapping(vm_flags)) {
872	/*
873	* COW mappings require pages in both
874	* parent and child to be set to read.
875	*/
876	make_migration_entry_read(&entry);
877	pte = swp_entry_to_pte(entry);
878	if (pte_swp_soft_dirty(*src_pte))
879	pte = pte_swp_mksoft_dirty(pte);
880	set_pte_at(src_mm, addr, src_pte, pte);
881	}
882	}
883	goto out_set_pte;
884	}
885
886	/*
887	* If it's a COW mapping, write protect it both
888	* in the parent and the child
889	*/
890	if (is_cow_mapping(vm_flags)) {
891	ptep_set_wrprotect(src_mm, addr, src_pte);
892	pte = pte_wrprotect(pte);
893	}
894
895	/*
896	* If it's a shared mapping, mark it clean in
897	* the child
898	*/
899	if (vm_flags & VM_SHARED)
900	pte = pte_mkclean(pte);
901	pte = pte_mkold(pte);
902
903	page = vm_normal_page(vma, addr, pte);
904	if (page) {
905	get_page(page);
906	page_dup_rmap(page, false);
907	rss[mm_counter(page)]++;
908	}
909
910	out_set_pte:
911	set_pte_at(dst_mm, addr, dst_pte, pte);
912	return 0;
913	}
914
915	static int copy_pte_range(struct mm_struct *dst_mm, struct mm_struct *src_mm,
916	pmd_t *dst_pmd, pmd_t *src_pmd, struct vm_area_struct *vma,
917	unsigned long addr, unsigned long end)
918	{
919	pte_t *orig_src_pte, *orig_dst_pte;
920	pte_t *src_pte, *dst_pte;
921	spinlock_t *src_ptl, *dst_ptl;
922	int progress = 0;
923	int rss[NR_MM_COUNTERS];
924	swp_entry_t entry = (swp_entry_t){0};
925
926	again:
927	init_rss_vec(rss);
928
929	dst_pte = pte_alloc_map_lock(dst_mm, dst_pmd, addr, &dst_ptl);
930	if (!dst_pte)
931	return -ENOMEM;
932	src_pte = pte_offset_map(src_pmd, addr);
933	src_ptl = pte_lockptr(src_mm, src_pmd);
934	spin_lock_nested(src_ptl, SINGLE_DEPTH_NESTING);
935	orig_src_pte = src_pte;
936	orig_dst_pte = dst_pte;
937	arch_enter_lazy_mmu_mode();
938
939	do {
940	/*
941	* We are holding two locks at this point - either of them
942	* could generate latencies in another task on another CPU.
943	*/
944	if (progress >= 32) {
945	progress = 0;
946	if (need_resched() ||
947	spin_needbreak(src_ptl) || spin_needbreak(dst_ptl))
948	break;
949	}
950	if (pte_none(*src_pte)) {
951	progress++;
952	continue;
953	}
954	entry.val = copy_one_pte(dst_mm, src_mm, dst_pte, src_pte,
955	vma, addr, rss);
956	if (entry.val)
957	break;
958	progress += 8;
959	} while (dst_pte++, src_pte++, addr += PAGE_SIZE, addr != end);
960
961	arch_leave_lazy_mmu_mode();
962	spin_unlock(src_ptl);
963	pte_unmap(orig_src_pte);
964	add_mm_rss_vec(dst_mm, rss);
965	pte_unmap_unlock(orig_dst_pte, dst_ptl);
966	cond_resched();
967
968	if (entry.val) {
969	if (add_swap_count_continuation(entry, GFP_KERNEL) < 0)
970	return -ENOMEM;
971	progress = 0;
972	}
973	if (addr != end)
974	goto again;
975	return 0;
976	}
977
978	static inline int copy_pmd_range(struct mm_struct *dst_mm, struct mm_struct *src_mm,
979	pud_t *dst_pud, pud_t *src_pud, struct vm_area_struct *vma,
980	unsigned long addr, unsigned long end)
981	{
982	pmd_t *src_pmd, *dst_pmd;
983	unsigned long next;
984
985	dst_pmd = pmd_alloc(dst_mm, dst_pud, addr);
986	if (!dst_pmd)
987	return -ENOMEM;
988	src_pmd = pmd_offset(src_pud, addr);
989	do {
990	next = pmd_addr_end(addr, end);
991	if (pmd_trans_huge(*src_pmd) || pmd_devmap(*src_pmd)) {
992	int err;
993	VM_BUG_ON(next-addr != HPAGE_PMD_SIZE);
994	err = copy_huge_pmd(dst_mm, src_mm,
995	dst_pmd, src_pmd, addr, vma);
996	if (err == -ENOMEM)
997	return -ENOMEM;
998	if (!err)
999	continue;
1000	/* fall through */
1001	}
1002	if (pmd_none_or_clear_bad(src_pmd))
1003	continue;
1004	if (copy_pte_range(dst_mm, src_mm, dst_pmd, src_pmd,
1005	vma, addr, next))
1006	return -ENOMEM;
1007	} while (dst_pmd++, src_pmd++, addr = next, addr != end);
1008	return 0;
1009	}
1010
1011	static inline int copy_pud_range(struct mm_struct *dst_mm, struct mm_struct *src_mm,
1012	pgd_t *dst_pgd, pgd_t *src_pgd, struct vm_area_struct *vma,
1013	unsigned long addr, unsigned long end)
1014	{
1015	pud_t *src_pud, *dst_pud;
1016	unsigned long next;
1017
1018	dst_pud = pud_alloc(dst_mm, dst_pgd, addr);
1019	if (!dst_pud)
1020	return -ENOMEM;
1021	src_pud = pud_offset(src_pgd, addr);
1022	do {
1023	next = pud_addr_end(addr, end);
1024	if (pud_none_or_clear_bad(src_pud))
1025	continue;
1026	if (copy_pmd_range(dst_mm, src_mm, dst_pud, src_pud,
1027	vma, addr, next))
1028	return -ENOMEM;
1029	} while (dst_pud++, src_pud++, addr = next, addr != end);
1030	return 0;
1031	}
1032
1033	int copy_page_range(struct mm_struct *dst_mm, struct mm_struct *src_mm,
1034	struct vm_area_struct *vma)
1035	{
1036	pgd_t *src_pgd, *dst_pgd;
1037	unsigned long next;
1038	unsigned long addr = vma->vm_start;
1039	unsigned long end = vma->vm_end;
1040	unsigned long mmun_start; /* For mmu_notifiers */
1041	unsigned long mmun_end; /* For mmu_notifiers */
1042	bool is_cow;
1043	int ret;
1044
1045	/*
1046	* Don't copy ptes where a page fault will fill them correctly.
1047	* Fork becomes much lighter when there are big shared or private
1048	* readonly mappings. The tradeoff is that copy_page_range is more
1049	* efficient than faulting.
1050	*/
1051	if (!(vma->vm_flags & (VM_HUGETLB | VM_PFNMAP | VM_MIXEDMAP)) &&
1052	!vma->anon_vma)
1053	return 0;
1054
1055	if (is_vm_hugetlb_page(vma))
1056	return copy_hugetlb_page_range(dst_mm, src_mm, vma);
1057
1058	if (unlikely(vma->vm_flags & VM_PFNMAP)) {
1059	/*
1060	* We do not free on error cases below as remove_vma
1061	* gets called on error from higher level routine
1062	*/
1063	ret = track_pfn_copy(vma);
1064	if (ret)
1065	return ret;
1066	}
1067
1068	/*
1069	* We need to invalidate the secondary MMU mappings only when
1070	* there could be a permission downgrade on the ptes of the
1071	* parent mm. And a permission downgrade will only happen if
1072	* is_cow_mapping() returns true.
1073	*/
1074	is_cow = is_cow_mapping(vma->vm_flags);
1075	mmun_start = addr;
1076	mmun_end = end;
1077	if (is_cow)
1078	mmu_notifier_invalidate_range_start(src_mm, mmun_start,
1079	mmun_end);
1080
1081	ret = 0;
1082	dst_pgd = pgd_offset(dst_mm, addr);
1083	src_pgd = pgd_offset(src_mm, addr);
1084	do {
1085	next = pgd_addr_end(addr, end);
1086	if (pgd_none_or_clear_bad(src_pgd))
1087	continue;
1088	if (unlikely(copy_pud_range(dst_mm, src_mm, dst_pgd, src_pgd,
1089	vma, addr, next))) {
1090	ret = -ENOMEM;
1091	break;
1092	}
1093	} while (dst_pgd++, src_pgd++, addr = next, addr != end);
1094
1095	if (is_cow)
1096	mmu_notifier_invalidate_range_end(src_mm, mmun_start, mmun_end);
1097	return ret;
1098	}
1099
1100	static unsigned long zap_pte_range(struct mmu_gather *tlb,
1101	struct vm_area_struct *vma, pmd_t *pmd,
1102	unsigned long addr, unsigned long end,
1103	struct zap_details *details)
1104	{
1105	struct mm_struct *mm = tlb->mm;
1106	int force_flush = 0;
1107	int rss[NR_MM_COUNTERS];
1108	spinlock_t *ptl;
1109	pte_t *start_pte;
1110	pte_t *pte;
1111	swp_entry_t entry;
1112	struct page *pending_page = NULL;
1113
1114	again:
1115	init_rss_vec(rss);
1116	start_pte = pte_offset_map_lock(mm, pmd, addr, &ptl);
1117	pte = start_pte;
1118	flush_tlb_batched_pending(mm);
1119	arch_enter_lazy_mmu_mode();
1120	do {
1121	pte_t ptent = *pte;
1122	if (pte_none(ptent)) {
1123	continue;
1124	}
1125
1126	if (pte_present(ptent)) {
1127	struct page *page;
1128
1129	page = vm_normal_page(vma, addr, ptent);
1130	if (unlikely(details) && page) {
1131	/*
1132	* unmap_shared_mapping_pages() wants to
1133	* invalidate cache without truncating:
1134	* unmap shared but keep private pages.
1135	*/
1136	if (details->check_mapping &&
1137	details->check_mapping != page_rmapping(page))
1138	continue;
1139	}
1140	ptent = ptep_get_and_clear_full(mm, addr, pte,
1141	tlb->fullmm);
1142	tlb_remove_tlb_entry(tlb, pte, addr);
1143	if (unlikely(!page))
1144	continue;
1145
1146	if (!PageAnon(page)) {
1147	if (pte_dirty(ptent)) {
1148	/*
1149	* oom_reaper cannot tear down dirty
1150	* pages
1151	*/
1152	if (unlikely(details && details->ignore_dirty))
1153	continue;
1154	force_flush = 1;
1155	set_page_dirty(page);
1156	}
1157	if (pte_young(ptent) &&
1158	likely(!(vma->vm_flags & VM_SEQ_READ)))
1159	mark_page_accessed(page);
1160	}
1161	rss[mm_counter(page)]--;
1162	page_remove_rmap(page, false);
1163	if (unlikely(page_mapcount(page) < 0))
1164	print_bad_pte(vma, addr, ptent, page);
1165	if (unlikely(__tlb_remove_page(tlb, page))) {
1166	force_flush = 1;
1167	pending_page = page;
1168	addr += PAGE_SIZE;
1169	break;
1170	}
1171	continue;
1172	}
1173	/* only check swap_entries if explicitly asked for in details */
1174	if (unlikely(details && !details->check_swap_entries))
1175	continue;
1176
1177	entry = pte_to_swp_entry(ptent);
1178	if (!non_swap_entry(entry))
1179	rss[MM_SWAPENTS]--;
1180	else if (is_migration_entry(entry)) {
1181	struct page *page;
1182
1183	page = migration_entry_to_page(entry);
1184	rss[mm_counter(page)]--;
1185	}
1186	if (unlikely(!free_swap_and_cache(entry)))
1187	print_bad_pte(vma, addr, ptent, NULL);
1188	pte_clear_not_present_full(mm, addr, pte, tlb->fullmm);
1189	} while (pte++, addr += PAGE_SIZE, addr != end);
1190
1191	add_mm_rss_vec(mm, rss);
1192	arch_leave_lazy_mmu_mode();
1193
1194	/* Do the actual TLB flush before dropping ptl */
1195	if (force_flush)
1196	tlb_flush_mmu_tlbonly(tlb);
1197	pte_unmap_unlock(start_pte, ptl);
1198
1199	/*
1200	* If we forced a TLB flush (either due to running out of
1201	* batch buffers or because we needed to flush dirty TLB
1202	* entries before releasing the ptl), free the batched
1203	* memory too. Restart if we didn't do everything.
1204	*/
1205	if (force_flush) {
1206	force_flush = 0;
1207	tlb_flush_mmu_free(tlb);
1208	if (pending_page) {
1209	/* remove the page with new size */
1210	__tlb_remove_pte_page(tlb, pending_page);
1211	pending_page = NULL;
1212	}
1213	if (addr != end)
1214	goto again;
1215	}
1216
1217	return addr;
1218	}
1219
1220	static inline unsigned long zap_pmd_range(struct mmu_gather *tlb,
1221	struct vm_area_struct *vma, pud_t *pud,
1222	unsigned long addr, unsigned long end,
1223	struct zap_details *details)
1224	{
1225	pmd_t *pmd;
1226	unsigned long next;
1227
1228	pmd = pmd_offset(pud, addr);
1229	do {
1230	next = pmd_addr_end(addr, end);
1231	if (pmd_trans_huge(*pmd) || pmd_devmap(*pmd)) {
1232	if (next - addr != HPAGE_PMD_SIZE) {
1233	VM_BUG_ON_VMA(vma_is_anonymous(vma) &&
1234	!rwsem_is_locked(&tlb->mm->mmap_sem), vma);
1235	split_huge_pmd(vma, pmd, addr);
1236	} else if (zap_huge_pmd(tlb, vma, pmd, addr))
1237	goto next;
1238	/* fall through */
1239	}
1240	/*
1241	* Here there can be other concurrent MADV_DONTNEED or
1242	* trans huge page faults running, and if the pmd is
1243	* none or trans huge it can change under us. This is
1244	* because MADV_DONTNEED holds the mmap_sem in read
1245	* mode.
1246	*/
1247	if (pmd_none_or_trans_huge_or_clear_bad(pmd))
1248	goto next;
1249	next = zap_pte_range(tlb, vma, pmd, addr, next, details);
1250	next:
1251	cond_resched();
1252	} while (pmd++, addr = next, addr != end);
1253
1254	return addr;
1255	}
1256
1257	static inline unsigned long zap_pud_range(struct mmu_gather *tlb,
1258	struct vm_area_struct *vma, pgd_t *pgd,
1259	unsigned long addr, unsigned long end,
1260	struct zap_details *details)
1261	{
1262	pud_t *pud;
1263	unsigned long next;
1264
1265	pud = pud_offset(pgd, addr);
1266	do {
1267	next = pud_addr_end(addr, end);
1268	if (pud_none_or_clear_bad(pud))
1269	continue;
1270	next = zap_pmd_range(tlb, vma, pud, addr, next, details);
1271	} while (pud++, addr = next, addr != end);
1272
1273	return addr;
1274	}
1275
1276	void unmap_page_range(struct mmu_gather *tlb,
1277	struct vm_area_struct *vma,
1278	unsigned long addr, unsigned long end,
1279	struct zap_details *details)
1280	{
1281	pgd_t *pgd;
1282	unsigned long next;
1283
1284	BUG_ON(addr >= end);
1285	tlb_start_vma(tlb, vma);
1286	pgd = pgd_offset(vma->vm_mm, addr);
1287	do {
1288	next = pgd_addr_end(addr, end);
1289	if (pgd_none_or_clear_bad(pgd))
1290	continue;
1291	next = zap_pud_range(tlb, vma, pgd, addr, next, details);
1292	} while (pgd++, addr = next, addr != end);
1293	tlb_end_vma(tlb, vma);
1294	}
1295
1296
1297	static void unmap_single_vma(struct mmu_gather *tlb,
1298	struct vm_area_struct *vma, unsigned long start_addr,
1299	unsigned long end_addr,
1300	struct zap_details *details)
1301	{
1302	unsigned long start = max(vma->vm_start, start_addr);
1303	unsigned long end;
1304
1305	if (start >= vma->vm_end)
1306	return;
1307	end = min(vma->vm_end, end_addr);
1308	if (end <= vma->vm_start)
1309	return;
1310
1311	if (vma->vm_file)
1312	uprobe_munmap(vma, start, end);
1313
1314	if (unlikely(vma->vm_flags & VM_PFNMAP))
1315	untrack_pfn(vma, 0, 0);
1316
1317	if (start != end) {
1318	if (unlikely(is_vm_hugetlb_page(vma))) {
1319	/*
1320	* It is undesirable to test vma->vm_file as it
1321	* should be non-null for valid hugetlb area.
1322	* However, vm_file will be NULL in the error
1323	* cleanup path of mmap_region. When
1324	* hugetlbfs ->mmap method fails,
1325	* mmap_region() nullifies vma->vm_file
1326	* before calling this function to clean up.
1327	* Since no pte has actually been setup, it is
1328	* safe to do nothing in this case.
1329	*/
1330	if (vma->vm_file) {
1331	i_mmap_lock_write(vma->vm_file->f_mapping);
1332	__unmap_hugepage_range_final(tlb, vma, start, end, NULL);
1333	i_mmap_unlock_write(vma->vm_file->f_mapping);
1334	}
1335	} else
1336	unmap_page_range(tlb, vma, start, end, details);
1337	}
1338	}
1339
1340	/**
1341	* unmap_vmas - unmap a range of memory covered by a list of vma's
1342	* @tlb: address of the caller's struct mmu_gather
1343	* @vma: the starting vma
1344	* @start_addr: virtual address at which to start unmapping
1345	* @end_addr: virtual address at which to end unmapping
1346	*
1347	* Unmap all pages in the vma list.
1348	*
1349	* Only addresses between `start' and `end' will be unmapped.
1350	*
1351	* The VMA list must be sorted in ascending virtual address order.
1352	*
1353	* unmap_vmas() assumes that the caller will flush the whole unmapped address
1354	* range after unmap_vmas() returns. So the only responsibility here is to
1355	* ensure that any thus-far unmapped pages are flushed before unmap_vmas()
1356	* drops the lock and schedules.
1357	*/
1358	void unmap_vmas(struct mmu_gather *tlb,
1359	struct vm_area_struct *vma, unsigned long start_addr,
1360	unsigned long end_addr)
1361	{
1362	struct mm_struct *mm = vma->vm_mm;
1363
1364	mmu_notifier_invalidate_range_start(mm, start_addr, end_addr);
1365	for ( ; vma && vma->vm_start < end_addr; vma = vma->vm_next)
1366	unmap_single_vma(tlb, vma, start_addr, end_addr, NULL);
1367	mmu_notifier_invalidate_range_end(mm, start_addr, end_addr);
1368	}
1369
1370	/**
1371	* zap_page_range - remove user pages in a given range
1372	* @vma: vm_area_struct holding the applicable pages
1373	* @start: starting address of pages to zap
1374	* @size: number of bytes to zap
1375	* @details: details of shared cache invalidation
1376	*
1377	* Caller must protect the VMA list
1378	*/
1379	void zap_page_range(struct vm_area_struct *vma, unsigned long start,
1380	unsigned long size, struct zap_details *details)
1381	{
1382	struct mm_struct *mm = vma->vm_mm;
1383	struct mmu_gather tlb;
1384	unsigned long end = start + size;
1385
1386	lru_add_drain();
1387	tlb_gather_mmu(&tlb, mm, start, end);
1388	update_hiwater_rss(mm);
1389	mmu_notifier_invalidate_range_start(mm, start, end);
1390	for ( ; vma && vma->vm_start < end; vma = vma->vm_next)
1391	unmap_single_vma(&tlb, vma, start, end, details);
1392	mmu_notifier_invalidate_range_end(mm, start, end);
1393	tlb_finish_mmu(&tlb, start, end);
1394	}
1395
1396	/**
1397	* zap_page_range_single - remove user pages in a given range
1398	* @vma: vm_area_struct holding the applicable pages
1399	* @address: starting address of pages to zap
1400	* @size: number of bytes to zap
1401	* @details: details of shared cache invalidation
1402	*
1403	* The range must fit into one VMA.
1404	*/
1405	static void zap_page_range_single(struct vm_area_struct *vma, unsigned long address,
1406	unsigned long size, struct zap_details *details)
1407	{
1408	struct mm_struct *mm = vma->vm_mm;
1409	struct mmu_gather tlb;
1410	unsigned long end = address + size;
1411
1412	lru_add_drain();
1413	tlb_gather_mmu(&tlb, mm, address, end);
1414	update_hiwater_rss(mm);
1415	mmu_notifier_invalidate_range_start(mm, address, end);
1416	unmap_single_vma(&tlb, vma, address, end, details);
1417	mmu_notifier_invalidate_range_end(mm, address, end);
1418	tlb_finish_mmu(&tlb, address, end);
1419	}
1420
1421	/**
1422	* zap_vma_ptes - remove ptes mapping the vma
1423	* @vma: vm_area_struct holding ptes to be zapped
1424	* @address: starting address of pages to zap
1425	* @size: number of bytes to zap
1426	*
1427	* This function only unmaps ptes assigned to VM_PFNMAP vmas.
1428	*
1429	* The entire address range must be fully contained within the vma.
1430	*
1431	* Returns 0 if successful.
1432	*/
1433	int zap_vma_ptes(struct vm_area_struct *vma, unsigned long address,
1434	unsigned long size)
1435	{
1436	if (address < vma->vm_start || address + size > vma->vm_end ||
1437	!(vma->vm_flags & VM_PFNMAP))
1438	return -1;
1439	zap_page_range_single(vma, address, size, NULL);
1440	return 0;
1441	}
1442	EXPORT_SYMBOL_GPL(zap_vma_ptes);
1443
1444	pte_t *__get_locked_pte(struct mm_struct *mm, unsigned long addr,
1445	spinlock_t **ptl)
1446	{
1447	pgd_t * pgd = pgd_offset(mm, addr);
1448	pud_t * pud = pud_alloc(mm, pgd, addr);
1449	if (pud) {
1450	pmd_t * pmd = pmd_alloc(mm, pud, addr);
1451	if (pmd) {
1452	VM_BUG_ON(pmd_trans_huge(*pmd));
1453	return pte_alloc_map_lock(mm, pmd, addr, ptl);
1454	}
1455	}
1456	return NULL;
1457	}
1458
1459	/*
1460	* This is the old fallback for page remapping.
1461	*
1462	* For historical reasons, it only allows reserved pages. Only
1463	* old drivers should use this, and they needed to mark their
1464	* pages reserved for the old functions anyway.
1465	*/
1466	static int insert_page(struct vm_area_struct *vma, unsigned long addr,
1467	struct page *page, pgprot_t prot)
1468	{
1469	struct mm_struct *mm = vma->vm_mm;
1470	int retval;
1471	pte_t *pte;
1472	spinlock_t *ptl;
1473
1474	retval = -EINVAL;
1475	if (PageAnon(page))
1476	goto out;
1477	retval = -ENOMEM;
1478	flush_dcache_page(page);
1479	pte = get_locked_pte(mm, addr, &ptl);
1480	if (!pte)
1481	goto out;
1482	retval = -EBUSY;
1483	if (!pte_none(*pte))
1484	goto out_unlock;
1485
1486	/* Ok, finally just insert the thing.. */
1487	get_page(page);
1488	inc_mm_counter_fast(mm, mm_counter_file(page));
1489	page_add_file_rmap(page, false);
1490	set_pte_at(mm, addr, pte, mk_pte(page, prot));
1491
1492	retval = 0;
1493	pte_unmap_unlock(pte, ptl);
1494	return retval;
1495	out_unlock:
1496	pte_unmap_unlock(pte, ptl);
1497	out:
1498	return retval;
1499	}
1500
1501	/**
1502	* vm_insert_page - insert single page into user vma
1503	* @vma: user vma to map to
1504	* @addr: target user address of this page
1505	* @page: source kernel page
1506	*
1507	* This allows drivers to insert individual pages they've allocated
1508	* into a user vma.
1509	*
1510	* The page has to be a nice clean _individual_ kernel allocation.
1511	* If you allocate a compound page, you need to have marked it as
1512	* such (__GFP_COMP), or manually just split the page up yourself
1513	* (see split_page()).
1514	*
1515	* NOTE! Traditionally this was done with "remap_pfn_range()" which
1516	* took an arbitrary page protection parameter. This doesn't allow
1517	* that. Your vma protection will have to be set up correctly, which
1518	* means that if you want a shared writable mapping, you'd better
1519	* ask for a shared writable mapping!
1520	*
1521	* The page does not need to be reserved.
1522	*
1523	* Usually this function is called from f_op->mmap() handler
1524	* under mm->mmap_sem write-lock, so it can change vma->vm_flags.
1525	* Caller must set VM_MIXEDMAP on vma if it wants to call this
1526	* function from other places, for example from page-fault handler.
1527	*/
1528	int vm_insert_page(struct vm_area_struct *vma, unsigned long addr,
1529	struct page *page)
1530	{
1531	if (addr < vma->vm_start || addr >= vma->vm_end)
1532	return -EFAULT;
1533	if (!page_count(page))
1534	return -EINVAL;
1535	if (!(vma->vm_flags & VM_MIXEDMAP)) {
1536	BUG_ON(down_read_trylock(&vma->vm_mm->mmap_sem));
1537	BUG_ON(vma->vm_flags & VM_PFNMAP);
1538	vma->vm_flags |= VM_MIXEDMAP;
1539	}
1540	return insert_page(vma, addr, page, vma->vm_page_prot);
1541	}
1542	EXPORT_SYMBOL(vm_insert_page);
1543
1544	static int insert_pfn(struct vm_area_struct *vma, unsigned long addr,
1545	pfn_t pfn, pgprot_t prot)
1546	{
1547	struct mm_struct *mm = vma->vm_mm;
1548	int retval;
1549	pte_t *pte, entry;
1550	spinlock_t *ptl;
1551
1552	retval = -ENOMEM;
1553	pte = get_locked_pte(mm, addr, &ptl);
1554	if (!pte)
1555	goto out;
1556	retval = -EBUSY;
1557	if (!pte_none(*pte))
1558	goto out_unlock;
1559
1560	/* Ok, finally just insert the thing.. */
1561	if (pfn_t_devmap(pfn))
1562	entry = pte_mkdevmap(pfn_t_pte(pfn, prot));
1563	else
1564	entry = pte_mkspecial(pfn_t_pte(pfn, prot));
1565	set_pte_at(mm, addr, pte, entry);
1566	update_mmu_cache(vma, addr, pte); /* XXX: why not for insert_page? */
1567
1568	retval = 0;
1569	out_unlock:
1570	pte_unmap_unlock(pte, ptl);
1571	out:
1572	return retval;
1573	}
1574
1575	/**
1576	* vm_insert_pfn - insert single pfn into user vma
1577	* @vma: user vma to map to
1578	* @addr: target user address of this page
1579	* @pfn: source kernel pfn
1580	*
1581	* Similar to vm_insert_page, this allows drivers to insert individual pages
1582	* they've allocated into a user vma. Same comments apply.
1583	*
1584	* This function should only be called from a vm_ops->fault handler, and
1585	* in that case the handler should return NULL.
1586	*
1587	* vma cannot be a COW mapping.
1588	*
1589	* As this is called only for pages that do not currently exist, we
1590	* do not need to flush old virtual caches or the TLB.
1591	*/
1592	int vm_insert_pfn(struct vm_area_struct *vma, unsigned long addr,
1593	unsigned long pfn)
1594	{
1595	return vm_insert_pfn_prot(vma, addr, pfn, vma->vm_page_prot);
1596	}
1597	EXPORT_SYMBOL(vm_insert_pfn);
1598
1599	/**
1600	* vm_insert_pfn_prot - insert single pfn into user vma with specified pgprot
1601	* @vma: user vma to map to
1602	* @addr: target user address of this page
1603	* @pfn: source kernel pfn
1604	* @pgprot: pgprot flags for the inserted page
1605	*
1606	* This is exactly like vm_insert_pfn, except that it allows drivers to
1607	* to override pgprot on a per-page basis.
1608	*
1609	* This only makes sense for IO mappings, and it makes no sense for
1610	* cow mappings. In general, using multiple vmas is preferable;
1611	* vm_insert_pfn_prot should only be used if using multiple VMAs is
1612	* impractical.
1613	*/
1614	int vm_insert_pfn_prot(struct vm_area_struct *vma, unsigned long addr,
1615	unsigned long pfn, pgprot_t pgprot)
1616	{
1617	int ret;
1618	/*
1619	* Technically, architectures with pte_special can avoid all these
1620	* restrictions (same for remap_pfn_range). However we would like
1621	* consistency in testing and feature parity among all, so we should
1622	* try to keep these invariants in place for everybody.
1623	*/
1624	BUG_ON(!(vma->vm_flags & (VM_PFNMAP|VM_MIXEDMAP)));
1625	BUG_ON((vma->vm_flags & (VM_PFNMAP|VM_MIXEDMAP)) ==
1626	(VM_PFNMAP|VM_MIXEDMAP));
1627	BUG_ON((vma->vm_flags & VM_PFNMAP) && is_cow_mapping(vma->vm_flags));
1628	BUG_ON((vma->vm_flags & VM_MIXEDMAP) && pfn_valid(pfn));
1629
1630	if (addr < vma->vm_start || addr >= vma->vm_end)
1631	return -EFAULT;
1632	if (track_pfn_insert(vma, &pgprot, __pfn_to_pfn_t(pfn, PFN_DEV)))
1633	return -EINVAL;
1634
1635	if (!pfn_modify_allowed(pfn, pgprot))
1636	return -EACCES;
1637
1638	ret = insert_pfn(vma, addr, __pfn_to_pfn_t(pfn, PFN_DEV), pgprot);
1639
1640	return ret;
1641	}
1642	EXPORT_SYMBOL(vm_insert_pfn_prot);
1643
1644	int vm_insert_mixed(struct vm_area_struct *vma, unsigned long addr,
1645	pfn_t pfn)
1646	{
1647	pgprot_t pgprot = vma->vm_page_prot;
1648
1649	BUG_ON(!(vma->vm_flags & VM_MIXEDMAP));
1650
1651	if (addr < vma->vm_start || addr >= vma->vm_end)
1652	return -EFAULT;
1653	if (track_pfn_insert(vma, &pgprot, pfn))
1654	return -EINVAL;
1655
1656	if (!pfn_modify_allowed(pfn_t_to_pfn(pfn), pgprot))
1657	return -EACCES;
1658
1659	/*
1660	* If we don't have pte special, then we have to use the pfn_valid()
1661	* based VM_MIXEDMAP scheme (see vm_normal_page), and thus we *must*
1662	* refcount the page if pfn_valid is true (hence insert_page rather
1663	* than insert_pfn). If a zero_pfn were inserted into a VM_MIXEDMAP
1664	* without pte special, it would there be refcounted as a normal page.
1665	*/
1666	if (!HAVE_PTE_SPECIAL && !pfn_t_devmap(pfn) && pfn_t_valid(pfn)) {
1667	struct page *page;
1668
1669	/*
1670	* At this point we are committed to insert_page()
1671	* regardless of whether the caller specified flags that
1672	* result in pfn_t_has_page() == false.
1673	*/
1674	page = pfn_to_page(pfn_t_to_pfn(pfn));
1675	return insert_page(vma, addr, page, pgprot);
1676	}
1677	return insert_pfn(vma, addr, pfn, pgprot);
1678	}
1679	EXPORT_SYMBOL(vm_insert_mixed);
1680
1681	/*
1682	* maps a range of physical memory into the requested pages. the old
1683	* mappings are removed. any references to nonexistent pages results
1684	* in null mappings (currently treated as "copy-on-access")
1685	*/
1686	static int remap_pte_range(struct mm_struct *mm, pmd_t *pmd,
1687	unsigned long addr, unsigned long end,
1688	unsigned long pfn, pgprot_t prot)
1689	{
1690	pte_t *pte;
1691	spinlock_t *ptl;
1692	int err = 0;
1693
1694	pte = pte_alloc_map_lock(mm, pmd, addr, &ptl);
1695	if (!pte)
1696	return -ENOMEM;
1697	arch_enter_lazy_mmu_mode();
1698	do {
1699	BUG_ON(!pte_none(*pte));
1700	if (!pfn_modify_allowed(pfn, prot)) {
1701	err = -EACCES;
1702	break;
1703	}
1704	set_pte_at(mm, addr, pte, pte_mkspecial(pfn_pte(pfn, prot)));
1705	pfn++;
1706	} while (pte++, addr += PAGE_SIZE, addr != end);
1707	arch_leave_lazy_mmu_mode();
1708	pte_unmap_unlock(pte - 1, ptl);
1709	return err;
1710	}
1711
1712	static inline int remap_pmd_range(struct mm_struct *mm, pud_t *pud,
1713	unsigned long addr, unsigned long end,
1714	unsigned long pfn, pgprot_t prot)
1715	{
1716	pmd_t *pmd;
1717	unsigned long next;
1718	int err;
1719
1720	pfn -= addr >> PAGE_SHIFT;
1721	pmd = pmd_alloc(mm, pud, addr);
1722	if (!pmd)
1723	return -ENOMEM;
1724	VM_BUG_ON(pmd_trans_huge(*pmd));
1725	do {
1726	next = pmd_addr_end(addr, end);
1727	err = remap_pte_range(mm, pmd, addr, next,
1728	pfn + (addr >> PAGE_SHIFT), prot);
1729	if (err)
1730	return err;
1731	} while (pmd++, addr = next, addr != end);
1732	return 0;
1733	}
1734
1735	static inline int remap_pud_range(struct mm_struct *mm, pgd_t *pgd,
1736	unsigned long addr, unsigned long end,
1737	unsigned long pfn, pgprot_t prot)
1738	{
1739	pud_t *pud;
1740	unsigned long next;
1741	int err;
1742
1743	pfn -= addr >> PAGE_SHIFT;
1744	pud = pud_alloc(mm, pgd, addr);
1745	if (!pud)
1746	return -ENOMEM;
1747	do {
1748	next = pud_addr_end(addr, end);
1749	err = remap_pmd_range(mm, pud, addr, next,
1750	pfn + (addr >> PAGE_SHIFT), prot);
1751	if (err)
1752	return err;
1753	} while (pud++, addr = next, addr != end);
1754	return 0;
1755	}
1756
1757	/**
1758	* remap_pfn_range - remap kernel memory to userspace
1759	* @vma: user vma to map to
1760	* @addr: target user address to start at
1761	* @pfn: physical address of kernel memory
1762	* @size: size of map area
1763	* @prot: page protection flags for this mapping
1764	*
1765	* Note: this is only safe if the mm semaphore is held when called.
1766	*/
1767	int remap_pfn_range(struct vm_area_struct *vma, unsigned long addr,
1768	unsigned long pfn, unsigned long size, pgprot_t prot)
1769	{
1770	pgd_t *pgd;
1771	unsigned long next;
1772	unsigned long end = addr + PAGE_ALIGN(size);
1773	struct mm_struct *mm = vma->vm_mm;
1774	unsigned long remap_pfn = pfn;
1775	int err;
1776
1777	/*
1778	* Physically remapped pages are special. Tell the
1779	* rest of the world about it:
1780	* VM_IO tells people not to look at these pages
1781	* (accesses can have side effects).
1782	* VM_PFNMAP tells the core MM that the base pages are just
1783	* raw PFN mappings, and do not have a "struct page" associated
1784	* with them.
1785	* VM_DONTEXPAND
1786	* Disable vma merging and expanding with mremap().
1787	* VM_DONTDUMP
1788	* Omit vma from core dump, even when VM_IO turned off.
1789	*
1790	* There's a horrible special case to handle copy-on-write
1791	* behaviour that some programs depend on. We mark the "original"
1792	* un-COW'ed pages by matching them up with "vma->vm_pgoff".
1793	* See vm_normal_page() for details.
1794	*/
1795	if (is_cow_mapping(vma->vm_flags)) {
1796	if (addr != vma->vm_start || end != vma->vm_end)
1797	return -EINVAL;
1798	vma->vm_pgoff = pfn;
1799	}
1800
1801	err = track_pfn_remap(vma, &prot, remap_pfn, addr, PAGE_ALIGN(size));
1802	if (err)
1803	return -EINVAL;
1804
1805	vma->vm_flags |= VM_IO | VM_PFNMAP | VM_DONTEXPAND | VM_DONTDUMP;
1806
1807	BUG_ON(addr >= end);
1808	pfn -= addr >> PAGE_SHIFT;
1809	pgd = pgd_offset(mm, addr);
1810	flush_cache_range(vma, addr, end);
1811	do {
1812	next = pgd_addr_end(addr, end);
1813	err = remap_pud_range(mm, pgd, addr, next,
1814	pfn + (addr >> PAGE_SHIFT), prot);
1815	if (err)
1816	break;
1817	} while (pgd++, addr = next, addr != end);
1818
1819	if (err)
1820	untrack_pfn(vma, remap_pfn, PAGE_ALIGN(size));
1821
1822	return err;
1823	}
1824	EXPORT_SYMBOL(remap_pfn_range);
1825
1826	/**
1827	* vm_iomap_memory - remap memory to userspace
1828	* @vma: user vma to map to
1829	* @start: start of area
1830	* @len: size of area
1831	*
1832	* This is a simplified io_remap_pfn_range() for common driver use. The
1833	* driver just needs to give us the physical memory range to be mapped,
1834	* we'll figure out the rest from the vma information.
1835	*
1836	* NOTE! Some drivers might want to tweak vma->vm_page_prot first to get
1837	* whatever write-combining details or similar.
1838	*/
1839	int vm_iomap_memory(struct vm_area_struct *vma, phys_addr_t start, unsigned long len)
1840	{
1841	unsigned long vm_len, pfn, pages;
1842
1843	/* Check that the physical memory area passed in looks valid */
1844	if (start + len < start)
1845	return -EINVAL;
1846	/*
1847	* You *really* shouldn't map things that aren't page-aligned,
1848	* but we've historically allowed it because IO memory might
1849	* just have smaller alignment.
1850	*/
1851	len += start & ~PAGE_MASK;
1852	pfn = start >> PAGE_SHIFT;
1853	pages = (len + ~PAGE_MASK) >> PAGE_SHIFT;
1854	if (pfn + pages < pfn)
1855	return -EINVAL;
1856
1857	/* We start the mapping 'vm_pgoff' pages into the area */
1858	if (vma->vm_pgoff > pages)
1859	return -EINVAL;
1860	pfn += vma->vm_pgoff;
1861	pages -= vma->vm_pgoff;
1862
1863	/* Can we fit all of the mapping? */
1864	vm_len = vma->vm_end - vma->vm_start;
1865	if (vm_len >> PAGE_SHIFT > pages)
1866	return -EINVAL;
1867
1868	/* Ok, let it rip */
1869	return io_remap_pfn_range(vma, vma->vm_start, pfn, vm_len, vma->vm_page_prot);
1870	}
1871	EXPORT_SYMBOL(vm_iomap_memory);
1872
1873	static int apply_to_pte_range(struct mm_struct *mm, pmd_t *pmd,
1874	unsigned long addr, unsigned long end,
1875	pte_fn_t fn, void *data)
1876	{
1877	pte_t *pte;
1878	int err;
1879	pgtable_t token;
1880	spinlock_t *uninitialized_var(ptl);
1881
1882	pte = (mm == &init_mm) ?
1883	pte_alloc_kernel(pmd, addr) :
1884	pte_alloc_map_lock(mm, pmd, addr, &ptl);
1885	if (!pte)
1886	return -ENOMEM;
1887
1888	BUG_ON(pmd_huge(*pmd));
1889
1890	arch_enter_lazy_mmu_mode();
1891
1892	token = pmd_pgtable(*pmd);
1893
1894	do {
1895	err = fn(pte++, token, addr, data);
1896	if (err)
1897	break;
1898	} while (addr += PAGE_SIZE, addr != end);
1899
1900	arch_leave_lazy_mmu_mode();
1901
1902	if (mm != &init_mm)
1903	pte_unmap_unlock(pte-1, ptl);
1904	return err;
1905	}
1906
1907	static int apply_to_pmd_range(struct mm_struct *mm, pud_t *pud,
1908	unsigned long addr, unsigned long end,
1909	pte_fn_t fn, void *data)
1910	{
1911	pmd_t *pmd;
1912	unsigned long next;
1913	int err;
1914
1915	BUG_ON(pud_huge(*pud));
1916
1917	pmd = pmd_alloc(mm, pud, addr);
1918	if (!pmd)
1919	return -ENOMEM;
1920	do {
1921	next = pmd_addr_end(addr, end);
1922	err = apply_to_pte_range(mm, pmd, addr, next, fn, data);
1923	if (err)
1924	break;
1925	} while (pmd++, addr = next, addr != end);
1926	return err;
1927	}
1928
1929	static int apply_to_pud_range(struct mm_struct *mm, pgd_t *pgd,
1930	unsigned long addr, unsigned long end,
1931	pte_fn_t fn, void *data)
1932	{
1933	pud_t *pud;
1934	unsigned long next;
1935	int err;
1936
1937	pud = pud_alloc(mm, pgd, addr);
1938	if (!pud)
1939	return -ENOMEM;
1940	do {
1941	next = pud_addr_end(addr, end);
1942	err = apply_to_pmd_range(mm, pud, addr, next, fn, data);
1943	if (err)
1944	break;
1945	} while (pud++, addr = next, addr != end);
1946	return err;
1947	}
1948
1949	/*
1950	* Scan a region of virtual memory, filling in page tables as necessary
1951	* and calling a provided function on each leaf page table.
1952	*/
1953	int apply_to_page_range(struct mm_struct *mm, unsigned long addr,
1954	unsigned long size, pte_fn_t fn, void *data)
1955	{
1956	pgd_t *pgd;
1957	unsigned long next;
1958	unsigned long end = addr + size;
1959	int err;
1960
1961	if (WARN_ON(addr >= end))
1962	return -EINVAL;
1963
1964	pgd = pgd_offset(mm, addr);
1965	do {
1966	next = pgd_addr_end(addr, end);
1967	err = apply_to_pud_range(mm, pgd, addr, next, fn, data);
1968	if (err)
1969	break;
1970	} while (pgd++, addr = next, addr != end);
1971
1972	return err;
1973	}
1974	EXPORT_SYMBOL_GPL(apply_to_page_range);
1975
1976	/*
1977	* handle_pte_fault chooses page fault handler according to an entry which was
1978	* read non-atomically. Before making any commitment, on those architectures
1979	* or configurations (e.g. i386 with PAE) which might give a mix of unmatched
1980	* parts, do_swap_page must check under lock before unmapping the pte and
1981	* proceeding (but do_wp_page is only called after already making such a check;
1982	* and do_anonymous_page can safely check later on).
1983	*/
1984	static inline int pte_unmap_same(struct mm_struct *mm, pmd_t *pmd,
1985	pte_t *page_table, pte_t orig_pte)
1986	{
1987	int same = 1;
1988	#if defined(CONFIG_SMP) || defined(CONFIG_PREEMPT)
1989	if (sizeof(pte_t) > sizeof(unsigned long)) {
1990	spinlock_t *ptl = pte_lockptr(mm, pmd);
1991	spin_lock(ptl);
1992	same = pte_same(*page_table, orig_pte);
1993	spin_unlock(ptl);
1994	}
1995	#endif
1996	pte_unmap(page_table);
1997	return same;
1998	}
1999
2000	static inline void cow_user_page(struct page *dst, struct page *src, unsigned long va, struct vm_area_struct *vma)
2001	{
2002	debug_dma_assert_idle(src);
2003
2004	/*
2005	* If the source page was a PFN mapping, we don't have
2006	* a "struct page" for it. We do a best-effort copy by
2007	* just copying from the original user address. If that
2008	* fails, we just zero-fill it. Live with it.
2009	*/
2010	if (unlikely(!src)) {
2011	void *kaddr = kmap_atomic(dst);
2012	void __user *uaddr = (void __user *)(va & PAGE_MASK);
2013
2014	/*
2015	* This really shouldn't fail, because the page is there
2016	* in the page tables. But it might just be unreadable,
2017	* in which case we just give up and fill the result with
2018	* zeroes.
2019	*/
2020	if (__copy_from_user_inatomic(kaddr, uaddr, PAGE_SIZE))
2021	clear_page(kaddr);
2022	kunmap_atomic(kaddr);
2023	flush_dcache_page(dst);
2024	} else
2025	copy_user_highpage(dst, src, va, vma);
2026	}
2027
2028	static gfp_t __get_fault_gfp_mask(struct vm_area_struct *vma)
2029	{
2030	struct file *vm_file = vma->vm_file;
2031
2032	if (vm_file)
2033	return mapping_gfp_mask(vm_file->f_mapping) | __GFP_FS | __GFP_IO;
2034
2035	/*
2036	* Special mappings (e.g. VDSO) do not have any file so fake
2037	* a default GFP_KERNEL for them.
2038	*/
2039	return GFP_KERNEL;
2040	}
2041
2042	/*
2043	* Notify the address space that the page is about to become writable so that
2044	* it can prohibit this or wait for the page to get into an appropriate state.
2045	*
2046	* We do this without the lock held, so that it can sleep if it needs to.
2047	*/
2048	static int do_page_mkwrite(struct vm_area_struct *vma, struct page *page,
2049	unsigned long address)
2050	{
2051	struct vm_fault vmf;
2052	int ret;
2053
2054	vmf.virtual_address = (void __user *)(address & PAGE_MASK);
2055	vmf.pgoff = page->index;
2056	vmf.flags = FAULT_FLAG_WRITE|FAULT_FLAG_MKWRITE;
2057	vmf.gfp_mask = __get_fault_gfp_mask(vma);
2058	vmf.page = page;
2059	vmf.cow_page = NULL;
2060
2061	ret = vma->vm_ops->page_mkwrite(vma, &vmf);
2062	if (unlikely(ret & (VM_FAULT_ERROR | VM_FAULT_NOPAGE)))
2063	return ret;
2064	if (unlikely(!(ret & VM_FAULT_LOCKED))) {
2065	lock_page(page);
2066	if (!page->mapping) {
2067	unlock_page(page);
2068	return 0; /* retry */
2069	}
2070	ret |= VM_FAULT_LOCKED;
2071	} else
2072	VM_BUG_ON_PAGE(!PageLocked(page), page);
2073	return ret;
2074	}
2075
2076	/*
2077	* Handle write page faults for pages that can be reused in the current vma
2078	*
2079	* This can happen either due to the mapping being with the VM_SHARED flag,
2080	* or due to us being the last reference standing to the page. In either
2081	* case, all we need to do here is to mark the page as writable and update
2082	* any related book-keeping.
2083	*/
2084	static inline int wp_page_reuse(struct fault_env *fe, pte_t orig_pte,
2085	struct page *page, int page_mkwrite, int dirty_shared)
2086	__releases(fe->ptl)
2087	{
2088	struct vm_area_struct *vma = fe->vma;
2089	pte_t entry;
2090	/*
2091	* Clear the pages cpupid information as the existing
2092	* information potentially belongs to a now completely
2093	* unrelated process.
2094	*/
2095	if (page)
2096	page_cpupid_xchg_last(page, (1 << LAST_CPUPID_SHIFT) - 1);
2097
2098	flush_cache_page(vma, fe->address, pte_pfn(orig_pte));
2099	entry = pte_mkyoung(orig_pte);
2100	entry = maybe_mkwrite(pte_mkdirty(entry), vma);
2101	if (ptep_set_access_flags(vma, fe->address, fe->pte, entry, 1))
2102	update_mmu_cache(vma, fe->address, fe->pte);
2103	pte_unmap_unlock(fe->pte, fe->ptl);
2104
2105	if (dirty_shared) {
2106	struct address_space *mapping;
2107	int dirtied;
2108
2109	if (!page_mkwrite)
2110	lock_page(page);
2111
2112	dirtied = set_page_dirty(page);
2113	VM_BUG_ON_PAGE(PageAnon(page), page);
2114	mapping = page->mapping;
2115	unlock_page(page);
2116	put_page(page);
2117
2118	if ((dirtied || page_mkwrite) && mapping) {
2119	/*
2120	* Some device drivers do not set page.mapping
2121	* but still dirty their pages
2122	*/
2123	balance_dirty_pages_ratelimited(mapping);
2124	}
2125
2126	if (!page_mkwrite)
2127	file_update_time(vma->vm_file);
2128	}
2129
2130	return VM_FAULT_WRITE;
2131	}
2132
2133	/*
2134	* Handle the case of a page which we actually need to copy to a new page.
2135	*
2136	* Called with mmap_sem locked and the old page referenced, but
2137	* without the ptl held.
2138	*
2139	* High level logic flow:
2140	*
2141	* - Allocate a page, copy the content of the old page to the new one.
2142	* - Handle book keeping and accounting - cgroups, mmu-notifiers, etc.
2143	* - Take the PTL. If the pte changed, bail out and release the allocated page
2144	* - If the pte is still the way we remember it, update the page table and all
2145	* relevant references. This includes dropping the reference the page-table
2146	* held to the old page, as well as updating the rmap.
2147	* - In any case, unlock the PTL and drop the reference we took to the old page.
2148	*/
2149	static int wp_page_copy(struct fault_env *fe, pte_t orig_pte,
2150	struct page *old_page)
2151	{
2152	struct vm_area_struct *vma = fe->vma;
2153	struct mm_struct *mm = vma->vm_mm;
2154	struct page *new_page = NULL;
2155	pte_t entry;
2156	int page_copied = 0;
2157	const unsigned long mmun_start = fe->address & PAGE_MASK;
2158	const unsigned long mmun_end = mmun_start + PAGE_SIZE;
2159	struct mem_cgroup *memcg;
2160
2161	if (unlikely(anon_vma_prepare(vma)))
2162	goto oom;
2163
2164	if (is_zero_pfn(pte_pfn(orig_pte))) {
2165	#ifdef CONFIG_AMLOGIC_CMA
2166	gfp_t tmp_flag = __GFP_MOVABLE | __GFP_BDEV;
2167
2168	new_page = __alloc_zeroed_user_highpage(tmp_flag, vma,
2169	fe->address);
2170	#else
2171	new_page = alloc_zeroed_user_highpage_movable(vma, fe->address);
2172	#endif
2173	if (!new_page)
2174	goto oom;
2175	} else {
2176	#ifdef CONFIG_AMLOGIC_CMA
2177	new_page = alloc_page_vma(GFP_HIGHUSER_MOVABLE | __GFP_BDEV,
2178	vma, fe->address);
2179	#else
2180	new_page = alloc_page_vma(GFP_HIGHUSER_MOVABLE, vma,
2181	fe->address);
2182	#endif
2183	if (!new_page)
2184	goto oom;
2185	cow_user_page(new_page, old_page, fe->address, vma);
2186	}
2187
2188	if (mem_cgroup_try_charge(new_page, mm, GFP_KERNEL, &memcg, false))
2189	goto oom_free_new;
2190
2191	__SetPageUptodate(new_page);
2192
2193	mmu_notifier_invalidate_range_start(mm, mmun_start, mmun_end);
2194
2195	/*
2196	* Re-check the pte - we dropped the lock
2197	*/
2198	fe->pte = pte_offset_map_lock(mm, fe->pmd, fe->address, &fe->ptl);
2199	if (likely(pte_same(*fe->pte, orig_pte))) {
2200	if (old_page) {
2201	if (!PageAnon(old_page)) {
2202	dec_mm_counter_fast(mm,
2203	mm_counter_file(old_page));
2204	inc_mm_counter_fast(mm, MM_ANONPAGES);
2205	}
2206	} else {
2207	inc_mm_counter_fast(mm, MM_ANONPAGES);
2208	}
2209	flush_cache_page(vma, fe->address, pte_pfn(orig_pte));
2210	entry = mk_pte(new_page, vma->vm_page_prot);
2211	entry = maybe_mkwrite(pte_mkdirty(entry), vma);
2212	/*
2213	* Clear the pte entry and flush it first, before updating the
2214	* pte with the new entry. This will avoid a race condition
2215	* seen in the presence of one thread doing SMC and another
2216	* thread doing COW.
2217	*/
2218	ptep_clear_flush_notify(vma, fe->address, fe->pte);
2219	page_add_new_anon_rmap(new_page, vma, fe->address, false);
2220	mem_cgroup_commit_charge(new_page, memcg, false, false);
2221	lru_cache_add_active_or_unevictable(new_page, vma);
2222	/*
2223	* We call the notify macro here because, when using secondary
2224	* mmu page tables (such as kvm shadow page tables), we want the
2225	* new page to be mapped directly into the secondary page table.
2226	*/
2227	set_pte_at_notify(mm, fe->address, fe->pte, entry);
2228	update_mmu_cache(vma, fe->address, fe->pte);
2229	if (old_page) {
2230	/*
2231	* Only after switching the pte to the new page may
2232	* we remove the mapcount here. Otherwise another
2233	* process may come and find the rmap count decremented
2234	* before the pte is switched to the new page, and
2235	* "reuse" the old page writing into it while our pte
2236	* here still points into it and can be read by other
2237	* threads.
2238	*
2239	* The critical issue is to order this
2240	* page_remove_rmap with the ptp_clear_flush above.
2241	* Those stores are ordered by (if nothing else,)
2242	* the barrier present in the atomic_add_negative
2243	* in page_remove_rmap.
2244	*
2245	* Then the TLB flush in ptep_clear_flush ensures that
2246	* no process can access the old page before the
2247	* decremented mapcount is visible. And the old page
2248	* cannot be reused until after the decremented
2249	* mapcount is visible. So transitively, TLBs to
2250	* old page will be flushed before it can be reused.
2251	*/
2252	page_remove_rmap(old_page, false);
2253	}
2254
2255	/* Free the old page.. */
2256	new_page = old_page;
2257	page_copied = 1;
2258	} else {
2259	mem_cgroup_cancel_charge(new_page, memcg, false);
2260	}
2261
2262	if (new_page)
2263	put_page(new_page);
2264
2265	pte_unmap_unlock(fe->pte, fe->ptl);
2266	mmu_notifier_invalidate_range_end(mm, mmun_start, mmun_end);
2267	if (old_page) {
2268	/*
2269	* Don't let another task, with possibly unlocked vma,
2270	* keep the mlocked page.
2271	*/
2272	if (page_copied && (vma->vm_flags & VM_LOCKED)) {
2273	lock_page(old_page); /* LRU manipulation */
2274	if (PageMlocked(old_page))
2275	munlock_vma_page(old_page);
2276	unlock_page(old_page);
2277	}
2278	put_page(old_page);
2279	}
2280	return page_copied ? VM_FAULT_WRITE : 0;
2281	oom_free_new:
2282	put_page(new_page);
2283	oom:
2284	if (old_page)
2285	put_page(old_page);
2286	return VM_FAULT_OOM;
2287	}
2288
2289	/*
2290	* Handle write page faults for VM_MIXEDMAP or VM_PFNMAP for a VM_SHARED
2291	* mapping
2292	*/
2293	static int wp_pfn_shared(struct fault_env *fe, pte_t orig_pte)
2294	{
2295	struct vm_area_struct *vma = fe->vma;
2296
2297	if (vma->vm_ops && vma->vm_ops->pfn_mkwrite) {
2298	struct vm_fault vmf = {
2299	.page = NULL,
2300	.pgoff = linear_page_index(vma, fe->address),
2301	.virtual_address =
2302	(void __user *)(fe->address & PAGE_MASK),
2303	.flags = FAULT_FLAG_WRITE | FAULT_FLAG_MKWRITE,
2304	};
2305	int ret;
2306
2307	pte_unmap_unlock(fe->pte, fe->ptl);
2308	ret = vma->vm_ops->pfn_mkwrite(vma, &vmf);
2309	if (ret & VM_FAULT_ERROR)
2310	return ret;
2311	fe->pte = pte_offset_map_lock(vma->vm_mm, fe->pmd, fe->address,
2312	&fe->ptl);
2313	/*
2314	* We might have raced with another page fault while we
2315	* released the pte_offset_map_lock.
2316	*/
2317	if (!pte_same(*fe->pte, orig_pte)) {
2318	pte_unmap_unlock(fe->pte, fe->ptl);
2319	return 0;
2320	}
2321	}
2322	return wp_page_reuse(fe, orig_pte, NULL, 0, 0);
2323	}
2324
2325	static int wp_page_shared(struct fault_env *fe, pte_t orig_pte,
2326	struct page *old_page)
2327	__releases(fe->ptl)
2328	{
2329	struct vm_area_struct *vma = fe->vma;
2330	int page_mkwrite = 0;
2331
2332	get_page(old_page);
2333
2334	if (vma->vm_ops && vma->vm_ops->page_mkwrite) {
2335	int tmp;
2336
2337	pte_unmap_unlock(fe->pte, fe->ptl);
2338	tmp = do_page_mkwrite(vma, old_page, fe->address);
2339	if (unlikely(!tmp || (tmp &
2340	(VM_FAULT_ERROR | VM_FAULT_NOPAGE)))) {
2341	put_page(old_page);
2342	return tmp;
2343	}
2344	/*
2345	* Since we dropped the lock we need to revalidate
2346	* the PTE as someone else may have changed it. If
2347	* they did, we just return, as we can count on the
2348	* MMU to tell us if they didn't also make it writable.
2349	*/
2350	fe->pte = pte_offset_map_lock(vma->vm_mm, fe->pmd, fe->address,
2351	&fe->ptl);
2352	if (!pte_same(*fe->pte, orig_pte)) {
2353	unlock_page(old_page);
2354	pte_unmap_unlock(fe->pte, fe->ptl);
2355	put_page(old_page);
2356	return 0;
2357	}
2358	page_mkwrite = 1;
2359	}
2360
2361	return wp_page_reuse(fe, orig_pte, old_page, page_mkwrite, 1);
2362	}
2363
2364	/*
2365	* This routine handles present pages, when users try to write
2366	* to a shared page. It is done by copying the page to a new address
2367	* and decrementing the shared-page counter for the old page.
2368	*
2369	* Note that this routine assumes that the protection checks have been
2370	* done by the caller (the low-level page fault routine in most cases).
2371	* Thus we can safely just mark it writable once we've done any necessary
2372	* COW.
2373	*
2374	* We also mark the page dirty at this point even though the page will
2375	* change only once the write actually happens. This avoids a few races,
2376	* and potentially makes it more efficient.
2377	*
2378	* We enter with non-exclusive mmap_sem (to exclude vma changes,
2379	* but allow concurrent faults), with pte both mapped and locked.
2380	* We return with mmap_sem still held, but pte unmapped and unlocked.
2381	*/
2382	static int do_wp_page(struct fault_env *fe, pte_t orig_pte)
2383	__releases(fe->ptl)
2384	{
2385	struct vm_area_struct *vma = fe->vma;
2386	struct page *old_page;
2387
2388	old_page = vm_normal_page(vma, fe->address, orig_pte);
2389	if (!old_page) {
2390	/*
2391	* VM_MIXEDMAP !pfn_valid() case, or VM_SOFTDIRTY clear on a
2392	* VM_PFNMAP VMA.
2393	*
2394	* We should not cow pages in a shared writeable mapping.
2395	* Just mark the pages writable and/or call ops->pfn_mkwrite.
2396	*/
2397	if ((vma->vm_flags & (VM_WRITE|VM_SHARED)) ==
2398	(VM_WRITE|VM_SHARED))
2399	return wp_pfn_shared(fe, orig_pte);
2400
2401	pte_unmap_unlock(fe->pte, fe->ptl);
2402	return wp_page_copy(fe, orig_pte, old_page);
2403	}
2404
2405	/*
2406	* Take out anonymous pages first, anonymous shared vmas are
2407	* not dirty accountable.
2408	*/
2409	if (PageAnon(old_page) && !PageKsm(old_page)) {
2410	int total_mapcount;
2411	if (!trylock_page(old_page)) {
2412	get_page(old_page);
2413	pte_unmap_unlock(fe->pte, fe->ptl);
2414	lock_page(old_page);
2415	fe->pte = pte_offset_map_lock(vma->vm_mm, fe->pmd,
2416	fe->address, &fe->ptl);
2417	if (!pte_same(*fe->pte, orig_pte)) {
2418	unlock_page(old_page);
2419	pte_unmap_unlock(fe->pte, fe->ptl);
2420	put_page(old_page);
2421	return 0;
2422	}
2423	put_page(old_page);
2424	}
2425	if (reuse_swap_page(old_page, &total_mapcount)) {
2426	if (total_mapcount == 1) {
2427	/*
2428	* The page is all ours. Move it to
2429	* our anon_vma so the rmap code will
2430	* not search our parent or siblings.
2431	* Protected against the rmap code by
2432	* the page lock.
2433	*/
2434	page_move_anon_rmap(old_page, vma);
2435	}
2436	unlock_page(old_page);
2437	return wp_page_reuse(fe, orig_pte, old_page, 0, 0);
2438	}
2439	unlock_page(old_page);
2440	} else if (unlikely((vma->vm_flags & (VM_WRITE|VM_SHARED)) ==
2441	(VM_WRITE|VM_SHARED))) {
2442	return wp_page_shared(fe, orig_pte, old_page);
2443	}
2444
2445	/*
2446	* Ok, we need to copy. Oh, well..
2447	*/
2448	get_page(old_page);
2449
2450	pte_unmap_unlock(fe->pte, fe->ptl);
2451	return wp_page_copy(fe, orig_pte, old_page);
2452	}
2453
2454	static void unmap_mapping_range_vma(struct vm_area_struct *vma,
2455	unsigned long start_addr, unsigned long end_addr,
2456	struct zap_details *details)
2457	{
2458	zap_page_range_single(vma, start_addr, end_addr - start_addr, details);
2459	}
2460
2461	static inline void unmap_mapping_range_tree(struct rb_root *root,
2462	struct zap_details *details)
2463	{
2464	struct vm_area_struct *vma;
2465	pgoff_t vba, vea, zba, zea;
2466
2467	vma_interval_tree_foreach(vma, root,
2468	details->first_index, details->last_index) {
2469
2470	vba = vma->vm_pgoff;
2471	vea = vba + vma_pages(vma) - 1;
2472	zba = details->first_index;
2473	if (zba < vba)
2474	zba = vba;
2475	zea = details->last_index;
2476	if (zea > vea)
2477	zea = vea;
2478
2479	unmap_mapping_range_vma(vma,
2480	((zba - vba) << PAGE_SHIFT) + vma->vm_start,
2481	((zea - vba + 1) << PAGE_SHIFT) + vma->vm_start,
2482	details);
2483	}
2484	}
2485
2486	/**
2487	* unmap_mapping_range - unmap the portion of all mmaps in the specified
2488	* address_space corresponding to the specified page range in the underlying
2489	* file.
2490	*
2491	* @mapping: the address space containing mmaps to be unmapped.
2492	* @holebegin: byte in first page to unmap, relative to the start of
2493	* the underlying file. This will be rounded down to a PAGE_SIZE
2494	* boundary. Note that this is different from truncate_pagecache(), which
2495	* must keep the partial page. In contrast, we must get rid of
2496	* partial pages.
2497	* @holelen: size of prospective hole in bytes. This will be rounded
2498	* up to a PAGE_SIZE boundary. A holelen of zero truncates to the
2499	* end of the file.
2500	* @even_cows: 1 when truncating a file, unmap even private COWed pages;
2501	* but 0 when invalidating pagecache, don't throw away private data.
2502	*/
2503	void unmap_mapping_range(struct address_space *mapping,
2504	loff_t const holebegin, loff_t const holelen, int even_cows)
2505	{
2506	struct zap_details details = { };
2507	pgoff_t hba = holebegin >> PAGE_SHIFT;
2508	pgoff_t hlen = (holelen + PAGE_SIZE - 1) >> PAGE_SHIFT;
2509
2510	/* Check for overflow. */
2511	if (sizeof(holelen) > sizeof(hlen)) {
2512	long long holeend =
2513	(holebegin + holelen + PAGE_SIZE - 1) >> PAGE_SHIFT;
2514	if (holeend & ~(long long)ULONG_MAX)
2515	hlen = ULONG_MAX - hba + 1;
2516	}
2517
2518	details.check_mapping = even_cows? NULL: mapping;
2519	details.first_index = hba;
2520	details.last_index = hba + hlen - 1;
2521	if (details.last_index < details.first_index)
2522	details.last_index = ULONG_MAX;
2523
2524	i_mmap_lock_write(mapping);
2525	if (unlikely(!RB_EMPTY_ROOT(&mapping->i_mmap)))
2526	unmap_mapping_range_tree(&mapping->i_mmap, &details);
2527	i_mmap_unlock_write(mapping);
2528	}
2529	EXPORT_SYMBOL(unmap_mapping_range);
2530
2531	/*
2532	* We enter with non-exclusive mmap_sem (to exclude vma changes,
2533	* but allow concurrent faults), and pte mapped but not yet locked.
2534	* We return with pte unmapped and unlocked.
2535	*
2536	* We return with the mmap_sem locked or unlocked in the same cases
2537	* as does filemap_fault().
2538	*/
2539	int do_swap_page(struct fault_env *fe, pte_t orig_pte)
2540	{
2541	struct vm_area_struct *vma = fe->vma;
2542	struct page *page, *swapcache;
2543	struct mem_cgroup *memcg;
2544	swp_entry_t entry;
2545	pte_t pte;
2546	int locked;
2547	int exclusive = 0;
2548	int ret = 0;
2549
2550	if (!pte_unmap_same(vma->vm_mm, fe->pmd, fe->pte, orig_pte))
2551	goto out;
2552
2553	entry = pte_to_swp_entry(orig_pte);
2554	if (unlikely(non_swap_entry(entry))) {
2555	if (is_migration_entry(entry)) {
2556	migration_entry_wait(vma->vm_mm, fe->pmd, fe->address);
2557	} else if (is_hwpoison_entry(entry)) {
2558	ret = VM_FAULT_HWPOISON;
2559	} else {
2560	print_bad_pte(vma, fe->address, orig_pte, NULL);
2561	ret = VM_FAULT_SIGBUS;
2562	}
2563	goto out;
2564	}
2565	delayacct_set_flag(DELAYACCT_PF_SWAPIN);
2566	page = lookup_swap_cache(entry);
2567	if (!page) {
2568	page = swapin_readahead(entry,
2569	GFP_HIGHUSER_MOVABLE, vma, fe->address);
2570	if (!page) {
2571	/*
2572	* Back out if somebody else faulted in this pte
2573	* while we released the pte lock.
2574	*/
2575	fe->pte = pte_offset_map_lock(vma->vm_mm, fe->pmd,
2576	fe->address, &fe->ptl);
2577	if (likely(pte_same(*fe->pte, orig_pte)))
2578	ret = VM_FAULT_OOM;
2579	delayacct_clear_flag(DELAYACCT_PF_SWAPIN);
2580	goto unlock;
2581	}
2582
2583	/* Had to read the page from swap area: Major fault */
2584	ret = VM_FAULT_MAJOR;
2585	count_vm_event(PGMAJFAULT);
2586	mem_cgroup_count_vm_event(vma->vm_mm, PGMAJFAULT);
2587	} else if (PageHWPoison(page)) {
2588	/*
2589	* hwpoisoned dirty swapcache pages are kept for killing
2590	* owner processes (which may be unknown at hwpoison time)
2591	*/
2592	ret = VM_FAULT_HWPOISON;
2593	delayacct_clear_flag(DELAYACCT_PF_SWAPIN);
2594	swapcache = page;
2595	goto out_release;
2596	}
2597
2598	swapcache = page;
2599	locked = lock_page_or_retry(page, vma->vm_mm, fe->flags);
2600
2601	delayacct_clear_flag(DELAYACCT_PF_SWAPIN);
2602	if (!locked) {
2603	ret |= VM_FAULT_RETRY;
2604	goto out_release;
2605	}
2606
2607	/*
2608	* Make sure try_to_free_swap or reuse_swap_page or swapoff did not
2609	* release the swapcache from under us. The page pin, and pte_same
2610	* test below, are not enough to exclude that. Even if it is still
2611	* swapcache, we need to check that the page's swap has not changed.
2612	*/
2613	if (unlikely(!PageSwapCache(page) || page_private(page) != entry.val))
2614	goto out_page;
2615
2616	page = ksm_might_need_to_copy(page, vma, fe->address);
2617	if (unlikely(!page)) {
2618	ret = VM_FAULT_OOM;
2619	page = swapcache;
2620	goto out_page;
2621	}
2622
2623	if (mem_cgroup_try_charge(page, vma->vm_mm, GFP_KERNEL,
2624	&memcg, false)) {
2625	ret = VM_FAULT_OOM;
2626	goto out_page;
2627	}
2628
2629	/*
2630	* Back out if somebody else already faulted in this pte.
2631	*/
2632	fe->pte = pte_offset_map_lock(vma->vm_mm, fe->pmd, fe->address,
2633	&fe->ptl);
2634	if (unlikely(!pte_same(*fe->pte, orig_pte)))
2635	goto out_nomap;
2636
2637	if (unlikely(!PageUptodate(page))) {
2638	ret = VM_FAULT_SIGBUS;
2639	goto out_nomap;
2640	}
2641
2642	/*
2643	* The page isn't present yet, go ahead with the fault.
2644	*
2645	* Be careful about the sequence of operations here.
2646	* To get its accounting right, reuse_swap_page() must be called
2647	* while the page is counted on swap but not yet in mapcount i.e.
2648	* before page_add_anon_rmap() and swap_free(); try_to_free_swap()
2649	* must be called after the swap_free(), or it will never succeed.
2650	*/
2651
2652	inc_mm_counter_fast(vma->vm_mm, MM_ANONPAGES);
2653	dec_mm_counter_fast(vma->vm_mm, MM_SWAPENTS);
2654	pte = mk_pte(page, vma->vm_page_prot);
2655	if ((fe->flags & FAULT_FLAG_WRITE) && reuse_swap_page(page, NULL)) {
2656	pte = maybe_mkwrite(pte_mkdirty(pte), vma);
2657	fe->flags &= ~FAULT_FLAG_WRITE;
2658	ret |= VM_FAULT_WRITE;
2659	exclusive = RMAP_EXCLUSIVE;
2660	}
2661	flush_icache_page(vma, page);
2662	if (pte_swp_soft_dirty(orig_pte))
2663	pte = pte_mksoft_dirty(pte);
2664	set_pte_at(vma->vm_mm, fe->address, fe->pte, pte);
2665	if (page == swapcache) {
2666	do_page_add_anon_rmap(page, vma, fe->address, exclusive);
2667	mem_cgroup_commit_charge(page, memcg, true, false);
2668	activate_page(page);
2669	} else { /* ksm created a completely new copy */
2670	page_add_new_anon_rmap(page, vma, fe->address, false);
2671	mem_cgroup_commit_charge(page, memcg, false, false);
2672	lru_cache_add_active_or_unevictable(page, vma);
2673	}
2674
2675	swap_free(entry);
2676	if (mem_cgroup_swap_full(page) ||
2677	(vma->vm_flags & VM_LOCKED) || PageMlocked(page))
2678	try_to_free_swap(page);
2679	unlock_page(page);
2680	if (page != swapcache) {
2681	/*
2682	* Hold the lock to avoid the swap entry to be reused
2683	* until we take the PT lock for the pte_same() check
2684	* (to avoid false positives from pte_same). For
2685	* further safety release the lock after the swap_free
2686	* so that the swap count won't change under a
2687	* parallel locked swapcache.
2688	*/
2689	unlock_page(swapcache);
2690	put_page(swapcache);
2691	}
2692
2693	if (fe->flags & FAULT_FLAG_WRITE) {
2694	ret |= do_wp_page(fe, pte);
2695	if (ret & VM_FAULT_ERROR)
2696	ret &= VM_FAULT_ERROR;
2697	goto out;
2698	}
2699
2700	/* No need to invalidate - it was non-present before */
2701	update_mmu_cache(vma, fe->address, fe->pte);
2702	unlock:
2703	pte_unmap_unlock(fe->pte, fe->ptl);
2704	out:
2705	return ret;
2706	out_nomap:
2707	mem_cgroup_cancel_charge(page, memcg, false);
2708	pte_unmap_unlock(fe->pte, fe->ptl);
2709	out_page:
2710	unlock_page(page);
2711	out_release:
2712	put_page(page);
2713	if (page != swapcache) {
2714	unlock_page(swapcache);
2715	put_page(swapcache);
2716	}
2717	return ret;
2718	}
2719
2720	/*
2721	* We enter with non-exclusive mmap_sem (to exclude vma changes,
2722	* but allow concurrent faults), and pte mapped but not yet locked.
2723	* We return with mmap_sem still held, but pte unmapped and unlocked.
2724	*/
2725	static int do_anonymous_page(struct fault_env *fe)
2726	{
2727	struct vm_area_struct *vma = fe->vma;
2728	struct mem_cgroup *memcg;
2729	struct page *page;
2730	pte_t entry;
2731
2732	/* File mapping without ->vm_ops ? */
2733	if (vma->vm_flags & VM_SHARED)
2734	return VM_FAULT_SIGBUS;
2735
2736	/*
2737	* Use pte_alloc() instead of pte_alloc_map(). We can't run
2738	* pte_offset_map() on pmds where a huge pmd might be created
2739	* from a different thread.
2740	*
2741	* pte_alloc_map() is safe to use under down_write(mmap_sem) or when
2742	* parallel threads are excluded by other means.
2743	*
2744	* Here we only have down_read(mmap_sem).
2745	*/
2746	if (pte_alloc(vma->vm_mm, fe->pmd, fe->address))
2747	return VM_FAULT_OOM;
2748
2749	/* See the comment in pte_alloc_one_map() */
2750	if (unlikely(pmd_trans_unstable(fe->pmd)))
2751	return 0;
2752
2753	/* Use the zero-page for reads */
2754	if (!(fe->flags & FAULT_FLAG_WRITE) &&
2755	!mm_forbids_zeropage(vma->vm_mm)) {
2756	entry = pte_mkspecial(pfn_pte(my_zero_pfn(fe->address),
2757	vma->vm_page_prot));
2758	fe->pte = pte_offset_map_lock(vma->vm_mm, fe->pmd, fe->address,
2759	&fe->ptl);
2760	if (!pte_none(*fe->pte))
2761	goto unlock;
2762	/* Deliver the page fault to userland, check inside PT lock */
2763	if (userfaultfd_missing(vma)) {
2764	pte_unmap_unlock(fe->pte, fe->ptl);
2765	return handle_userfault(fe, VM_UFFD_MISSING);
2766	}
2767	goto setpte;
2768	}
2769
2770	/* Allocate our own private page. */
2771	if (unlikely(anon_vma_prepare(vma)))
2772	goto oom;
2773	page = alloc_zeroed_user_highpage_movable(vma, fe->address);
2774	if (!page)
2775	goto oom;
2776
2777	if (mem_cgroup_try_charge(page, vma->vm_mm, GFP_KERNEL, &memcg, false))
2778	goto oom_free_page;
2779
2780	/*
2781	* The memory barrier inside __SetPageUptodate makes sure that
2782	* preceeding stores to the page contents become visible before
2783	* the set_pte_at() write.
2784	*/
2785	__SetPageUptodate(page);
2786
2787	entry = mk_pte(page, vma->vm_page_prot);
2788	if (vma->vm_flags & VM_WRITE)
2789	entry = pte_mkwrite(pte_mkdirty(entry));
2790
2791	fe->pte = pte_offset_map_lock(vma->vm_mm, fe->pmd, fe->address,
2792	&fe->ptl);
2793	if (!pte_none(*fe->pte))
2794	goto release;
2795
2796	/* Deliver the page fault to userland, check inside PT lock */
2797	if (userfaultfd_missing(vma)) {
2798	pte_unmap_unlock(fe->pte, fe->ptl);
2799	mem_cgroup_cancel_charge(page, memcg, false);
2800	put_page(page);
2801	return handle_userfault(fe, VM_UFFD_MISSING);
2802	}
2803
2804	inc_mm_counter_fast(vma->vm_mm, MM_ANONPAGES);
2805	page_add_new_anon_rmap(page, vma, fe->address, false);
2806	mem_cgroup_commit_charge(page, memcg, false, false);
2807	lru_cache_add_active_or_unevictable(page, vma);
2808	setpte:
2809	set_pte_at(vma->vm_mm, fe->address, fe->pte, entry);
2810
2811	/* No need to invalidate - it was non-present before */
2812	update_mmu_cache(vma, fe->address, fe->pte);
2813	unlock:
2814	pte_unmap_unlock(fe->pte, fe->ptl);
2815	return 0;
2816	release:
2817	mem_cgroup_cancel_charge(page, memcg, false);
2818	put_page(page);
2819	goto unlock;
2820	oom_free_page:
2821	put_page(page);
2822	oom:
2823	return VM_FAULT_OOM;
2824	}
2825
2826	/*
2827	* The mmap_sem must have been held on entry, and may have been
2828	* released depending on flags and vma->vm_ops->fault() return value.
2829	* See filemap_fault() and __lock_page_retry().
2830	*/
2831	static int __do_fault(struct fault_env *fe, pgoff_t pgoff,
2832	struct page *cow_page, struct page **page, void **entry)
2833	{
2834	struct vm_area_struct *vma = fe->vma;
2835	struct vm_fault vmf;
2836	int ret;
2837
2838	/*
2839	* Preallocate pte before we take page_lock because this might lead to
2840	* deadlocks for memcg reclaim which waits for pages under writeback:
2841	* lock_page(A)
2842	* SetPageWriteback(A)
2843	* unlock_page(A)
2844	* lock_page(B)
2845	* lock_page(B)
2846	* pte_alloc_pne
2847	* shrink_page_list
2848	* wait_on_page_writeback(A)
2849	* SetPageWriteback(B)
2850	* unlock_page(B)
2851	* # flush A, B to clear the writeback
2852	*/
2853	if (pmd_none(*fe->pmd) && !fe->prealloc_pte) {
2854	fe->prealloc_pte = pte_alloc_one(vma->vm_mm, fe->address);
2855	if (!fe->prealloc_pte)
2856	return VM_FAULT_OOM;
2857	smp_wmb(); /* See comment in __pte_alloc() */
2858	}
2859
2860	vmf.virtual_address = (void __user *)(fe->address & PAGE_MASK);
2861	vmf.pgoff = pgoff;
2862	vmf.flags = fe->flags;
2863	vmf.page = NULL;
2864	vmf.gfp_mask = __get_fault_gfp_mask(vma);
2865	vmf.cow_page = cow_page;
2866
2867	ret = vma->vm_ops->fault(vma, &vmf);
2868	if (unlikely(ret & (VM_FAULT_ERROR | VM_FAULT_NOPAGE | VM_FAULT_RETRY)))
2869	return ret;
2870	if (ret & VM_FAULT_DAX_LOCKED) {
2871	*entry = vmf.entry;
2872	return ret;
2873	}
2874
2875	if (unlikely(PageHWPoison(vmf.page))) {
2876	if (ret & VM_FAULT_LOCKED)
2877	unlock_page(vmf.page);
2878	put_page(vmf.page);
2879	return VM_FAULT_HWPOISON;
2880	}
2881
2882	if (unlikely(!(ret & VM_FAULT_LOCKED)))
2883	lock_page(vmf.page);
2884	else
2885	VM_BUG_ON_PAGE(!PageLocked(vmf.page), vmf.page);
2886
2887	*page = vmf.page;
2888	return ret;
2889	}
2890
2891	/*
2892	* The ordering of these checks is important for pmds with _PAGE_DEVMAP set.
2893	* If we check pmd_trans_unstable() first we will trip the bad_pmd() check
2894	* inside of pmd_none_or_trans_huge_or_clear_bad(). This will end up correctly
2895	* returning 1 but not before it spams dmesg with the pmd_clear_bad() output.
2896	*/
2897	static int pmd_devmap_trans_unstable(pmd_t *pmd)
2898	{
2899	return pmd_devmap(*pmd) || pmd_trans_unstable(pmd);
2900	}
2901
2902	static int pte_alloc_one_map(struct fault_env *fe)
2903	{
2904	struct vm_area_struct *vma = fe->vma;
2905
2906	if (!pmd_none(*fe->pmd))
2907	goto map_pte;
2908	if (fe->prealloc_pte) {
2909	fe->ptl = pmd_lock(vma->vm_mm, fe->pmd);
2910	if (unlikely(!pmd_none(*fe->pmd))) {
2911	spin_unlock(fe->ptl);
2912	goto map_pte;
2913	}
2914
2915	atomic_long_inc(&vma->vm_mm->nr_ptes);
2916	pmd_populate(vma->vm_mm, fe->pmd, fe->prealloc_pte);
2917	spin_unlock(fe->ptl);
2918	fe->prealloc_pte = 0;
2919	} else if (unlikely(pte_alloc(vma->vm_mm, fe->pmd, fe->address))) {
2920	return VM_FAULT_OOM;
2921	}
2922	map_pte:
2923	/*
2924	* If a huge pmd materialized under us just retry later. Use
2925	* pmd_trans_unstable() via pmd_devmap_trans_unstable() instead of
2926	* pmd_trans_huge() to ensure the pmd didn't become pmd_trans_huge
2927	* under us and then back to pmd_none, as a result of MADV_DONTNEED
2928	* running immediately after a huge pmd fault in a different thread of
2929	* this mm, in turn leading to a misleading pmd_trans_huge() retval.
2930	* All we have to ensure is that it is a regular pmd that we can walk
2931	* with pte_offset_map() and we can do that through an atomic read in
2932	* C, which is what pmd_trans_unstable() provides.
2933	*/
2934	if (pmd_devmap_trans_unstable(fe->pmd))
2935	return VM_FAULT_NOPAGE;
2936
2937	/*
2938	* At this point we know that our vmf->pmd points to a page of ptes
2939	* and it cannot become pmd_none(), pmd_devmap() or pmd_trans_huge()
2940	* for the duration of the fault. If a racing MADV_DONTNEED runs and
2941	* we zap the ptes pointed to by our vmf->pmd, the vmf->ptl will still
2942	* be valid and we will re-check to make sure the vmf->pte isn't
2943	* pte_none() under vmf->ptl protection when we return to
2944	* alloc_set_pte().
2945	*/
2946	fe->pte = pte_offset_map_lock(vma->vm_mm, fe->pmd, fe->address,
2947	&fe->ptl);
2948	return 0;
2949	}
2950
2951	#ifdef CONFIG_TRANSPARENT_HUGE_PAGECACHE
2952
2953	#define HPAGE_CACHE_INDEX_MASK (HPAGE_PMD_NR - 1)
2954	static inline bool transhuge_vma_suitable(struct vm_area_struct *vma,
2955	unsigned long haddr)
2956	{
2957	if (((vma->vm_start >> PAGE_SHIFT) & HPAGE_CACHE_INDEX_MASK) !=
2958	(vma->vm_pgoff & HPAGE_CACHE_INDEX_MASK))
2959	return false;
2960	if (haddr < vma->vm_start || haddr + HPAGE_PMD_SIZE > vma->vm_end)
2961	return false;
2962	return true;
2963	}
2964
2965	static int do_set_pmd(struct fault_env *fe, struct page *page)
2966	{
2967	struct vm_area_struct *vma = fe->vma;
2968	bool write = fe->flags & FAULT_FLAG_WRITE;
2969	unsigned long haddr = fe->address & HPAGE_PMD_MASK;
2970	pmd_t entry;
2971	int i, ret;
2972
2973	if (!transhuge_vma_suitable(vma, haddr))
2974	return VM_FAULT_FALLBACK;
2975
2976	ret = VM_FAULT_FALLBACK;
2977	page = compound_head(page);
2978
2979	fe->ptl = pmd_lock(vma->vm_mm, fe->pmd);
2980	if (unlikely(!pmd_none(*fe->pmd)))
2981	goto out;
2982
2983	for (i = 0; i < HPAGE_PMD_NR; i++)
2984	flush_icache_page(vma, page + i);
2985
2986	entry = mk_huge_pmd(page, vma->vm_page_prot);
2987	if (write)
2988	entry = maybe_pmd_mkwrite(pmd_mkdirty(entry), vma);
2989
2990	add_mm_counter(vma->vm_mm, MM_FILEPAGES, HPAGE_PMD_NR);
2991	page_add_file_rmap(page, true);
2992
2993	set_pmd_at(vma->vm_mm, haddr, fe->pmd, entry);
2994
2995	update_mmu_cache_pmd(vma, haddr, fe->pmd);
2996
2997	/* fault is handled */
2998	ret = 0;
2999	count_vm_event(THP_FILE_MAPPED);
3000	out:
3001	spin_unlock(fe->ptl);
3002	return ret;
3003	}
3004	#else
3005	static int do_set_pmd(struct fault_env *fe, struct page *page)
3006	{
3007	BUILD_BUG();
3008	return 0;
3009	}
3010	#endif
3011
3012	/**
3013	* alloc_set_pte - setup new PTE entry for given page and add reverse page
3014	* mapping. If needed, the fucntion allocates page table or use pre-allocated.
3015	*
3016	* @fe: fault environment
3017	* @memcg: memcg to charge page (only for private mappings)
3018	* @page: page to map
3019	*
3020	* Caller must take care of unlocking fe->ptl, if fe->pte is non-NULL on return.
3021	*
3022	* Target users are page handler itself and implementations of
3023	* vm_ops->map_pages.
3024	*/
3025	int alloc_set_pte(struct fault_env *fe, struct mem_cgroup *memcg,
3026	struct page *page)
3027	{
3028	struct vm_area_struct *vma = fe->vma;
3029	bool write = fe->flags & FAULT_FLAG_WRITE;
3030	pte_t entry;
3031	int ret;
3032
3033	if (pmd_none(*fe->pmd) && PageTransCompound(page) &&
3034	IS_ENABLED(CONFIG_TRANSPARENT_HUGE_PAGECACHE)) {
3035	/* THP on COW? */
3036	VM_BUG_ON_PAGE(memcg, page);
3037
3038	ret = do_set_pmd(fe, page);
3039	if (ret != VM_FAULT_FALLBACK)
3040	return ret;
3041	}
3042
3043	if (!fe->pte) {
3044	ret = pte_alloc_one_map(fe);
3045	if (ret)
3046	return ret;
3047	}
3048
3049	/* Re-check under ptl */
3050	if (unlikely(!pte_none(*fe->pte)))
3051	return VM_FAULT_NOPAGE;
3052
3053	flush_icache_page(vma, page);
3054	entry = mk_pte(page, vma->vm_page_prot);
3055	if (write)
3056	entry = maybe_mkwrite(pte_mkdirty(entry), vma);
3057	/* copy-on-write page */
3058	if (write && !(vma->vm_flags & VM_SHARED)) {
3059	inc_mm_counter_fast(vma->vm_mm, MM_ANONPAGES);
3060	page_add_new_anon_rmap(page, vma, fe->address, false);
3061	mem_cgroup_commit_charge(page, memcg, false, false);
3062	lru_cache_add_active_or_unevictable(page, vma);
3063	} else {
3064	inc_mm_counter_fast(vma->vm_mm, mm_counter_file(page));
3065	page_add_file_rmap(page, false);
3066	}
3067	set_pte_at(vma->vm_mm, fe->address, fe->pte, entry);
3068
3069	/* no need to invalidate: a not-present page won't be cached */
3070	update_mmu_cache(vma, fe->address, fe->pte);
3071
3072	return 0;
3073	}
3074
3075	static unsigned long fault_around_bytes __read_mostly =
3076	rounddown_pow_of_two(65536);
3077
3078	#ifdef CONFIG_DEBUG_FS
3079	static int fault_around_bytes_get(void *data, u64 *val)
3080	{
3081	*val = fault_around_bytes;
3082	return 0;
3083	}
3084
3085	/*
3086	* fault_around_pages() and fault_around_mask() expects fault_around_bytes
3087	* rounded down to nearest page order. It's what do_fault_around() expects to
3088	* see.
3089	*/
3090	static int fault_around_bytes_set(void *data, u64 val)
3091	{
3092	if (val / PAGE_SIZE > PTRS_PER_PTE)
3093	return -EINVAL;
3094	if (val > PAGE_SIZE)
3095	fault_around_bytes = rounddown_pow_of_two(val);
3096	else
3097	fault_around_bytes = PAGE_SIZE; /* rounddown_pow_of_two(0) is undefined */
3098	return 0;
3099	}
3100	DEFINE_SIMPLE_ATTRIBUTE(fault_around_bytes_fops,
3101	fault_around_bytes_get, fault_around_bytes_set, "%llu\n");
3102
3103	static int __init fault_around_debugfs(void)
3104	{
3105	void *ret;
3106
3107	ret = debugfs_create_file("fault_around_bytes", 0644, NULL, NULL,
3108	&fault_around_bytes_fops);
3109	if (!ret)
3110	pr_warn("Failed to create fault_around_bytes in debugfs");
3111	return 0;
3112	}
3113	late_initcall(fault_around_debugfs);
3114	#endif
3115
3116	/*
3117	* do_fault_around() tries to map few pages around the fault address. The hope
3118	* is that the pages will be needed soon and this will lower the number of
3119	* faults to handle.
3120	*
3121	* It uses vm_ops->map_pages() to map the pages, which skips the page if it's
3122	* not ready to be mapped: not up-to-date, locked, etc.
3123	*
3124	* This function is called with the page table lock taken. In the split ptlock
3125	* case the page table lock only protects only those entries which belong to
3126	* the page table corresponding to the fault address.
3127	*
3128	* This function doesn't cross the VMA boundaries, in order to call map_pages()
3129	* only once.
3130	*
3131	* fault_around_pages() defines how many pages we'll try to map.
3132	* do_fault_around() expects it to return a power of two less than or equal to
3133	* PTRS_PER_PTE.
3134	*
3135	* The virtual address of the area that we map is naturally aligned to the
3136	* fault_around_pages() value (and therefore to page order). This way it's
3137	* easier to guarantee that we don't cross page table boundaries.
3138	*/
3139	static int do_fault_around(struct fault_env *fe, pgoff_t start_pgoff)
3140	{
3141	unsigned long address = fe->address, nr_pages, mask;
3142	pgoff_t end_pgoff;
3143	int off, ret = 0;
3144
3145	nr_pages = READ_ONCE(fault_around_bytes) >> PAGE_SHIFT;
3146	mask = ~(nr_pages * PAGE_SIZE - 1) & PAGE_MASK;
3147
3148	fe->address = max(address & mask, fe->vma->vm_start);
3149	off = ((address - fe->address) >> PAGE_SHIFT) & (PTRS_PER_PTE - 1);
3150	start_pgoff -= off;
3151
3152	/*
3153	* end_pgoff is either end of page table or end of vma
3154	* or fault_around_pages() from start_pgoff, depending what is nearest.
3155	*/
3156	end_pgoff = start_pgoff -
3157	((fe->address >> PAGE_SHIFT) & (PTRS_PER_PTE - 1)) +
3158	PTRS_PER_PTE - 1;
3159	end_pgoff = min3(end_pgoff, vma_pages(fe->vma) + fe->vma->vm_pgoff - 1,
3160	start_pgoff + nr_pages - 1);
3161
3162	if (pmd_none(*fe->pmd)) {
3163	fe->prealloc_pte = pte_alloc_one(fe->vma->vm_mm, fe->address);
3164	if (!fe->prealloc_pte)
3165	goto out;
3166	smp_wmb(); /* See comment in __pte_alloc() */
3167	}
3168
3169	fe->vma->vm_ops->map_pages(fe, start_pgoff, end_pgoff);
3170
3171	/* preallocated pagetable is unused: free it */
3172	if (fe->prealloc_pte) {
3173	pte_free(fe->vma->vm_mm, fe->prealloc_pte);
3174	fe->prealloc_pte = 0;
3175	}
3176	/* Huge page is mapped? Page fault is solved */
3177	if (pmd_trans_huge(*fe->pmd)) {
3178	ret = VM_FAULT_NOPAGE;
3179	goto out;
3180	}
3181
3182	/* ->map_pages() haven't done anything useful. Cold page cache? */
3183	if (!fe->pte)
3184	goto out;
3185
3186	/* check if the page fault is solved */
3187	fe->pte -= (fe->address >> PAGE_SHIFT) - (address >> PAGE_SHIFT);
3188	if (!pte_none(*fe->pte))
3189	ret = VM_FAULT_NOPAGE;
3190	pte_unmap_unlock(fe->pte, fe->ptl);
3191	out:
3192	fe->address = address;
3193	fe->pte = NULL;
3194	return ret;
3195	}
3196
3197	static int do_read_fault(struct fault_env *fe, pgoff_t pgoff)
3198	{
3199	struct vm_area_struct *vma = fe->vma;
3200	struct page *fault_page;
3201	int ret = 0;
3202
3203	/*
3204	* Let's call ->map_pages() first and use ->fault() as fallback
3205	* if page by the offset is not ready to be mapped (cold cache or
3206	* something).
3207	*/
3208	if (vma->vm_ops->map_pages && fault_around_bytes >> PAGE_SHIFT > 1) {
3209	ret = do_fault_around(fe, pgoff);
3210	if (ret)
3211	return ret;
3212	}
3213
3214	ret = __do_fault(fe, pgoff, NULL, &fault_page, NULL);
3215	if (unlikely(ret & (VM_FAULT_ERROR | VM_FAULT_NOPAGE | VM_FAULT_RETRY)))
3216	return ret;
3217
3218	ret |= alloc_set_pte(fe, NULL, fault_page);
3219	if (fe->pte)
3220	pte_unmap_unlock(fe->pte, fe->ptl);
3221	unlock_page(fault_page);
3222	if (unlikely(ret & (VM_FAULT_ERROR | VM_FAULT_NOPAGE | VM_FAULT_RETRY)))
3223	put_page(fault_page);
3224	return ret;
3225	}
3226
3227	static int do_cow_fault(struct fault_env *fe, pgoff_t pgoff)
3228	{
3229	struct vm_area_struct *vma = fe->vma;
3230	struct page *fault_page, *new_page;
3231	void *fault_entry;
3232	struct mem_cgroup *memcg;
3233	int ret;
3234
3235	if (unlikely(anon_vma_prepare(vma)))
3236	return VM_FAULT_OOM;
3237
3238	new_page = alloc_page_vma(GFP_HIGHUSER_MOVABLE, vma, fe->address);
3239	if (!new_page)
3240	return VM_FAULT_OOM;
3241
3242	if (mem_cgroup_try_charge(new_page, vma->vm_mm, GFP_KERNEL,
3243	&memcg, false)) {
3244	put_page(new_page);
3245	return VM_FAULT_OOM;
3246	}
3247
3248	ret = __do_fault(fe, pgoff, new_page, &fault_page, &fault_entry);
3249	if (unlikely(ret & (VM_FAULT_ERROR | VM_FAULT_NOPAGE | VM_FAULT_RETRY)))
3250	goto uncharge_out;
3251
3252	if (!(ret & VM_FAULT_DAX_LOCKED))
3253	copy_user_highpage(new_page, fault_page, fe->address, vma);
3254	__SetPageUptodate(new_page);
3255
3256	ret |= alloc_set_pte(fe, memcg, new_page);
3257	if (fe->pte)
3258	pte_unmap_unlock(fe->pte, fe->ptl);
3259	if (!(ret & VM_FAULT_DAX_LOCKED)) {
3260	unlock_page(fault_page);
3261	put_page(fault_page);
3262	} else {
3263	dax_unlock_mapping_entry(vma->vm_file->f_mapping, pgoff);
3264	}
3265	if (unlikely(ret & (VM_FAULT_ERROR | VM_FAULT_NOPAGE | VM_FAULT_RETRY)))
3266	goto uncharge_out;
3267	return ret;
3268	uncharge_out:
3269	mem_cgroup_cancel_charge(new_page, memcg, false);
3270	put_page(new_page);
3271	return ret;
3272	}
3273
3274	static int do_shared_fault(struct fault_env *fe, pgoff_t pgoff)
3275	{
3276	struct vm_area_struct *vma = fe->vma;
3277	struct page *fault_page;
3278	struct address_space *mapping;
3279	int dirtied = 0;
3280	int ret, tmp;
3281
3282	ret = __do_fault(fe, pgoff, NULL, &fault_page, NULL);
3283	if (unlikely(ret & (VM_FAULT_ERROR | VM_FAULT_NOPAGE | VM_FAULT_RETRY)))
3284	return ret;
3285
3286	/*
3287	* Check if the backing address space wants to know that the page is
3288	* about to become writable
3289	*/
3290	if (vma->vm_ops->page_mkwrite) {
3291	unlock_page(fault_page);
3292	tmp = do_page_mkwrite(vma, fault_page, fe->address);
3293	if (unlikely(!tmp ||
3294	(tmp & (VM_FAULT_ERROR | VM_FAULT_NOPAGE)))) {
3295	put_page(fault_page);
3296	return tmp;
3297	}
3298	}
3299
3300	ret |= alloc_set_pte(fe, NULL, fault_page);
3301	if (fe->pte)
3302	pte_unmap_unlock(fe->pte, fe->ptl);
3303	if (unlikely(ret & (VM_FAULT_ERROR | VM_FAULT_NOPAGE |
3304	VM_FAULT_RETRY))) {
3305	unlock_page(fault_page);
3306	put_page(fault_page);
3307	return ret;
3308	}
3309
3310	if (set_page_dirty(fault_page))
3311	dirtied = 1;
3312	/*
3313	* Take a local copy of the address_space - page.mapping may be zeroed
3314	* by truncate after unlock_page(). The address_space itself remains
3315	* pinned by vma->vm_file's reference. We rely on unlock_page()'s
3316	* release semantics to prevent the compiler from undoing this copying.
3317	*/
3318	mapping = page_rmapping(fault_page);
3319	unlock_page(fault_page);
3320	if ((dirtied || vma->vm_ops->page_mkwrite) && mapping) {
3321	/*
3322	* Some device drivers do not set page.mapping but still
3323	* dirty their pages
3324	*/
3325	balance_dirty_pages_ratelimited(mapping);
3326	}
3327
3328	if (!vma->vm_ops->page_mkwrite)
3329	file_update_time(vma->vm_file);
3330
3331	return ret;
3332	}
3333
3334	/*
3335	* We enter with non-exclusive mmap_sem (to exclude vma changes,
3336	* but allow concurrent faults).
3337	* The mmap_sem may have been released depending on flags and our
3338	* return value. See filemap_fault() and __lock_page_or_retry().
3339	*/
3340	static int do_fault(struct fault_env *fe)
3341	{
3342	struct vm_area_struct *vma = fe->vma;
3343	pgoff_t pgoff = linear_page_index(vma, fe->address);
3344	int ret;
3345
3346	/* The VMA was not fully populated on mmap() or missing VM_DONTEXPAND */
3347	if (!vma->vm_ops->fault)
3348	ret = VM_FAULT_SIGBUS;
3349	else if (!(fe->flags & FAULT_FLAG_WRITE))
3350	ret = do_read_fault(fe, pgoff);
3351	else if (!(vma->vm_flags & VM_SHARED))
3352	ret = do_cow_fault(fe, pgoff);
3353	else
3354	ret = do_shared_fault(fe, pgoff);
3355
3356	/* preallocated pagetable is unused: free it */
3357	if (fe->prealloc_pte) {
3358	pte_free(vma->vm_mm, fe->prealloc_pte);
3359	fe->prealloc_pte = 0;
3360	}
3361	return ret;
3362	}
3363
3364	static int numa_migrate_prep(struct page *page, struct vm_area_struct *vma,
3365	unsigned long addr, int page_nid,
3366	int *flags)
3367	{
3368	get_page(page);
3369
3370	count_vm_numa_event(NUMA_HINT_FAULTS);
3371	if (page_nid == numa_node_id()) {
3372	count_vm_numa_event(NUMA_HINT_FAULTS_LOCAL);
3373	*flags |= TNF_FAULT_LOCAL;
3374	}
3375
3376	return mpol_misplaced(page, vma, addr);
3377	}
3378
3379	static int do_numa_page(struct fault_env *fe, pte_t pte)
3380	{
3381	struct vm_area_struct *vma = fe->vma;
3382	struct page *page = NULL;
3383	int page_nid = -1;
3384	int last_cpupid;
3385	int target_nid;
3386	bool migrated = false;
3387	bool was_writable = pte_write(pte);
3388	int flags = 0;
3389
3390	/*
3391	* The "pte" at this point cannot be used safely without
3392	* validation through pte_unmap_same(). It's of NUMA type but
3393	* the pfn may be screwed if the read is non atomic.
3394	*
3395	* We can safely just do a "set_pte_at()", because the old
3396	* page table entry is not accessible, so there would be no
3397	* concurrent hardware modifications to the PTE.
3398	*/
3399	fe->ptl = pte_lockptr(vma->vm_mm, fe->pmd);
3400	spin_lock(fe->ptl);
3401	if (unlikely(!pte_same(*fe->pte, pte))) {
3402	pte_unmap_unlock(fe->pte, fe->ptl);
3403	goto out;
3404	}
3405
3406	/* Make it present again */
3407	pte = pte_modify(pte, vma->vm_page_prot);
3408	pte = pte_mkyoung(pte);
3409	if (was_writable)
3410	pte = pte_mkwrite(pte);
3411	set_pte_at(vma->vm_mm, fe->address, fe->pte, pte);
3412	update_mmu_cache(vma, fe->address, fe->pte);
3413
3414	page = vm_normal_page(vma, fe->address, pte);
3415	if (!page) {
3416	pte_unmap_unlock(fe->pte, fe->ptl);
3417	return 0;
3418	}
3419
3420	/* TODO: handle PTE-mapped THP */
3421	if (PageCompound(page)) {
3422	pte_unmap_unlock(fe->pte, fe->ptl);
3423	return 0;
3424	}
3425
3426	/*
3427	* Avoid grouping on RO pages in general. RO pages shouldn't hurt as
3428	* much anyway since they can be in shared cache state. This misses
3429	* the case where a mapping is writable but the process never writes
3430	* to it but pte_write gets cleared during protection updates and
3431	* pte_dirty has unpredictable behaviour between PTE scan updates,
3432	* background writeback, dirty balancing and application behaviour.
3433	*/
3434	if (!pte_write(pte))
3435	flags |= TNF_NO_GROUP;
3436
3437	/*
3438	* Flag if the page is shared between multiple address spaces. This
3439	* is later used when determining whether to group tasks together
3440	*/
3441	if (page_mapcount(page) > 1 && (vma->vm_flags & VM_SHARED))
3442	flags |= TNF_SHARED;
3443
3444	last_cpupid = page_cpupid_last(page);
3445	page_nid = page_to_nid(page);
3446	target_nid = numa_migrate_prep(page, vma, fe->address, page_nid,
3447	&flags);
3448	pte_unmap_unlock(fe->pte, fe->ptl);
3449	if (target_nid == -1) {
3450	put_page(page);
3451	goto out;
3452	}
3453
3454	/* Migrate to the requested node */
3455	migrated = migrate_misplaced_page(page, vma, target_nid);
3456	if (migrated) {
3457	page_nid = target_nid;
3458	flags |= TNF_MIGRATED;
3459	} else
3460	flags |= TNF_MIGRATE_FAIL;
3461
3462	out:
3463	if (page_nid != -1)
3464	task_numa_fault(last_cpupid, page_nid, 1, flags);
3465	return 0;
3466	}
3467
3468	static int create_huge_pmd(struct fault_env *fe)
3469	{
3470	struct vm_area_struct *vma = fe->vma;
3471	if (vma_is_anonymous(vma))
3472	return do_huge_pmd_anonymous_page(fe);
3473	if (vma->vm_ops->pmd_fault)
3474	return vma->vm_ops->pmd_fault(vma, fe->address, fe->pmd,
3475	fe->flags);
3476	return VM_FAULT_FALLBACK;
3477	}
3478
3479	static int wp_huge_pmd(struct fault_env *fe, pmd_t orig_pmd)
3480	{
3481	if (vma_is_anonymous(fe->vma))
3482	return do_huge_pmd_wp_page(fe, orig_pmd);
3483	if (fe->vma->vm_ops->pmd_fault)
3484	return fe->vma->vm_ops->pmd_fault(fe->vma, fe->address, fe->pmd,
3485	fe->flags);
3486
3487	/* COW handled on pte level: split pmd */
3488	VM_BUG_ON_VMA(fe->vma->vm_flags & VM_SHARED, fe->vma);
3489	split_huge_pmd(fe->vma, fe->pmd, fe->address);
3490
3491	return VM_FAULT_FALLBACK;
3492	}
3493
3494	static inline bool vma_is_accessible(struct vm_area_struct *vma)
3495	{
3496	return vma->vm_flags & (VM_READ | VM_EXEC | VM_WRITE);
3497	}
3498
3499	/*
3500	* These routines also need to handle stuff like marking pages dirty
3501	* and/or accessed for architectures that don't do it in hardware (most
3502	* RISC architectures). The early dirtying is also good on the i386.
3503	*
3504	* There is also a hook called "update_mmu_cache()" that architectures
3505	* with external mmu caches can use to update those (ie the Sparc or
3506	* PowerPC hashed page tables that act as extended TLBs).
3507	*
3508	* We enter with non-exclusive mmap_sem (to exclude vma changes, but allow
3509	* concurrent faults).
3510	*
3511	* The mmap_sem may have been released depending on flags and our return value.
3512	* See filemap_fault() and __lock_page_or_retry().
3513	*/
3514	static int handle_pte_fault(struct fault_env *fe)
3515	{
3516	pte_t entry;
3517
3518	if (unlikely(pmd_none(*fe->pmd))) {
3519	/*
3520	* Leave __pte_alloc() until later: because vm_ops->fault may
3521	* want to allocate huge page, and if we expose page table
3522	* for an instant, it will be difficult to retract from
3523	* concurrent faults and from rmap lookups.
3524	*/
3525	fe->pte = NULL;
3526	} else {
3527	/* See comment in pte_alloc_one_map() */
3528	if (pmd_devmap_trans_unstable(fe->pmd))
3529	return 0;
3530	/*
3531	* A regular pmd is established and it can't morph into a huge
3532	* pmd from under us anymore at this point because we hold the
3533	* mmap_sem read mode and khugepaged takes it in write mode.
3534	* So now it's safe to run pte_offset_map().
3535	*/
3536	fe->pte = pte_offset_map(fe->pmd, fe->address);
3537
3538	entry = *fe->pte;
3539
3540	/*
3541	* some architectures can have larger ptes than wordsize,
3542	* e.g.ppc44x-defconfig has CONFIG_PTE_64BIT=y and
3543	* CONFIG_32BIT=y, so READ_ONCE or ACCESS_ONCE cannot guarantee
3544	* atomic accesses. The code below just needs a consistent
3545	* view for the ifs and we later double check anyway with the
3546	* ptl lock held. So here a barrier will do.
3547	*/
3548	barrier();
3549	if (pte_none(entry)) {
3550	pte_unmap(fe->pte);
3551	fe->pte = NULL;
3552	}
3553	}
3554
3555	if (!fe->pte) {
3556	if (vma_is_anonymous(fe->vma))
3557	return do_anonymous_page(fe);
3558	else
3559	return do_fault(fe);
3560	}
3561
3562	if (!pte_present(entry))
3563	return do_swap_page(fe, entry);
3564
3565	if (pte_protnone(entry) && vma_is_accessible(fe->vma))
3566	return do_numa_page(fe, entry);
3567
3568	fe->ptl = pte_lockptr(fe->vma->vm_mm, fe->pmd);
3569	spin_lock(fe->ptl);
3570	if (unlikely(!pte_same(*fe->pte, entry)))
3571	goto unlock;
3572	if (fe->flags & FAULT_FLAG_WRITE) {
3573	if (!pte_write(entry))
3574	return do_wp_page(fe, entry);
3575	entry = pte_mkdirty(entry);
3576	}
3577	entry = pte_mkyoung(entry);
3578	if (ptep_set_access_flags(fe->vma, fe->address, fe->pte, entry,
3579	fe->flags & FAULT_FLAG_WRITE)) {
3580	update_mmu_cache(fe->vma, fe->address, fe->pte);
3581	} else {
3582	/*
3583	* This is needed only for protection faults but the arch code
3584	* is not yet telling us if this is a protection fault or not.
3585	* This still avoids useless tlb flushes for .text page faults
3586	* with threads.
3587	*/
3588	if (fe->flags & FAULT_FLAG_WRITE)
3589	flush_tlb_fix_spurious_fault(fe->vma, fe->address);
3590	}
3591	unlock:
3592	pte_unmap_unlock(fe->pte, fe->ptl);
3593	return 0;
3594	}
3595
3596	/*
3597	* By the time we get here, we already hold the mm semaphore
3598	*
3599	* The mmap_sem may have been released depending on flags and our
3600	* return value. See filemap_fault() and __lock_page_or_retry().
3601	*/
3602	static int __handle_mm_fault(struct vm_area_struct *vma, unsigned long address,
3603	unsigned int flags)
3604	{
3605	struct fault_env fe = {
3606	.vma = vma,
3607	.address = address,
3608	.flags = flags,
3609	};
3610	struct mm_struct *mm = vma->vm_mm;
3611	pgd_t *pgd;
3612	pud_t *pud;
3613
3614	pgd = pgd_offset(mm, address);
3615	pud = pud_alloc(mm, pgd, address);
3616	if (!pud)
3617	return VM_FAULT_OOM;
3618	fe.pmd = pmd_alloc(mm, pud, address);
3619	if (!fe.pmd)
3620	return VM_FAULT_OOM;
3621	if (pmd_none(*fe.pmd) && transparent_hugepage_enabled(vma)) {
3622	int ret = create_huge_pmd(&fe);
3623	if (!(ret & VM_FAULT_FALLBACK))
3624	return ret;
3625	} else {
3626	pmd_t orig_pmd = *fe.pmd;
3627	int ret;
3628
3629	barrier();
3630	if (pmd_trans_huge(orig_pmd) || pmd_devmap(orig_pmd)) {
3631	if (pmd_protnone(orig_pmd) && vma_is_accessible(vma))
3632	return do_huge_pmd_numa_page(&fe, orig_pmd);
3633
3634	if ((fe.flags & FAULT_FLAG_WRITE) &&
3635	!pmd_write(orig_pmd)) {
3636	ret = wp_huge_pmd(&fe, orig_pmd);
3637	if (!(ret & VM_FAULT_FALLBACK))
3638	return ret;
3639	} else {
3640	huge_pmd_set_accessed(&fe, orig_pmd);
3641	return 0;
3642	}
3643	}
3644	}
3645
3646	return handle_pte_fault(&fe);
3647	}
3648
3649	/*
3650	* By the time we get here, we already hold the mm semaphore
3651	*
3652	* The mmap_sem may have been released depending on flags and our
3653	* return value. See filemap_fault() and __lock_page_or_retry().
3654	*/
3655	int handle_mm_fault(struct vm_area_struct *vma, unsigned long address,
3656	unsigned int flags)
3657	{
3658	int ret;
3659
3660	__set_current_state(TASK_RUNNING);
3661
3662	count_vm_event(PGFAULT);
3663	mem_cgroup_count_vm_event(vma->vm_mm, PGFAULT);
3664
3665	/* do counter updates before entering really critical section. */
3666	check_sync_rss_stat(current);
3667
3668	if (!arch_vma_access_permitted(vma, flags & FAULT_FLAG_WRITE,
3669	flags & FAULT_FLAG_INSTRUCTION,
3670	flags & FAULT_FLAG_REMOTE))
3671	return VM_FAULT_SIGSEGV;
3672
3673	/*
3674	* Enable the memcg OOM handling for faults triggered in user
3675	* space. Kernel faults are handled more gracefully.
3676	*/
3677	if (flags & FAULT_FLAG_USER)
3678	mem_cgroup_oom_enable();
3679
3680	if (unlikely(is_vm_hugetlb_page(vma)))
3681	ret = hugetlb_fault(vma->vm_mm, vma, address, flags);
3682	else
3683	ret = __handle_mm_fault(vma, address, flags);
3684
3685	if (flags & FAULT_FLAG_USER) {
3686	mem_cgroup_oom_disable();
3687	/*
3688	* The task may have entered a memcg OOM situation but
3689	* if the allocation error was handled gracefully (no
3690	* VM_FAULT_OOM), there is no need to kill anything.
3691	* Just clean up the OOM state peacefully.
3692	*/
3693	if (task_in_memcg_oom(current) && !(ret & VM_FAULT_OOM))
3694	mem_cgroup_oom_synchronize(false);
3695	}
3696
3697	/*
3698	* This mm has been already reaped by the oom reaper and so the
3699	* refault cannot be trusted in general. Anonymous refaults would
3700	* lose data and give a zero page instead e.g. This is especially
3701	* problem for use_mm() because regular tasks will just die and
3702	* the corrupted data will not be visible anywhere while kthread
3703	* will outlive the oom victim and potentially propagate the date
3704	* further.
3705	*/
3706	if (unlikely((current->flags & PF_KTHREAD) && !(ret & VM_FAULT_ERROR)
3707	&& test_bit(MMF_UNSTABLE, &vma->vm_mm->flags))) {
3708
3709	/*
3710	* We are going to enforce SIGBUS but the PF path might have
3711	* dropped the mmap_sem already so take it again so that
3712	* we do not break expectations of all arch specific PF paths
3713	* and g-u-p
3714	*/
3715	if (ret & VM_FAULT_RETRY)
3716	down_read(&vma->vm_mm->mmap_sem);
3717	ret = VM_FAULT_SIGBUS;
3718	}
3719
3720	return ret;
3721	}
3722	EXPORT_SYMBOL_GPL(handle_mm_fault);
3723
3724	#ifndef __PAGETABLE_PUD_FOLDED
3725	/*
3726	* Allocate page upper directory.
3727	* We've already handled the fast-path in-line.
3728	*/
3729	int __pud_alloc(struct mm_struct *mm, pgd_t *pgd, unsigned long address)
3730	{
3731	pud_t *new = pud_alloc_one(mm, address);
3732	if (!new)
3733	return -ENOMEM;
3734
3735	smp_wmb(); /* See comment in __pte_alloc */
3736
3737	spin_lock(&mm->page_table_lock);
3738	if (pgd_present(*pgd)) /* Another has populated it */
3739	pud_free(mm, new);
3740	else
3741	pgd_populate(mm, pgd, new);
3742	spin_unlock(&mm->page_table_lock);
3743	return 0;
3744	}
3745	#endif /* __PAGETABLE_PUD_FOLDED */
3746
3747	#ifndef __PAGETABLE_PMD_FOLDED
3748	/*
3749	* Allocate page middle directory.
3750	* We've already handled the fast-path in-line.
3751	*/
3752	int __pmd_alloc(struct mm_struct *mm, pud_t *pud, unsigned long address)
3753	{
3754	pmd_t *new = pmd_alloc_one(mm, address);
3755	if (!new)
3756	return -ENOMEM;
3757
3758	smp_wmb(); /* See comment in __pte_alloc */
3759
3760	spin_lock(&mm->page_table_lock);
3761	#ifndef __ARCH_HAS_4LEVEL_HACK
3762	if (!pud_present(*pud)) {
3763	mm_inc_nr_pmds(mm);
3764	pud_populate(mm, pud, new);
3765	} else /* Another has populated it */
3766	pmd_free(mm, new);
3767	#else
3768	if (!pgd_present(*pud)) {
3769	mm_inc_nr_pmds(mm);
3770	pgd_populate(mm, pud, new);
3771	} else /* Another has populated it */
3772	pmd_free(mm, new);
3773	#endif /* __ARCH_HAS_4LEVEL_HACK */
3774	spin_unlock(&mm->page_table_lock);
3775	return 0;
3776	}
3777	#endif /* __PAGETABLE_PMD_FOLDED */
3778
3779	static int __follow_pte(struct mm_struct *mm, unsigned long address,
3780	pte_t **ptepp, spinlock_t **ptlp)
3781	{
3782	pgd_t *pgd;
3783	pud_t *pud;
3784	pmd_t *pmd;
3785	pte_t *ptep;
3786
3787	pgd = pgd_offset(mm, address);
3788	if (pgd_none(*pgd) || unlikely(pgd_bad(*pgd)))
3789	goto out;
3790
3791	pud = pud_offset(pgd, address);
3792	if (pud_none(*pud) || unlikely(pud_bad(*pud)))
3793	goto out;
3794
3795	pmd = pmd_offset(pud, address);
3796	VM_BUG_ON(pmd_trans_huge(*pmd));
3797	if (pmd_none(*pmd) || unlikely(pmd_bad(*pmd)))
3798	goto out;
3799
3800	/* We cannot handle huge page PFN maps. Luckily they don't exist. */
3801	if (pmd_huge(*pmd))
3802	goto out;
3803
3804	ptep = pte_offset_map_lock(mm, pmd, address, ptlp);
3805	if (!ptep)
3806	goto out;
3807	if (!pte_present(*ptep))
3808	goto unlock;
3809	*ptepp = ptep;
3810	return 0;
3811	unlock:
3812	pte_unmap_unlock(ptep, *ptlp);
3813	out:
3814	return -EINVAL;
3815	}
3816
3817	static inline int follow_pte(struct mm_struct *mm, unsigned long address,
3818	pte_t **ptepp, spinlock_t **ptlp)
3819	{
3820	int res;
3821
3822	/* (void) is needed to make gcc happy */
3823	(void) __cond_lock(*ptlp,
3824	!(res = __follow_pte(mm, address, ptepp, ptlp)));
3825	return res;
3826	}
3827
3828	/**
3829	* follow_pfn - look up PFN at a user virtual address
3830	* @vma: memory mapping
3831	* @address: user virtual address
3832	* @pfn: location to store found PFN
3833	*
3834	* Only IO mappings and raw PFN mappings are allowed.
3835	*
3836	* Returns zero and the pfn at @pfn on success, -ve otherwise.
3837	*/
3838	int follow_pfn(struct vm_area_struct *vma, unsigned long address,
3839	unsigned long *pfn)
3840	{
3841	int ret = -EINVAL;
3842	spinlock_t *ptl;
3843	pte_t *ptep;
3844
3845	if (!(vma->vm_flags & (VM_IO | VM_PFNMAP)))
3846	return ret;
3847
3848	ret = follow_pte(vma->vm_mm, address, &ptep, &ptl);
3849	if (ret)
3850	return ret;
3851	*pfn = pte_pfn(*ptep);
3852	pte_unmap_unlock(ptep, ptl);
3853	return 0;
3854	}
3855	EXPORT_SYMBOL(follow_pfn);
3856
3857	#ifdef CONFIG_HAVE_IOREMAP_PROT
3858	int follow_phys(struct vm_area_struct *vma,
3859	unsigned long address, unsigned int flags,
3860	unsigned long *prot, resource_size_t *phys)
3861	{
3862	int ret = -EINVAL;
3863	pte_t *ptep, pte;
3864	spinlock_t *ptl;
3865
3866	if (!(vma->vm_flags & (VM_IO | VM_PFNMAP)))
3867	goto out;
3868
3869	if (follow_pte(vma->vm_mm, address, &ptep, &ptl))
3870	goto out;
3871	pte = *ptep;
3872
3873	if ((flags & FOLL_WRITE) && !pte_write(pte))
3874	goto unlock;
3875
3876	*prot = pgprot_val(pte_pgprot(pte));
3877	*phys = (resource_size_t)pte_pfn(pte) << PAGE_SHIFT;
3878
3879	ret = 0;
3880	unlock:
3881	pte_unmap_unlock(ptep, ptl);
3882	out:
3883	return ret;
3884	}
3885
3886	int generic_access_phys(struct vm_area_struct *vma, unsigned long addr,
3887	void *buf, int len, int write)
3888	{
3889	resource_size_t phys_addr;
3890	unsigned long prot = 0;
3891	void __iomem *maddr;
3892	int offset = addr & (PAGE_SIZE-1);
3893
3894	if (follow_phys(vma, addr, write, &prot, &phys_addr))
3895	return -EINVAL;
3896
3897	maddr = ioremap_prot(phys_addr, PAGE_ALIGN(len + offset), prot);
3898	if (!maddr)
3899	return -ENOMEM;
3900
3901	if (write)
3902	memcpy_toio(maddr + offset, buf, len);
3903	else
3904	memcpy_fromio(buf, maddr + offset, len);
3905	iounmap(maddr);
3906
3907	return len;
3908	}
3909	EXPORT_SYMBOL_GPL(generic_access_phys);
3910	#endif
3911
3912	/*
3913	* Access another process' address space as given in mm. If non-NULL, use the
3914	* given task for page fault accounting.
3915	*/
3916	int __access_remote_vm(struct task_struct *tsk, struct mm_struct *mm,
3917	unsigned long addr, void *buf, int len, unsigned int gup_flags)
3918	{
3919	struct vm_area_struct *vma;
3920	void *old_buf = buf;
3921	int write = gup_flags & FOLL_WRITE;
3922
3923	down_read(&mm->mmap_sem);
3924	/* ignore errors, just check how much was successfully transferred */
3925	while (len) {
3926	int bytes, ret, offset;
3927	void *maddr;
3928	struct page *page = NULL;
3929
3930	ret = get_user_pages_remote(tsk, mm, addr, 1,
3931	gup_flags, &page, &vma);
3932	if (ret <= 0) {
3933	#ifndef CONFIG_HAVE_IOREMAP_PROT
3934	break;
3935	#else
3936	/*
3937	* Check if this is a VM_IO | VM_PFNMAP VMA, which
3938	* we can access using slightly different code.
3939	*/
3940	vma = find_vma(mm, addr);
3941	if (!vma || vma->vm_start > addr)
3942	break;
3943	if (vma->vm_ops && vma->vm_ops->access)
3944	ret = vma->vm_ops->access(vma, addr, buf,
3945	len, write);
3946	if (ret <= 0)
3947	break;
3948	bytes = ret;
3949	#endif
3950	} else {
3951	bytes = len;
3952	offset = addr & (PAGE_SIZE-1);
3953	if (bytes > PAGE_SIZE-offset)
3954	bytes = PAGE_SIZE-offset;
3955
3956	maddr = kmap(page);
3957	if (write) {
3958	copy_to_user_page(vma, page, addr,
3959	maddr + offset, buf, bytes);
3960	set_page_dirty_lock(page);
3961	} else {
3962	copy_from_user_page(vma, page, addr,
3963	buf, maddr + offset, bytes);
3964	}
3965	kunmap(page);
3966	put_page(page);
3967	}
3968	len -= bytes;
3969	buf += bytes;
3970	addr += bytes;
3971	}
3972	up_read(&mm->mmap_sem);
3973
3974	return buf - old_buf;
3975	}
3976
3977	/**
3978	* access_remote_vm - access another process' address space
3979	* @mm: the mm_struct of the target address space
3980	* @addr: start address to access
3981	* @buf: source or destination buffer
3982	* @len: number of bytes to transfer
3983	* @gup_flags: flags modifying lookup behaviour
3984	*
3985	* The caller must hold a reference on @mm.
3986	*/
3987	int access_remote_vm(struct mm_struct *mm, unsigned long addr,
3988	void *buf, int len, unsigned int gup_flags)
3989	{
3990	return __access_remote_vm(NULL, mm, addr, buf, len, gup_flags);
3991	}
3992
3993	/*
3994	* Access another process' address space.
3995	* Source/target buffer must be kernel space,
3996	* Do not walk the page table directly, use get_user_pages
3997	*/
3998	int access_process_vm(struct task_struct *tsk, unsigned long addr,
3999	void *buf, int len, unsigned int gup_flags)
4000	{
4001	struct mm_struct *mm;
4002	int ret;
4003
4004	mm = get_task_mm(tsk);
4005	if (!mm)
4006	return 0;
4007
4008	ret = __access_remote_vm(tsk, mm, addr, buf, len, gup_flags);
4009
4010	mmput(mm);
4011
4012	return ret;
4013	}
4014
4015	/*
4016	* Print the name of a VMA.
4017	*/
4018	void print_vma_addr(char *prefix, unsigned long ip)
4019	{
4020	struct mm_struct *mm = current->mm;
4021	struct vm_area_struct *vma;
4022
4023	/*
4024	* Do not print if we are in atomic
4025	* contexts (in exception stacks, etc.):
4026	*/
4027	if (preempt_count())
4028	return;
4029
4030	down_read(&mm->mmap_sem);
4031	vma = find_vma(mm, ip);
4032	if (vma && vma->vm_file) {
4033	struct file *f = vma->vm_file;
4034	char *buf = (char *)__get_free_page(GFP_KERNEL);
4035	if (buf) {
4036	char *p;
4037
4038	p = file_path(f, buf, PAGE_SIZE);
4039	if (IS_ERR(p))
4040	p = "?";
4041	printk("%s%s[%lx+%lx]", prefix, kbasename(p),
4042	vma->vm_start,
4043	vma->vm_end - vma->vm_start);
4044	free_page((unsigned long)buf);
4045	}
4046	}
4047	up_read(&mm->mmap_sem);
4048	}
4049
4050	#if defined(CONFIG_PROVE_LOCKING) || defined(CONFIG_DEBUG_ATOMIC_SLEEP)
4051	void __might_fault(const char *file, int line)
4052	{
4053	/*
4054	* Some code (nfs/sunrpc) uses socket ops on kernel memory while
4055	* holding the mmap_sem, this is safe because kernel memory doesn't
4056	* get paged out, therefore we'll never actually fault, and the
4057	* below annotations will generate false positives.
4058	*/
4059	if (segment_eq(get_fs(), KERNEL_DS))
4060	return;
4061	if (pagefault_disabled())
4062	return;
4063	__might_sleep(file, line, 0);
4064	#if defined(CONFIG_DEBUG_ATOMIC_SLEEP)
4065	if (current->mm)
4066	might_lock_read(&current->mm->mmap_sem);
4067	#endif
4068	}
4069	EXPORT_SYMBOL(__might_fault);
4070	#endif
4071
4072	#if defined(CONFIG_TRANSPARENT_HUGEPAGE) || defined(CONFIG_HUGETLBFS)
4073	static void clear_gigantic_page(struct page *page,
4074	unsigned long addr,
4075	unsigned int pages_per_huge_page)
4076	{
4077	int i;
4078	struct page *p = page;
4079
4080	might_sleep();
4081	for (i = 0; i < pages_per_huge_page;
4082	i++, p = mem_map_next(p, page, i)) {
4083	cond_resched();
4084	clear_user_highpage(p, addr + i * PAGE_SIZE);
4085	}
4086	}
4087	void clear_huge_page(struct page *page,
4088	unsigned long addr, unsigned int pages_per_huge_page)
4089	{
4090	int i;
4091
4092	if (unlikely(pages_per_huge_page > MAX_ORDER_NR_PAGES)) {
4093	clear_gigantic_page(page, addr, pages_per_huge_page);
4094	return;
4095	}
4096
4097	might_sleep();
4098	for (i = 0; i < pages_per_huge_page; i++) {
4099	cond_resched();
4100	clear_user_highpage(page + i, addr + i * PAGE_SIZE);
4101	}
4102	}
4103
4104	static void copy_user_gigantic_page(struct page *dst, struct page *src,
4105	unsigned long addr,
4106	struct vm_area_struct *vma,
4107	unsigned int pages_per_huge_page)
4108	{
4109	int i;
4110	struct page *dst_base = dst;
4111	struct page *src_base = src;
4112
4113	for (i = 0; i < pages_per_huge_page; ) {
4114	cond_resched();
4115	copy_user_highpage(dst, src, addr + i*PAGE_SIZE, vma);
4116
4117	i++;
4118	dst = mem_map_next(dst, dst_base, i);
4119	src = mem_map_next(src, src_base, i);
4120	}
4121	}
4122
4123	void copy_user_huge_page(struct page *dst, struct page *src,
4124	unsigned long addr, struct vm_area_struct *vma,
4125	unsigned int pages_per_huge_page)
4126	{
4127	int i;
4128
4129	if (unlikely(pages_per_huge_page > MAX_ORDER_NR_PAGES)) {
4130	copy_user_gigantic_page(dst, src, addr, vma,
4131	pages_per_huge_page);
4132	return;
4133	}
4134
4135	might_sleep();
4136	for (i = 0; i < pages_per_huge_page; i++) {
4137	cond_resched();
4138	copy_user_highpage(dst + i, src + i, addr + i*PAGE_SIZE, vma);
4139	}
4140	}
4141	#endif /* CONFIG_TRANSPARENT_HUGEPAGE || CONFIG_HUGETLBFS */
4142
4143	#if USE_SPLIT_PTE_PTLOCKS && ALLOC_SPLIT_PTLOCKS
4144
4145	static struct kmem_cache *page_ptl_cachep;
4146
4147	void __init ptlock_cache_init(void)
4148	{
4149	page_ptl_cachep = kmem_cache_create("page->ptl", sizeof(spinlock_t), 0,
4150	SLAB_PANIC, NULL);
4151	}
4152
4153	bool ptlock_alloc(struct page *page)
4154	{
4155	spinlock_t *ptl;
4156
4157	ptl = kmem_cache_alloc(page_ptl_cachep, GFP_KERNEL);
4158	if (!ptl)
4159	return false;
4160	page->ptl = ptl;
4161	return true;
4162	}
4163
4164	void ptlock_free(struct page *page)
4165	{
4166	kmem_cache_free(page_ptl_cachep, page->ptl);
4167	}
4168	#endif
4169