path: root/mm/rmap.c (plain)
blob: a7276d8c96f33f95ef7743e6e7247ecb83c07861

1	/*
2	* mm/rmap.c - physical to virtual reverse mappings
3	*
4	* Copyright 2001, Rik van Riel <riel@conectiva.com.br>
5	* Released under the General Public License (GPL).
6	*
7	* Simple, low overhead reverse mapping scheme.
8	* Please try to keep this thing as modular as possible.
9	*
10	* Provides methods for unmapping each kind of mapped page:
11	* the anon methods track anonymous pages, and
12	* the file methods track pages belonging to an inode.
13	*
14	* Original design by Rik van Riel <riel@conectiva.com.br> 2001
15	* File methods by Dave McCracken <dmccr@us.ibm.com> 2003, 2004
16	* Anonymous methods by Andrea Arcangeli <andrea@suse.de> 2004
17	* Contributions by Hugh Dickins 2003, 2004
18	*/
19
20	/*
21	* Lock ordering in mm:
22	*
23	* inode->i_mutex (while writing or truncating, not reading or faulting)
24	* mm->mmap_sem
25	* page->flags PG_locked (lock_page)
26	* hugetlbfs_i_mmap_rwsem_key (in huge_pmd_share)
27	* mapping->i_mmap_rwsem
28	* anon_vma->rwsem
29	* mm->page_table_lock or pte_lock
30	* zone_lru_lock (in mark_page_accessed, isolate_lru_page)
31	* swap_lock (in swap_duplicate, swap_info_get)
32	* mmlist_lock (in mmput, drain_mmlist and others)
33	* mapping->private_lock (in __set_page_dirty_buffers)
34	* mem_cgroup_{begin,end}_page_stat (memcg->move_lock)
35	* mapping->tree_lock (widely used)
36	* inode->i_lock (in set_page_dirty's __mark_inode_dirty)
37	* bdi.wb->list_lock (in set_page_dirty's __mark_inode_dirty)
38	* sb_lock (within inode_lock in fs/fs-writeback.c)
39	* mapping->tree_lock (widely used, in set_page_dirty,
40	* in arch-dependent flush_dcache_mmap_lock,
41	* within bdi.wb->list_lock in __sync_single_inode)
42	*
43	* anon_vma->rwsem,mapping->i_mutex (memory_failure, collect_procs_anon)
44	* ->tasklist_lock
45	* pte map lock
46	*/
47
48	#include <linux/mm.h>
49	#include <linux/pagemap.h>
50	#include <linux/swap.h>
51	#include <linux/swapops.h>
52	#include <linux/slab.h>
53	#include <linux/init.h>
54	#include <linux/ksm.h>
55	#include <linux/rmap.h>
56	#include <linux/rcupdate.h>
57	#include <linux/export.h>
58	#include <linux/memcontrol.h>
59	#include <linux/mmu_notifier.h>
60	#include <linux/migrate.h>
61	#include <linux/hugetlb.h>
62	#include <linux/backing-dev.h>
63	#include <linux/page_idle.h>
64
65	#include <asm/tlbflush.h>
66
67	#include <trace/events/tlb.h>
68
69	#include "internal.h"
70
71	static struct kmem_cache *anon_vma_cachep;
72	static struct kmem_cache *anon_vma_chain_cachep;
73
74	static inline struct anon_vma *anon_vma_alloc(void)
75	{
76	struct anon_vma *anon_vma;
77
78	anon_vma = kmem_cache_alloc(anon_vma_cachep, GFP_KERNEL);
79	if (anon_vma) {
80	atomic_set(&anon_vma->refcount, 1);
81	anon_vma->degree = 1; /* Reference for first vma */
82	anon_vma->parent = anon_vma;
83	/*
84	* Initialise the anon_vma root to point to itself. If called
85	* from fork, the root will be reset to the parents anon_vma.
86	*/
87	anon_vma->root = anon_vma;
88	}
89
90	return anon_vma;
91	}
92
93	static inline void anon_vma_free(struct anon_vma *anon_vma)
94	{
95	VM_BUG_ON(atomic_read(&anon_vma->refcount));
96
97	/*
98	* Synchronize against page_lock_anon_vma_read() such that
99	* we can safely hold the lock without the anon_vma getting
100	* freed.
101	*
102	* Relies on the full mb implied by the atomic_dec_and_test() from
103	* put_anon_vma() against the acquire barrier implied by
104	* down_read_trylock() from page_lock_anon_vma_read(). This orders:
105	*
106	* page_lock_anon_vma_read() VS put_anon_vma()
107	* down_read_trylock() atomic_dec_and_test()
108	* LOCK MB
109	* atomic_read() rwsem_is_locked()
110	*
111	* LOCK should suffice since the actual taking of the lock must
112	* happen _before_ what follows.
113	*/
114	might_sleep();
115	if (rwsem_is_locked(&anon_vma->root->rwsem)) {
116	anon_vma_lock_write(anon_vma);
117	anon_vma_unlock_write(anon_vma);
118	}
119
120	kmem_cache_free(anon_vma_cachep, anon_vma);
121	}
122
123	static inline struct anon_vma_chain *anon_vma_chain_alloc(gfp_t gfp)
124	{
125	return kmem_cache_alloc(anon_vma_chain_cachep, gfp);
126	}
127
128	static void anon_vma_chain_free(struct anon_vma_chain *anon_vma_chain)
129	{
130	kmem_cache_free(anon_vma_chain_cachep, anon_vma_chain);
131	}
132
133	static void anon_vma_chain_link(struct vm_area_struct *vma,
134	struct anon_vma_chain *avc,
135	struct anon_vma *anon_vma)
136	{
137	avc->vma = vma;
138	avc->anon_vma = anon_vma;
139	list_add(&avc->same_vma, &vma->anon_vma_chain);
140	anon_vma_interval_tree_insert(avc, &anon_vma->rb_root);
141	}
142
143	/**
144	* anon_vma_prepare - attach an anon_vma to a memory region
145	* @vma: the memory region in question
146	*
147	* This makes sure the memory mapping described by 'vma' has
148	* an 'anon_vma' attached to it, so that we can associate the
149	* anonymous pages mapped into it with that anon_vma.
150	*
151	* The common case will be that we already have one, but if
152	* not we either need to find an adjacent mapping that we
153	* can re-use the anon_vma from (very common when the only
154	* reason for splitting a vma has been mprotect()), or we
155	* allocate a new one.
156	*
157	* Anon-vma allocations are very subtle, because we may have
158	* optimistically looked up an anon_vma in page_lock_anon_vma_read()
159	* and that may actually touch the spinlock even in the newly
160	* allocated vma (it depends on RCU to make sure that the
161	* anon_vma isn't actually destroyed).
162	*
163	* As a result, we need to do proper anon_vma locking even
164	* for the new allocation. At the same time, we do not want
165	* to do any locking for the common case of already having
166	* an anon_vma.
167	*
168	* This must be called with the mmap_sem held for reading.
169	*/
170	int anon_vma_prepare(struct vm_area_struct *vma)
171	{
172	struct anon_vma *anon_vma = vma->anon_vma;
173	struct anon_vma_chain *avc;
174
175	might_sleep();
176	if (unlikely(!anon_vma)) {
177	struct mm_struct *mm = vma->vm_mm;
178	struct anon_vma *allocated;
179
180	avc = anon_vma_chain_alloc(GFP_KERNEL);
181	if (!avc)
182	goto out_enomem;
183
184	anon_vma = find_mergeable_anon_vma(vma);
185	allocated = NULL;
186	if (!anon_vma) {
187	anon_vma = anon_vma_alloc();
188	if (unlikely(!anon_vma))
189	goto out_enomem_free_avc;
190	allocated = anon_vma;
191	}
192
193	anon_vma_lock_write(anon_vma);
194	/* page_table_lock to protect against threads */
195	spin_lock(&mm->page_table_lock);
196	if (likely(!vma->anon_vma)) {
197	vma->anon_vma = anon_vma;
198	anon_vma_chain_link(vma, avc, anon_vma);
199	/* vma reference or self-parent link for new root */
200	anon_vma->degree++;
201	allocated = NULL;
202	avc = NULL;
203	}
204	spin_unlock(&mm->page_table_lock);
205	anon_vma_unlock_write(anon_vma);
206
207	if (unlikely(allocated))
208	put_anon_vma(allocated);
209	if (unlikely(avc))
210	anon_vma_chain_free(avc);
211	}
212	return 0;
213
214	out_enomem_free_avc:
215	anon_vma_chain_free(avc);
216	out_enomem:
217	return -ENOMEM;
218	}
219
220	/*
221	* This is a useful helper function for locking the anon_vma root as
222	* we traverse the vma->anon_vma_chain, looping over anon_vma's that
223	* have the same vma.
224	*
225	* Such anon_vma's should have the same root, so you'd expect to see
226	* just a single mutex_lock for the whole traversal.
227	*/
228	static inline struct anon_vma *lock_anon_vma_root(struct anon_vma *root, struct anon_vma *anon_vma)
229	{
230	struct anon_vma *new_root = anon_vma->root;
231	if (new_root != root) {
232	if (WARN_ON_ONCE(root))
233	up_write(&root->rwsem);
234	root = new_root;
235	down_write(&root->rwsem);
236	}
237	return root;
238	}
239
240	static inline void unlock_anon_vma_root(struct anon_vma *root)
241	{
242	if (root)
243	up_write(&root->rwsem);
244	}
245
246	/*
247	* Attach the anon_vmas from src to dst.
248	* Returns 0 on success, -ENOMEM on failure.
249	*
250	* If dst->anon_vma is NULL this function tries to find and reuse existing
251	* anon_vma which has no vmas and only one child anon_vma. This prevents
252	* degradation of anon_vma hierarchy to endless linear chain in case of
253	* constantly forking task. On the other hand, an anon_vma with more than one
254	* child isn't reused even if there was no alive vma, thus rmap walker has a
255	* good chance of avoiding scanning the whole hierarchy when it searches where
256	* page is mapped.
257	*/
258	int anon_vma_clone(struct vm_area_struct *dst, struct vm_area_struct *src)
259	{
260	struct anon_vma_chain *avc, *pavc;
261	struct anon_vma *root = NULL;
262
263	list_for_each_entry_reverse(pavc, &src->anon_vma_chain, same_vma) {
264	struct anon_vma *anon_vma;
265
266	avc = anon_vma_chain_alloc(GFP_NOWAIT | __GFP_NOWARN);
267	if (unlikely(!avc)) {
268	unlock_anon_vma_root(root);
269	root = NULL;
270	avc = anon_vma_chain_alloc(GFP_KERNEL);
271	if (!avc)
272	goto enomem_failure;
273	}
274	anon_vma = pavc->anon_vma;
275	root = lock_anon_vma_root(root, anon_vma);
276	anon_vma_chain_link(dst, avc, anon_vma);
277
278	/*
279	* Reuse existing anon_vma if its degree lower than two,
280	* that means it has no vma and only one anon_vma child.
281	*
282	* Do not chose parent anon_vma, otherwise first child
283	* will always reuse it. Root anon_vma is never reused:
284	* it has self-parent reference and at least one child.
285	*/
286	if (!dst->anon_vma && anon_vma != src->anon_vma &&
287	anon_vma->degree < 2)
288	dst->anon_vma = anon_vma;
289	}
290	if (dst->anon_vma)
291	dst->anon_vma->degree++;
292	unlock_anon_vma_root(root);
293	return 0;
294
295	enomem_failure:
296	/*
297	* dst->anon_vma is dropped here otherwise its degree can be incorrectly
298	* decremented in unlink_anon_vmas().
299	* We can safely do this because callers of anon_vma_clone() don't care
300	* about dst->anon_vma if anon_vma_clone() failed.
301	*/
302	dst->anon_vma = NULL;
303	unlink_anon_vmas(dst);
304	return -ENOMEM;
305	}
306
307	/*
308	* Attach vma to its own anon_vma, as well as to the anon_vmas that
309	* the corresponding VMA in the parent process is attached to.
310	* Returns 0 on success, non-zero on failure.
311	*/
312	int anon_vma_fork(struct vm_area_struct *vma, struct vm_area_struct *pvma)
313	{
314	struct anon_vma_chain *avc;
315	struct anon_vma *anon_vma;
316	int error;
317
318	/* Don't bother if the parent process has no anon_vma here. */
319	if (!pvma->anon_vma)
320	return 0;
321
322	/* Drop inherited anon_vma, we'll reuse existing or allocate new. */
323	vma->anon_vma = NULL;
324
325	/*
326	* First, attach the new VMA to the parent VMA's anon_vmas,
327	* so rmap can find non-COWed pages in child processes.
328	*/
329	error = anon_vma_clone(vma, pvma);
330	if (error)
331	return error;
332
333	/* An existing anon_vma has been reused, all done then. */
334	if (vma->anon_vma)
335	return 0;
336
337	/* Then add our own anon_vma. */
338	anon_vma = anon_vma_alloc();
339	if (!anon_vma)
340	goto out_error;
341	avc = anon_vma_chain_alloc(GFP_KERNEL);
342	if (!avc)
343	goto out_error_free_anon_vma;
344
345	/*
346	* The root anon_vma's spinlock is the lock actually used when we
347	* lock any of the anon_vmas in this anon_vma tree.
348	*/
349	anon_vma->root = pvma->anon_vma->root;
350	anon_vma->parent = pvma->anon_vma;
351	/*
352	* With refcounts, an anon_vma can stay around longer than the
353	* process it belongs to. The root anon_vma needs to be pinned until
354	* this anon_vma is freed, because the lock lives in the root.
355	*/
356	get_anon_vma(anon_vma->root);
357	/* Mark this anon_vma as the one where our new (COWed) pages go. */
358	vma->anon_vma = anon_vma;
359	anon_vma_lock_write(anon_vma);
360	anon_vma_chain_link(vma, avc, anon_vma);
361	anon_vma->parent->degree++;
362	anon_vma_unlock_write(anon_vma);
363
364	return 0;
365
366	out_error_free_anon_vma:
367	put_anon_vma(anon_vma);
368	out_error:
369	unlink_anon_vmas(vma);
370	return -ENOMEM;
371	}
372
373	void unlink_anon_vmas(struct vm_area_struct *vma)
374	{
375	struct anon_vma_chain *avc, *next;
376	struct anon_vma *root = NULL;
377
378	/*
379	* Unlink each anon_vma chained to the VMA. This list is ordered
380	* from newest to oldest, ensuring the root anon_vma gets freed last.
381	*/
382	list_for_each_entry_safe(avc, next, &vma->anon_vma_chain, same_vma) {
383	struct anon_vma *anon_vma = avc->anon_vma;
384
385	root = lock_anon_vma_root(root, anon_vma);
386	anon_vma_interval_tree_remove(avc, &anon_vma->rb_root);
387
388	/*
389	* Leave empty anon_vmas on the list - we'll need
390	* to free them outside the lock.
391	*/
392	if (RB_EMPTY_ROOT(&anon_vma->rb_root)) {
393	anon_vma->parent->degree--;
394	continue;
395	}
396
397	list_del(&avc->same_vma);
398	anon_vma_chain_free(avc);
399	}
400	if (vma->anon_vma)
401	vma->anon_vma->degree--;
402	unlock_anon_vma_root(root);
403
404	/*
405	* Iterate the list once more, it now only contains empty and unlinked
406	* anon_vmas, destroy them. Could not do before due to __put_anon_vma()
407	* needing to write-acquire the anon_vma->root->rwsem.
408	*/
409	list_for_each_entry_safe(avc, next, &vma->anon_vma_chain, same_vma) {
410	struct anon_vma *anon_vma = avc->anon_vma;
411
412	VM_WARN_ON(anon_vma->degree);
413	put_anon_vma(anon_vma);
414
415	list_del(&avc->same_vma);
416	anon_vma_chain_free(avc);
417	}
418	}
419
420	static void anon_vma_ctor(void *data)
421	{
422	struct anon_vma *anon_vma = data;
423
424	init_rwsem(&anon_vma->rwsem);
425	atomic_set(&anon_vma->refcount, 0);
426	anon_vma->rb_root = RB_ROOT;
427	}
428
429	void __init anon_vma_init(void)
430	{
431	anon_vma_cachep = kmem_cache_create("anon_vma", sizeof(struct anon_vma),
432	0, SLAB_DESTROY_BY_RCU|SLAB_PANIC|SLAB_ACCOUNT,
433	anon_vma_ctor);
434	anon_vma_chain_cachep = KMEM_CACHE(anon_vma_chain,
435	SLAB_PANIC|SLAB_ACCOUNT);
436	}
437
438	/*
439	* Getting a lock on a stable anon_vma from a page off the LRU is tricky!
440	*
441	* Since there is no serialization what so ever against page_remove_rmap()
442	* the best this function can do is return a locked anon_vma that might
443	* have been relevant to this page.
444	*
445	* The page might have been remapped to a different anon_vma or the anon_vma
446	* returned may already be freed (and even reused).
447	*
448	* In case it was remapped to a different anon_vma, the new anon_vma will be a
449	* child of the old anon_vma, and the anon_vma lifetime rules will therefore
450	* ensure that any anon_vma obtained from the page will still be valid for as
451	* long as we observe page_mapped() [ hence all those page_mapped() tests ].
452	*
453	* All users of this function must be very careful when walking the anon_vma
454	* chain and verify that the page in question is indeed mapped in it
455	* [ something equivalent to page_mapped_in_vma() ].
456	*
457	* Since anon_vma's slab is DESTROY_BY_RCU and we know from page_remove_rmap()
458	* that the anon_vma pointer from page->mapping is valid if there is a
459	* mapcount, we can dereference the anon_vma after observing those.
460	*/
461	struct anon_vma *page_get_anon_vma(struct page *page)
462	{
463	struct anon_vma *anon_vma = NULL;
464	unsigned long anon_mapping;
465
466	rcu_read_lock();
467	anon_mapping = (unsigned long)READ_ONCE(page->mapping);
468	if ((anon_mapping & PAGE_MAPPING_FLAGS) != PAGE_MAPPING_ANON)
469	goto out;
470	if (!page_mapped(page))
471	goto out;
472
473	anon_vma = (struct anon_vma *) (anon_mapping - PAGE_MAPPING_ANON);
474	if (!atomic_inc_not_zero(&anon_vma->refcount)) {
475	anon_vma = NULL;
476	goto out;
477	}
478
479	/*
480	* If this page is still mapped, then its anon_vma cannot have been
481	* freed. But if it has been unmapped, we have no security against the
482	* anon_vma structure being freed and reused (for another anon_vma:
483	* SLAB_DESTROY_BY_RCU guarantees that - so the atomic_inc_not_zero()
484	* above cannot corrupt).
485	*/
486	if (!page_mapped(page)) {
487	rcu_read_unlock();
488	put_anon_vma(anon_vma);
489	return NULL;
490	}
491	out:
492	rcu_read_unlock();
493
494	return anon_vma;
495	}
496
497	/*
498	* Similar to page_get_anon_vma() except it locks the anon_vma.
499	*
500	* Its a little more complex as it tries to keep the fast path to a single
501	* atomic op -- the trylock. If we fail the trylock, we fall back to getting a
502	* reference like with page_get_anon_vma() and then block on the mutex.
503	*/
504	struct anon_vma *page_lock_anon_vma_read(struct page *page)
505	{
506	struct anon_vma *anon_vma = NULL;
507	struct anon_vma *root_anon_vma;
508	unsigned long anon_mapping;
509
510	rcu_read_lock();
511	anon_mapping = (unsigned long)READ_ONCE(page->mapping);
512	if ((anon_mapping & PAGE_MAPPING_FLAGS) != PAGE_MAPPING_ANON)
513	goto out;
514	if (!page_mapped(page))
515	goto out;
516
517	anon_vma = (struct anon_vma *) (anon_mapping - PAGE_MAPPING_ANON);
518	root_anon_vma = READ_ONCE(anon_vma->root);
519	if (down_read_trylock(&root_anon_vma->rwsem)) {
520	/*
521	* If the page is still mapped, then this anon_vma is still
522	* its anon_vma, and holding the mutex ensures that it will
523	* not go away, see anon_vma_free().
524	*/
525	if (!page_mapped(page)) {
526	up_read(&root_anon_vma->rwsem);
527	anon_vma = NULL;
528	}
529	goto out;
530	}
531
532	/* trylock failed, we got to sleep */
533	if (!atomic_inc_not_zero(&anon_vma->refcount)) {
534	anon_vma = NULL;
535	goto out;
536	}
537
538	if (!page_mapped(page)) {
539	rcu_read_unlock();
540	put_anon_vma(anon_vma);
541	return NULL;
542	}
543
544	/* we pinned the anon_vma, its safe to sleep */
545	rcu_read_unlock();
546	anon_vma_lock_read(anon_vma);
547
548	if (atomic_dec_and_test(&anon_vma->refcount)) {
549	/*
550	* Oops, we held the last refcount, release the lock
551	* and bail -- can't simply use put_anon_vma() because
552	* we'll deadlock on the anon_vma_lock_write() recursion.
553	*/
554	anon_vma_unlock_read(anon_vma);
555	__put_anon_vma(anon_vma);
556	anon_vma = NULL;
557	}
558
559	return anon_vma;
560
561	out:
562	rcu_read_unlock();
563	return anon_vma;
564	}
565
566	void page_unlock_anon_vma_read(struct anon_vma *anon_vma)
567	{
568	anon_vma_unlock_read(anon_vma);
569	}
570
571	#ifdef CONFIG_ARCH_WANT_BATCHED_UNMAP_TLB_FLUSH
572	/*
573	* Flush TLB entries for recently unmapped pages from remote CPUs. It is
574	* important if a PTE was dirty when it was unmapped that it's flushed
575	* before any IO is initiated on the page to prevent lost writes. Similarly,
576	* it must be flushed before freeing to prevent data leakage.
577	*/
578	void try_to_unmap_flush(void)
579	{
580	struct tlbflush_unmap_batch *tlb_ubc = &current->tlb_ubc;
581	int cpu;
582
583	if (!tlb_ubc->flush_required)
584	return;
585
586	cpu = get_cpu();
587
588	if (cpumask_test_cpu(cpu, &tlb_ubc->cpumask)) {
589	count_vm_tlb_event(NR_TLB_LOCAL_FLUSH_ALL);
590	local_flush_tlb();
591	trace_tlb_flush(TLB_LOCAL_SHOOTDOWN, TLB_FLUSH_ALL);
592	}
593
594	if (cpumask_any_but(&tlb_ubc->cpumask, cpu) < nr_cpu_ids)
595	flush_tlb_others(&tlb_ubc->cpumask, NULL, 0, TLB_FLUSH_ALL);
596	cpumask_clear(&tlb_ubc->cpumask);
597	tlb_ubc->flush_required = false;
598	tlb_ubc->writable = false;
599	put_cpu();
600	}
601
602	/* Flush iff there are potentially writable TLB entries that can race with IO */
603	void try_to_unmap_flush_dirty(void)
604	{
605	struct tlbflush_unmap_batch *tlb_ubc = &current->tlb_ubc;
606
607	if (tlb_ubc->writable)
608	try_to_unmap_flush();
609	}
610
611	static void set_tlb_ubc_flush_pending(struct mm_struct *mm,
612	struct page *page, bool writable)
613	{
614	struct tlbflush_unmap_batch *tlb_ubc = &current->tlb_ubc;
615
616	cpumask_or(&tlb_ubc->cpumask, &tlb_ubc->cpumask, mm_cpumask(mm));
617	tlb_ubc->flush_required = true;
618
619	/*
620	* Ensure compiler does not re-order the setting of tlb_flush_batched
621	* before the PTE is cleared.
622	*/
623	barrier();
624	mm->tlb_flush_batched = true;
625
626	/*
627	* If the PTE was dirty then it's best to assume it's writable. The
628	* caller must use try_to_unmap_flush_dirty() or try_to_unmap_flush()
629	* before the page is queued for IO.
630	*/
631	if (writable)
632	tlb_ubc->writable = true;
633	}
634
635	/*
636	* Returns true if the TLB flush should be deferred to the end of a batch of
637	* unmap operations to reduce IPIs.
638	*/
639	static bool should_defer_flush(struct mm_struct *mm, enum ttu_flags flags)
640	{
641	bool should_defer = false;
642
643	if (!(flags & TTU_BATCH_FLUSH))
644	return false;
645
646	/* If remote CPUs need to be flushed then defer batch the flush */
647	if (cpumask_any_but(mm_cpumask(mm), get_cpu()) < nr_cpu_ids)
648	should_defer = true;
649	put_cpu();
650
651	return should_defer;
652	}
653
654	/*
655	* Reclaim unmaps pages under the PTL but do not flush the TLB prior to
656	* releasing the PTL if TLB flushes are batched. It's possible for a parallel
657	* operation such as mprotect or munmap to race between reclaim unmapping
658	* the page and flushing the page. If this race occurs, it potentially allows
659	* access to data via a stale TLB entry. Tracking all mm's that have TLB
660	* batching in flight would be expensive during reclaim so instead track
661	* whether TLB batching occurred in the past and if so then do a flush here
662	* if required. This will cost one additional flush per reclaim cycle paid
663	* by the first operation at risk such as mprotect and mumap.
664	*
665	* This must be called under the PTL so that an access to tlb_flush_batched
666	* that is potentially a "reclaim vs mprotect/munmap/etc" race will synchronise
667	* via the PTL.
668	*/
669	void flush_tlb_batched_pending(struct mm_struct *mm)
670	{
671	if (mm->tlb_flush_batched) {
672	flush_tlb_mm(mm);
673
674	/*
675	* Do not allow the compiler to re-order the clearing of
676	* tlb_flush_batched before the tlb is flushed.
677	*/
678	barrier();
679	mm->tlb_flush_batched = false;
680	}
681	}
682	#else
683	static void set_tlb_ubc_flush_pending(struct mm_struct *mm,
684	struct page *page, bool writable)
685	{
686	}
687
688	static bool should_defer_flush(struct mm_struct *mm, enum ttu_flags flags)
689	{
690	return false;
691	}
692	#endif /* CONFIG_ARCH_WANT_BATCHED_UNMAP_TLB_FLUSH */
693
694	/*
695	* At what user virtual address is page expected in vma?
696	* Caller should check the page is actually part of the vma.
697	*/
698	unsigned long page_address_in_vma(struct page *page, struct vm_area_struct *vma)
699	{
700	unsigned long address;
701	if (PageAnon(page)) {
702	struct anon_vma *page__anon_vma = page_anon_vma(page);
703	/*
704	* Note: swapoff's unuse_vma() is more efficient with this
705	* check, and needs it to match anon_vma when KSM is active.
706	*/
707	if (!vma->anon_vma || !page__anon_vma ||
708	vma->anon_vma->root != page__anon_vma->root)
709	return -EFAULT;
710	} else if (page->mapping) {
711	if (!vma->vm_file || vma->vm_file->f_mapping != page->mapping)
712	return -EFAULT;
713	} else
714	return -EFAULT;
715	address = __vma_address(page, vma);
716	if (unlikely(address < vma->vm_start || address >= vma->vm_end))
717	return -EFAULT;
718	return address;
719	}
720
721	pmd_t *mm_find_pmd(struct mm_struct *mm, unsigned long address)
722	{
723	pgd_t *pgd;
724	pud_t *pud;
725	pmd_t *pmd = NULL;
726	pmd_t pmde;
727
728	pgd = pgd_offset(mm, address);
729	if (!pgd_present(*pgd))
730	goto out;
731
732	pud = pud_offset(pgd, address);
733	if (!pud_present(*pud))
734	goto out;
735
736	pmd = pmd_offset(pud, address);
737	/*
738	* Some THP functions use the sequence pmdp_huge_clear_flush(), set_pmd_at()
739	* without holding anon_vma lock for write. So when looking for a
740	* genuine pmde (in which to find pte), test present and !THP together.
741	*/
742	pmde = *pmd;
743	barrier();
744	if (!pmd_present(pmde) || pmd_trans_huge(pmde))
745	pmd = NULL;
746	out:
747	return pmd;
748	}
749
750	/*
751	* Check that @page is mapped at @address into @mm.
752	*
753	* If @sync is false, page_check_address may perform a racy check to avoid
754	* the page table lock when the pte is not present (helpful when reclaiming
755	* highly shared pages).
756	*
757	* On success returns with pte mapped and locked.
758	*/
759	pte_t *__page_check_address(struct page *page, struct mm_struct *mm,
760	unsigned long address, spinlock_t **ptlp, int sync)
761	{
762	pmd_t *pmd;
763	pte_t *pte;
764	spinlock_t *ptl;
765
766	if (unlikely(PageHuge(page))) {
767	/* when pud is not present, pte will be NULL */
768	pte = huge_pte_offset(mm, address);
769	if (!pte)
770	return NULL;
771
772	ptl = huge_pte_lockptr(page_hstate(page), mm, pte);
773	goto check;
774	}
775
776	pmd = mm_find_pmd(mm, address);
777	if (!pmd)
778	return NULL;
779
780	pte = pte_offset_map(pmd, address);
781	/* Make a quick check before getting the lock */
782	if (!sync && !pte_present(*pte)) {
783	pte_unmap(pte);
784	return NULL;
785	}
786
787	ptl = pte_lockptr(mm, pmd);
788	check:
789	spin_lock(ptl);
790	if (pte_present(*pte) && page_to_pfn(page) == pte_pfn(*pte)) {
791	*ptlp = ptl;
792	return pte;
793	}
794	pte_unmap_unlock(pte, ptl);
795	return NULL;
796	}
797
798	/**
799	* page_mapped_in_vma - check whether a page is really mapped in a VMA
800	* @page: the page to test
801	* @vma: the VMA to test
802	*
803	* Returns 1 if the page is mapped into the page tables of the VMA, 0
804	* if the page is not mapped into the page tables of this VMA. Only
805	* valid for normal file or anonymous VMAs.
806	*/
807	int page_mapped_in_vma(struct page *page, struct vm_area_struct *vma)
808	{
809	unsigned long address;
810	pte_t *pte;
811	spinlock_t *ptl;
812
813	address = __vma_address(page, vma);
814	if (unlikely(address < vma->vm_start || address >= vma->vm_end))
815	return 0;
816	pte = page_check_address(page, vma->vm_mm, address, &ptl, 1);
817	if (!pte) /* the page is not in this mm */
818	return 0;
819	pte_unmap_unlock(pte, ptl);
820
821	return 1;
822	}
823
824	#ifdef CONFIG_TRANSPARENT_HUGEPAGE
825	/*
826	* Check that @page is mapped at @address into @mm. In contrast to
827	* page_check_address(), this function can handle transparent huge pages.
828	*
829	* On success returns true with pte mapped and locked. For PMD-mapped
830	* transparent huge pages *@ptep is set to NULL.
831	*/
832	bool page_check_address_transhuge(struct page *page, struct mm_struct *mm,
833	unsigned long address, pmd_t **pmdp,
834	pte_t **ptep, spinlock_t **ptlp)
835	{
836	pgd_t *pgd;
837	pud_t *pud;
838	pmd_t *pmd;
839	pte_t *pte;
840	spinlock_t *ptl;
841
842	if (unlikely(PageHuge(page))) {
843	/* when pud is not present, pte will be NULL */
844	pte = huge_pte_offset(mm, address);
845	if (!pte)
846	return false;
847
848	ptl = huge_pte_lockptr(page_hstate(page), mm, pte);
849	pmd = NULL;
850	goto check_pte;
851	}
852
853	pgd = pgd_offset(mm, address);
854	if (!pgd_present(*pgd))
855	return false;
856	pud = pud_offset(pgd, address);
857	if (!pud_present(*pud))
858	return false;
859	pmd = pmd_offset(pud, address);
860
861	if (pmd_trans_huge(*pmd)) {
862	ptl = pmd_lock(mm, pmd);
863	if (!pmd_present(*pmd))
864	goto unlock_pmd;
865	if (unlikely(!pmd_trans_huge(*pmd))) {
866	spin_unlock(ptl);
867	goto map_pte;
868	}
869
870	if (pmd_page(*pmd) != page)
871	goto unlock_pmd;
872
873	pte = NULL;
874	goto found;
875	unlock_pmd:
876	spin_unlock(ptl);
877	return false;
878	} else {
879	pmd_t pmde = *pmd;
880
881	barrier();
882	if (!pmd_present(pmde) || pmd_trans_huge(pmde))
883	return false;
884	}
885	map_pte:
886	pte = pte_offset_map(pmd, address);
887	if (!pte_present(*pte)) {
888	pte_unmap(pte);
889	return false;
890	}
891
892	ptl = pte_lockptr(mm, pmd);
893	check_pte:
894	spin_lock(ptl);
895
896	if (!pte_present(*pte)) {
897	pte_unmap_unlock(pte, ptl);
898	return false;
899	}
900
901	/* THP can be referenced by any subpage */
902	if (pte_pfn(*pte) - page_to_pfn(page) >= hpage_nr_pages(page)) {
903	pte_unmap_unlock(pte, ptl);
904	return false;
905	}
906	found:
907	*ptep = pte;
908	*pmdp = pmd;
909	*ptlp = ptl;
910	return true;
911	}
912	#endif /* CONFIG_TRANSPARENT_HUGEPAGE */
913
914	struct page_referenced_arg {
915	int mapcount;
916	int referenced;
917	unsigned long vm_flags;
918	struct mem_cgroup *memcg;
919	};
920	/*
921	* arg: page_referenced_arg will be passed
922	*/
923	static int page_referenced_one(struct page *page, struct vm_area_struct *vma,
924	unsigned long address, void *arg)
925	{
926	struct mm_struct *mm = vma->vm_mm;
927	struct page_referenced_arg *pra = arg;
928	pmd_t *pmd;
929	pte_t *pte;
930	spinlock_t *ptl;
931	int referenced = 0;
932
933	if (!page_check_address_transhuge(page, mm, address, &pmd, &pte, &ptl))
934	return SWAP_AGAIN;
935
936	if (vma->vm_flags & VM_LOCKED) {
937	if (pte)
938	pte_unmap(pte);
939	spin_unlock(ptl);
940	pra->vm_flags |= VM_LOCKED;
941	return SWAP_FAIL; /* To break the loop */
942	}
943
944	if (pte) {
945	if (ptep_clear_flush_young_notify(vma, address, pte)) {
946	/*
947	* Don't treat a reference through a sequentially read
948	* mapping as such. If the page has been used in
949	* another mapping, we will catch it; if this other
950	* mapping is already gone, the unmap path will have
951	* set PG_referenced or activated the page.
952	*/
953	if (likely(!(vma->vm_flags & VM_SEQ_READ)))
954	referenced++;
955	}
956	pte_unmap(pte);
957	} else if (IS_ENABLED(CONFIG_TRANSPARENT_HUGEPAGE)) {
958	if (pmdp_clear_flush_young_notify(vma, address, pmd))
959	referenced++;
960	} else {
961	/* unexpected pmd-mapped page? */
962	WARN_ON_ONCE(1);
963	}
964	spin_unlock(ptl);
965
966	if (referenced)
967	clear_page_idle(page);
968	if (test_and_clear_page_young(page))
969	referenced++;
970
971	if (referenced) {
972	pra->referenced++;
973	pra->vm_flags |= vma->vm_flags;
974	}
975
976	pra->mapcount--;
977	if (!pra->mapcount)
978	return SWAP_SUCCESS; /* To break the loop */
979
980	return SWAP_AGAIN;
981	}
982
983	static bool invalid_page_referenced_vma(struct vm_area_struct *vma, void *arg)
984	{
985	struct page_referenced_arg *pra = arg;
986	struct mem_cgroup *memcg = pra->memcg;
987
988	if (!mm_match_cgroup(vma->vm_mm, memcg))
989	return true;
990
991	return false;
992	}
993
994	/**
995	* page_referenced - test if the page was referenced
996	* @page: the page to test
997	* @is_locked: caller holds lock on the page
998	* @memcg: target memory cgroup
999	* @vm_flags: collect encountered vma->vm_flags who actually referenced the page
1000	*
1001	* Quick test_and_clear_referenced for all mappings to a page,
1002	* returns the number of ptes which referenced the page.
1003	*/
1004	int page_referenced(struct page *page,
1005	int is_locked,
1006	struct mem_cgroup *memcg,
1007	unsigned long *vm_flags)
1008	{
1009	int ret;
1010	int we_locked = 0;
1011	struct page_referenced_arg pra = {
1012	.mapcount = total_mapcount(page),
1013	.memcg = memcg,
1014	};
1015	struct rmap_walk_control rwc = {
1016	.rmap_one = page_referenced_one,
1017	.arg = (void *)&pra,
1018	.anon_lock = page_lock_anon_vma_read,
1019	};
1020
1021	*vm_flags = 0;
1022	if (!page_mapped(page))
1023	return 0;
1024
1025	if (!page_rmapping(page))
1026	return 0;
1027
1028	if (!is_locked && (!PageAnon(page) || PageKsm(page))) {
1029	we_locked = trylock_page(page);
1030	if (!we_locked)
1031	return 1;
1032	}
1033
1034	/*
1035	* If we are reclaiming on behalf of a cgroup, skip
1036	* counting on behalf of references from different
1037	* cgroups
1038	*/
1039	if (memcg) {
1040	rwc.invalid_vma = invalid_page_referenced_vma;
1041	}
1042
1043	ret = rmap_walk(page, &rwc);
1044	*vm_flags = pra.vm_flags;
1045
1046	if (we_locked)
1047	unlock_page(page);
1048
1049	return pra.referenced;
1050	}
1051
1052	static int page_mkclean_one(struct page *page, struct vm_area_struct *vma,
1053	unsigned long address, void *arg)
1054	{
1055	struct mm_struct *mm = vma->vm_mm;
1056	pte_t *pte;
1057	spinlock_t *ptl;
1058	int ret = 0;
1059	int *cleaned = arg;
1060
1061	pte = page_check_address(page, mm, address, &ptl, 1);
1062	if (!pte)
1063	goto out;
1064
1065	if (pte_dirty(*pte) || pte_write(*pte)) {
1066	pte_t entry;
1067
1068	flush_cache_page(vma, address, pte_pfn(*pte));
1069	entry = ptep_clear_flush(vma, address, pte);
1070	entry = pte_wrprotect(entry);
1071	entry = pte_mkclean(entry);
1072	set_pte_at(mm, address, pte, entry);
1073	ret = 1;
1074	}
1075
1076	pte_unmap_unlock(pte, ptl);
1077
1078	if (ret) {
1079	mmu_notifier_invalidate_page(mm, address);
1080	(*cleaned)++;
1081	}
1082	out:
1083	return SWAP_AGAIN;
1084	}
1085
1086	static bool invalid_mkclean_vma(struct vm_area_struct *vma, void *arg)
1087	{
1088	if (vma->vm_flags & VM_SHARED)
1089	return false;
1090
1091	return true;
1092	}
1093
1094	int page_mkclean(struct page *page)
1095	{
1096	int cleaned = 0;
1097	struct address_space *mapping;
1098	struct rmap_walk_control rwc = {
1099	.arg = (void *)&cleaned,
1100	.rmap_one = page_mkclean_one,
1101	.invalid_vma = invalid_mkclean_vma,
1102	};
1103
1104	BUG_ON(!PageLocked(page));
1105
1106	if (!page_mapped(page))
1107	return 0;
1108
1109	mapping = page_mapping(page);
1110	if (!mapping)
1111	return 0;
1112
1113	rmap_walk(page, &rwc);
1114
1115	return cleaned;
1116	}
1117	EXPORT_SYMBOL_GPL(page_mkclean);
1118
1119	/**
1120	* page_move_anon_rmap - move a page to our anon_vma
1121	* @page: the page to move to our anon_vma
1122	* @vma: the vma the page belongs to
1123	*
1124	* When a page belongs exclusively to one process after a COW event,
1125	* that page can be moved into the anon_vma that belongs to just that
1126	* process, so the rmap code will not search the parent or sibling
1127	* processes.
1128	*/
1129	void page_move_anon_rmap(struct page *page, struct vm_area_struct *vma)
1130	{
1131	struct anon_vma *anon_vma = vma->anon_vma;
1132
1133	page = compound_head(page);
1134
1135	VM_BUG_ON_PAGE(!PageLocked(page), page);
1136	VM_BUG_ON_VMA(!anon_vma, vma);
1137
1138	anon_vma = (void *) anon_vma + PAGE_MAPPING_ANON;
1139	/*
1140	* Ensure that anon_vma and the PAGE_MAPPING_ANON bit are written
1141	* simultaneously, so a concurrent reader (eg page_referenced()'s
1142	* PageAnon()) will not see one without the other.
1143	*/
1144	WRITE_ONCE(page->mapping, (struct address_space *) anon_vma);
1145	}
1146
1147	/**
1148	* __page_set_anon_rmap - set up new anonymous rmap
1149	* @page: Page to add to rmap
1150	* @vma: VM area to add page to.
1151	* @address: User virtual address of the mapping
1152	* @exclusive: the page is exclusively owned by the current process
1153	*/
1154	static void __page_set_anon_rmap(struct page *page,
1155	struct vm_area_struct *vma, unsigned long address, int exclusive)
1156	{
1157	struct anon_vma *anon_vma = vma->anon_vma;
1158
1159	BUG_ON(!anon_vma);
1160
1161	if (PageAnon(page))
1162	return;
1163
1164	/*
1165	* If the page isn't exclusively mapped into this vma,
1166	* we must use the _oldest_ possible anon_vma for the
1167	* page mapping!
1168	*/
1169	if (!exclusive)
1170	anon_vma = anon_vma->root;
1171
1172	anon_vma = (void *) anon_vma + PAGE_MAPPING_ANON;
1173	page->mapping = (struct address_space *) anon_vma;
1174	page->index = linear_page_index(vma, address);
1175	}
1176
1177	/**
1178	* __page_check_anon_rmap - sanity check anonymous rmap addition
1179	* @page: the page to add the mapping to
1180	* @vma: the vm area in which the mapping is added
1181	* @address: the user virtual address mapped
1182	*/
1183	static void __page_check_anon_rmap(struct page *page,
1184	struct vm_area_struct *vma, unsigned long address)
1185	{
1186	#ifdef CONFIG_DEBUG_VM
1187	/*
1188	* The page's anon-rmap details (mapping and index) are guaranteed to
1189	* be set up correctly at this point.
1190	*
1191	* We have exclusion against page_add_anon_rmap because the caller
1192	* always holds the page locked, except if called from page_dup_rmap,
1193	* in which case the page is already known to be setup.
1194	*
1195	* We have exclusion against page_add_new_anon_rmap because those pages
1196	* are initially only visible via the pagetables, and the pte is locked
1197	* over the call to page_add_new_anon_rmap.
1198	*/
1199	BUG_ON(page_anon_vma(page)->root != vma->anon_vma->root);
1200	BUG_ON(page_to_pgoff(page) != linear_page_index(vma, address));
1201	#endif
1202	}
1203
1204	/**
1205	* page_add_anon_rmap - add pte mapping to an anonymous page
1206	* @page: the page to add the mapping to
1207	* @vma: the vm area in which the mapping is added
1208	* @address: the user virtual address mapped
1209	* @compound: charge the page as compound or small page
1210	*
1211	* The caller needs to hold the pte lock, and the page must be locked in
1212	* the anon_vma case: to serialize mapping,index checking after setting,
1213	* and to ensure that PageAnon is not being upgraded racily to PageKsm
1214	* (but PageKsm is never downgraded to PageAnon).
1215	*/
1216	void page_add_anon_rmap(struct page *page,
1217	struct vm_area_struct *vma, unsigned long address, bool compound)
1218	{
1219	do_page_add_anon_rmap(page, vma, address, compound ? RMAP_COMPOUND : 0);
1220	}
1221
1222	/*
1223	* Special version of the above for do_swap_page, which often runs
1224	* into pages that are exclusively owned by the current process.
1225	* Everybody else should continue to use page_add_anon_rmap above.
1226	*/
1227	void do_page_add_anon_rmap(struct page *page,
1228	struct vm_area_struct *vma, unsigned long address, int flags)
1229	{
1230	bool compound = flags & RMAP_COMPOUND;
1231	bool first;
1232
1233	if (compound) {
1234	atomic_t *mapcount;
1235	VM_BUG_ON_PAGE(!PageLocked(page), page);
1236	VM_BUG_ON_PAGE(!PageTransHuge(page), page);
1237	mapcount = compound_mapcount_ptr(page);
1238	first = atomic_inc_and_test(mapcount);
1239	} else {
1240	first = atomic_inc_and_test(&page->_mapcount);
1241	}
1242
1243	if (first) {
1244	int nr = compound ? hpage_nr_pages(page) : 1;
1245	/*
1246	* We use the irq-unsafe __{inc|mod}_zone_page_stat because
1247	* these counters are not modified in interrupt context, and
1248	* pte lock(a spinlock) is held, which implies preemption
1249	* disabled.
1250	*/
1251	if (compound)
1252	__inc_node_page_state(page, NR_ANON_THPS);
1253	__mod_node_page_state(page_pgdat(page), NR_ANON_MAPPED, nr);
1254	}
1255	if (unlikely(PageKsm(page)))
1256	return;
1257
1258	VM_BUG_ON_PAGE(!PageLocked(page), page);
1259
1260	/* address might be in next vma when migration races vma_adjust */
1261	if (first)
1262	__page_set_anon_rmap(page, vma, address,
1263	flags & RMAP_EXCLUSIVE);
1264	else
1265	__page_check_anon_rmap(page, vma, address);
1266	}
1267
1268	/**
1269	* page_add_new_anon_rmap - add pte mapping to a new anonymous page
1270	* @page: the page to add the mapping to
1271	* @vma: the vm area in which the mapping is added
1272	* @address: the user virtual address mapped
1273	* @compound: charge the page as compound or small page
1274	*
1275	* Same as page_add_anon_rmap but must only be called on *new* pages.
1276	* This means the inc-and-test can be bypassed.
1277	* Page does not have to be locked.
1278	*/
1279	void page_add_new_anon_rmap(struct page *page,
1280	struct vm_area_struct *vma, unsigned long address, bool compound)
1281	{
1282	int nr = compound ? hpage_nr_pages(page) : 1;
1283
1284	VM_BUG_ON_VMA(address < vma->vm_start || address >= vma->vm_end, vma);
1285	__SetPageSwapBacked(page);
1286	if (compound) {
1287	VM_BUG_ON_PAGE(!PageTransHuge(page), page);
1288	/* increment count (starts at -1) */
1289	atomic_set(compound_mapcount_ptr(page), 0);
1290	__inc_node_page_state(page, NR_ANON_THPS);
1291	} else {
1292	/* Anon THP always mapped first with PMD */
1293	VM_BUG_ON_PAGE(PageTransCompound(page), page);
1294	/* increment count (starts at -1) */
1295	atomic_set(&page->_mapcount, 0);
1296	}
1297	__mod_node_page_state(page_pgdat(page), NR_ANON_MAPPED, nr);
1298	__page_set_anon_rmap(page, vma, address, 1);
1299	}
1300
1301	/**
1302	* page_add_file_rmap - add pte mapping to a file page
1303	* @page: the page to add the mapping to
1304	*
1305	* The caller needs to hold the pte lock.
1306	*/
1307	void page_add_file_rmap(struct page *page, bool compound)
1308	{
1309	int i, nr = 1;
1310
1311	VM_BUG_ON_PAGE(compound && !PageTransHuge(page), page);
1312	lock_page_memcg(page);
1313	if (compound && PageTransHuge(page)) {
1314	for (i = 0, nr = 0; i < HPAGE_PMD_NR; i++) {
1315	if (atomic_inc_and_test(&page[i]._mapcount))
1316	nr++;
1317	}
1318	if (!atomic_inc_and_test(compound_mapcount_ptr(page)))
1319	goto out;
1320	VM_BUG_ON_PAGE(!PageSwapBacked(page), page);
1321	__inc_node_page_state(page, NR_SHMEM_PMDMAPPED);
1322	} else {
1323	if (PageTransCompound(page) && page_mapping(page)) {
1324	VM_WARN_ON_ONCE(!PageLocked(page));
1325
1326	SetPageDoubleMap(compound_head(page));
1327	if (PageMlocked(page))
1328	clear_page_mlock(compound_head(page));
1329	}
1330	if (!atomic_inc_and_test(&page->_mapcount))
1331	goto out;
1332	}
1333	__mod_node_page_state(page_pgdat(page), NR_FILE_MAPPED, nr);
1334	mem_cgroup_update_page_stat(page, MEM_CGROUP_STAT_FILE_MAPPED, nr);
1335	out:
1336	unlock_page_memcg(page);
1337	}
1338
1339	static void page_remove_file_rmap(struct page *page, bool compound)
1340	{
1341	int i, nr = 1;
1342
1343	VM_BUG_ON_PAGE(compound && !PageHead(page), page);
1344	lock_page_memcg(page);
1345
1346	/* Hugepages are not counted in NR_FILE_MAPPED for now. */
1347	if (unlikely(PageHuge(page))) {
1348	/* hugetlb pages are always mapped with pmds */
1349	atomic_dec(compound_mapcount_ptr(page));
1350	goto out;
1351	}
1352
1353	/* page still mapped by someone else? */
1354	if (compound && PageTransHuge(page)) {
1355	for (i = 0, nr = 0; i < HPAGE_PMD_NR; i++) {
1356	if (atomic_add_negative(-1, &page[i]._mapcount))
1357	nr++;
1358	}
1359	if (!atomic_add_negative(-1, compound_mapcount_ptr(page)))
1360	goto out;
1361	VM_BUG_ON_PAGE(!PageSwapBacked(page), page);
1362	__dec_node_page_state(page, NR_SHMEM_PMDMAPPED);
1363	} else {
1364	if (!atomic_add_negative(-1, &page->_mapcount))
1365	goto out;
1366	}
1367
1368	/*
1369	* We use the irq-unsafe __{inc|mod}_zone_page_state because
1370	* these counters are not modified in interrupt context, and
1371	* pte lock(a spinlock) is held, which implies preemption disabled.
1372	*/
1373	__mod_node_page_state(page_pgdat(page), NR_FILE_MAPPED, -nr);
1374	mem_cgroup_update_page_stat(page, MEM_CGROUP_STAT_FILE_MAPPED, -nr);
1375
1376	if (unlikely(PageMlocked(page)))
1377	clear_page_mlock(page);
1378	out:
1379	unlock_page_memcg(page);
1380	}
1381
1382	static void page_remove_anon_compound_rmap(struct page *page)
1383	{
1384	int i, nr;
1385
1386	if (!atomic_add_negative(-1, compound_mapcount_ptr(page)))
1387	return;
1388
1389	/* Hugepages are not counted in NR_ANON_PAGES for now. */
1390	if (unlikely(PageHuge(page)))
1391	return;
1392
1393	if (!IS_ENABLED(CONFIG_TRANSPARENT_HUGEPAGE))
1394	return;
1395
1396	__dec_node_page_state(page, NR_ANON_THPS);
1397
1398	if (TestClearPageDoubleMap(page)) {
1399	/*
1400	* Subpages can be mapped with PTEs too. Check how many of
1401	* themi are still mapped.
1402	*/
1403	for (i = 0, nr = 0; i < HPAGE_PMD_NR; i++) {
1404	if (atomic_add_negative(-1, &page[i]._mapcount))
1405	nr++;
1406	}
1407	} else {
1408	nr = HPAGE_PMD_NR;
1409	}
1410
1411	if (unlikely(PageMlocked(page)))
1412	clear_page_mlock(page);
1413
1414	if (nr) {
1415	__mod_node_page_state(page_pgdat(page), NR_ANON_MAPPED, -nr);
1416	deferred_split_huge_page(page);
1417	}
1418	}
1419
1420	/**
1421	* page_remove_rmap - take down pte mapping from a page
1422	* @page: page to remove mapping from
1423	* @compound: uncharge the page as compound or small page
1424	*
1425	* The caller needs to hold the pte lock.
1426	*/
1427	void page_remove_rmap(struct page *page, bool compound)
1428	{
1429	if (!PageAnon(page))
1430	return page_remove_file_rmap(page, compound);
1431
1432	if (compound)
1433	return page_remove_anon_compound_rmap(page);
1434
1435	/* page still mapped by someone else? */
1436	if (!atomic_add_negative(-1, &page->_mapcount))
1437	return;
1438
1439	/*
1440	* We use the irq-unsafe __{inc|mod}_zone_page_stat because
1441	* these counters are not modified in interrupt context, and
1442	* pte lock(a spinlock) is held, which implies preemption disabled.
1443	*/
1444	__dec_node_page_state(page, NR_ANON_MAPPED);
1445
1446	if (unlikely(PageMlocked(page)))
1447	clear_page_mlock(page);
1448
1449	if (PageTransCompound(page))
1450	deferred_split_huge_page(compound_head(page));
1451
1452	/*
1453	* It would be tidy to reset the PageAnon mapping here,
1454	* but that might overwrite a racing page_add_anon_rmap
1455	* which increments mapcount after us but sets mapping
1456	* before us: so leave the reset to free_hot_cold_page,
1457	* and remember that it's only reliable while mapped.
1458	* Leaving it set also helps swapoff to reinstate ptes
1459	* faster for those pages still in swapcache.
1460	*/
1461	}
1462
1463	struct rmap_private {
1464	enum ttu_flags flags;
1465	int lazyfreed;
1466	};
1467
1468	/*
1469	* @arg: enum ttu_flags will be passed to this argument
1470	*/
1471	static int try_to_unmap_one(struct page *page, struct vm_area_struct *vma,
1472	unsigned long address, void *arg)
1473	{
1474	struct mm_struct *mm = vma->vm_mm;
1475	pte_t *pte;
1476	pte_t pteval;
1477	spinlock_t *ptl;
1478	int ret = SWAP_AGAIN;
1479	unsigned long sh_address;
1480	bool pmd_sharing_possible = false;
1481	unsigned long spmd_start, spmd_end;
1482	struct rmap_private *rp = arg;
1483	enum ttu_flags flags = rp->flags;
1484
1485	/* munlock has nothing to gain from examining un-locked vmas */
1486	if ((flags & TTU_MUNLOCK) && !(vma->vm_flags & VM_LOCKED))
1487	goto out;
1488
1489	if (flags & TTU_SPLIT_HUGE_PMD) {
1490	split_huge_pmd_address(vma, address,
1491	flags & TTU_MIGRATION, page);
1492	/* check if we have anything to do after split */
1493	if (page_mapcount(page) == 0)
1494	goto out;
1495	}
1496
1497	/*
1498	* Only use the range_start/end mmu notifiers if huge pmd sharing
1499	* is possible. In the normal case, mmu_notifier_invalidate_page
1500	* is sufficient as we only unmap a page. However, if we unshare
1501	* a pmd, we will unmap a PUD_SIZE range.
1502	*/
1503	if (PageHuge(page)) {
1504	spmd_start = address;
1505	spmd_end = spmd_start + vma_mmu_pagesize(vma);
1506
1507	/*
1508	* Check if pmd sharing is possible. If possible, we could
1509	* unmap a PUD_SIZE range. spmd_start/spmd_end will be
1510	* modified if sharing is possible.
1511	*/
1512	adjust_range_if_pmd_sharing_possible(vma, &spmd_start,
1513	&spmd_end);
1514	if (spmd_end - spmd_start != vma_mmu_pagesize(vma)) {
1515	sh_address = address;
1516
1517	pmd_sharing_possible = true;
1518	mmu_notifier_invalidate_range_start(vma->vm_mm,
1519	spmd_start, spmd_end);
1520	}
1521	}
1522
1523	pte = page_check_address(page, mm, address, &ptl,
1524	PageTransCompound(page));
1525	if (!pte)
1526	goto out;
1527
1528	/*
1529	* If the page is mlock()d, we cannot swap it out.
1530	* If it's recently referenced (perhaps page_referenced
1531	* skipped over this mm) then we should reactivate it.
1532	*/
1533	if (!(flags & TTU_IGNORE_MLOCK)) {
1534	if (vma->vm_flags & VM_LOCKED) {
1535	/* PTE-mapped THP are never mlocked */
1536	if (!PageTransCompound(page)) {
1537	/*
1538	* Holding pte lock, we do *not* need
1539	* mmap_sem here
1540	*/
1541	mlock_vma_page(page);
1542	}
1543	ret = SWAP_MLOCK;
1544	goto out_unmap;
1545	}
1546	if (flags & TTU_MUNLOCK)
1547	goto out_unmap;
1548	}
1549	if (!(flags & TTU_IGNORE_ACCESS)) {
1550	if (ptep_clear_flush_young_notify(vma, address, pte)) {
1551	ret = SWAP_FAIL;
1552	goto out_unmap;
1553	}
1554	}
1555
1556	/*
1557	* Call huge_pmd_unshare to potentially unshare a huge pmd. Pass
1558	* sh_address as it will be modified if unsharing is successful.
1559	*/
1560	if (PageHuge(page) && huge_pmd_unshare(mm, &sh_address, pte)) {
1561	/*
1562	* huge_pmd_unshare unmapped an entire PMD page. There is
1563	* no way of knowing exactly which PMDs may be cached for
1564	* this mm, so flush them all. spmd_start/spmd_end cover
1565	* this PUD_SIZE range.
1566	*/
1567	flush_cache_range(vma, spmd_start, spmd_end);
1568	flush_tlb_range(vma, spmd_start, spmd_end);
1569
1570	/*
1571	* The ref count of the PMD page was dropped which is part
1572	* of the way map counting is done for shared PMDs. When
1573	* there is no other sharing, huge_pmd_unshare returns false
1574	* and we will unmap the actual page and drop map count
1575	* to zero.
1576	*/
1577	goto out_unmap;
1578	}
1579
1580	/* Nuke the page table entry. */
1581	flush_cache_page(vma, address, page_to_pfn(page));
1582	if (should_defer_flush(mm, flags)) {
1583	/*
1584	* We clear the PTE but do not flush so potentially a remote
1585	* CPU could still be writing to the page. If the entry was
1586	* previously clean then the architecture must guarantee that
1587	* a clear->dirty transition on a cached TLB entry is written
1588	* through and traps if the PTE is unmapped.
1589	*/
1590	pteval = ptep_get_and_clear(mm, address, pte);
1591
1592	set_tlb_ubc_flush_pending(mm, page, pte_dirty(pteval));
1593	} else {
1594	pteval = ptep_clear_flush(vma, address, pte);
1595	}
1596
1597	/* Move the dirty bit to the physical page now the pte is gone. */
1598	if (pte_dirty(pteval))
1599	set_page_dirty(page);
1600
1601	/* Update high watermark before we lower rss */
1602	update_hiwater_rss(mm);
1603
1604	if (PageHWPoison(page) && !(flags & TTU_IGNORE_HWPOISON)) {
1605	if (PageHuge(page)) {
1606	hugetlb_count_sub(1 << compound_order(page), mm);
1607	} else {
1608	dec_mm_counter(mm, mm_counter(page));
1609	}
1610	set_pte_at(mm, address, pte,
1611	swp_entry_to_pte(make_hwpoison_entry(page)));
1612	} else if (pte_unused(pteval)) {
1613	/*
1614	* The guest indicated that the page content is of no
1615	* interest anymore. Simply discard the pte, vmscan
1616	* will take care of the rest.
1617	*/
1618	dec_mm_counter(mm, mm_counter(page));
1619	} else if (IS_ENABLED(CONFIG_MIGRATION) && (flags & TTU_MIGRATION)) {
1620	swp_entry_t entry;
1621	pte_t swp_pte;
1622	/*
1623	* Store the pfn of the page in a special migration
1624	* pte. do_swap_page() will wait until the migration
1625	* pte is removed and then restart fault handling.
1626	*/
1627	entry = make_migration_entry(page, pte_write(pteval));
1628	swp_pte = swp_entry_to_pte(entry);
1629	if (pte_soft_dirty(pteval))
1630	swp_pte = pte_swp_mksoft_dirty(swp_pte);
1631	set_pte_at(mm, address, pte, swp_pte);
1632	} else if (PageAnon(page)) {
1633	swp_entry_t entry = { .val = page_private(page) };
1634	pte_t swp_pte;
1635	/*
1636	* Store the swap location in the pte.
1637	* See handle_pte_fault() ...
1638	*/
1639	VM_BUG_ON_PAGE(!PageSwapCache(page), page);
1640
1641	if (!PageDirty(page) && (flags & TTU_LZFREE)) {
1642	/* It's a freeable page by MADV_FREE */
1643	dec_mm_counter(mm, MM_ANONPAGES);
1644	rp->lazyfreed++;
1645	goto discard;
1646	}
1647
1648	if (swap_duplicate(entry) < 0) {
1649	set_pte_at(mm, address, pte, pteval);
1650	ret = SWAP_FAIL;
1651	goto out_unmap;
1652	}
1653	if (list_empty(&mm->mmlist)) {
1654	spin_lock(&mmlist_lock);
1655	if (list_empty(&mm->mmlist))
1656	list_add(&mm->mmlist, &init_mm.mmlist);
1657	spin_unlock(&mmlist_lock);
1658	}
1659	dec_mm_counter(mm, MM_ANONPAGES);
1660	inc_mm_counter(mm, MM_SWAPENTS);
1661	swp_pte = swp_entry_to_pte(entry);
1662	if (pte_soft_dirty(pteval))
1663	swp_pte = pte_swp_mksoft_dirty(swp_pte);
1664	set_pte_at(mm, address, pte, swp_pte);
1665	} else
1666	dec_mm_counter(mm, mm_counter_file(page));
1667
1668	discard:
1669	page_remove_rmap(page, PageHuge(page));
1670	put_page(page);
1671
1672	out_unmap:
1673	pte_unmap_unlock(pte, ptl);
1674	if (ret != SWAP_FAIL && ret != SWAP_MLOCK && !(flags & TTU_MUNLOCK))
1675	mmu_notifier_invalidate_page(mm, address);
1676	out:
1677	if (pmd_sharing_possible)
1678	mmu_notifier_invalidate_range_end(vma->vm_mm,
1679	spmd_start, spmd_end);
1680	return ret;
1681	}
1682
1683	bool is_vma_temporary_stack(struct vm_area_struct *vma)
1684	{
1685	int maybe_stack = vma->vm_flags & (VM_GROWSDOWN | VM_GROWSUP);
1686
1687	if (!maybe_stack)
1688	return false;
1689
1690	if ((vma->vm_flags & VM_STACK_INCOMPLETE_SETUP) ==
1691	VM_STACK_INCOMPLETE_SETUP)
1692	return true;
1693
1694	return false;
1695	}
1696
1697	static bool invalid_migration_vma(struct vm_area_struct *vma, void *arg)
1698	{
1699	return is_vma_temporary_stack(vma);
1700	}
1701
1702	static int page_mapcount_is_zero(struct page *page)
1703	{
1704	return !page_mapcount(page);
1705	}
1706
1707	/**
1708	* try_to_unmap - try to remove all page table mappings to a page
1709	* @page: the page to get unmapped
1710	* @flags: action and flags
1711	*
1712	* Tries to remove all the page table entries which are mapping this
1713	* page, used in the pageout path. Caller must hold the page lock.
1714	* Return values are:
1715	*
1716	* SWAP_SUCCESS - we succeeded in removing all mappings
1717	* SWAP_AGAIN - we missed a mapping, try again later
1718	* SWAP_FAIL - the page is unswappable
1719	* SWAP_MLOCK - page is mlocked.
1720	*/
1721	int try_to_unmap(struct page *page, enum ttu_flags flags)
1722	{
1723	int ret;
1724	struct rmap_private rp = {
1725	.flags = flags,
1726	.lazyfreed = 0,
1727	};
1728
1729	struct rmap_walk_control rwc = {
1730	.rmap_one = try_to_unmap_one,
1731	.arg = &rp,
1732	.done = page_mapcount_is_zero,
1733	.anon_lock = page_lock_anon_vma_read,
1734	};
1735
1736	/*
1737	* During exec, a temporary VMA is setup and later moved.
1738	* The VMA is moved under the anon_vma lock but not the
1739	* page tables leading to a race where migration cannot
1740	* find the migration ptes. Rather than increasing the
1741	* locking requirements of exec(), migration skips
1742	* temporary VMAs until after exec() completes.
1743	*/
1744	if ((flags & TTU_MIGRATION) && !PageKsm(page) && PageAnon(page))
1745	rwc.invalid_vma = invalid_migration_vma;
1746
1747	if (flags & TTU_RMAP_LOCKED)
1748	ret = rmap_walk_locked(page, &rwc);
1749	else
1750	ret = rmap_walk(page, &rwc);
1751
1752	if (ret != SWAP_MLOCK && !page_mapcount(page)) {
1753	ret = SWAP_SUCCESS;
1754	if (rp.lazyfreed && !PageDirty(page))
1755	ret = SWAP_LZFREE;
1756	}
1757	return ret;
1758	}
1759
1760	static int page_not_mapped(struct page *page)
1761	{
1762	return !page_mapped(page);
1763	};
1764
1765	/**
1766	* try_to_munlock - try to munlock a page
1767	* @page: the page to be munlocked
1768	*
1769	* Called from munlock code. Checks all of the VMAs mapping the page
1770	* to make sure nobody else has this page mlocked. The page will be
1771	* returned with PG_mlocked cleared if no other vmas have it mlocked.
1772	*
1773	* Return values are:
1774	*
1775	* SWAP_AGAIN - no vma is holding page mlocked, or,
1776	* SWAP_AGAIN - page mapped in mlocked vma -- couldn't acquire mmap sem
1777	* SWAP_FAIL - page cannot be located at present
1778	* SWAP_MLOCK - page is now mlocked.
1779	*/
1780	int try_to_munlock(struct page *page)
1781	{
1782	int ret;
1783	struct rmap_private rp = {
1784	.flags = TTU_MUNLOCK,
1785	.lazyfreed = 0,
1786	};
1787
1788	struct rmap_walk_control rwc = {
1789	.rmap_one = try_to_unmap_one,
1790	.arg = &rp,
1791	.done = page_not_mapped,
1792	.anon_lock = page_lock_anon_vma_read,
1793
1794	};
1795
1796	VM_BUG_ON_PAGE(!PageLocked(page) || PageLRU(page), page);
1797
1798	ret = rmap_walk(page, &rwc);
1799	return ret;
1800	}
1801
1802	void __put_anon_vma(struct anon_vma *anon_vma)
1803	{
1804	struct anon_vma *root = anon_vma->root;
1805
1806	anon_vma_free(anon_vma);
1807	if (root != anon_vma && atomic_dec_and_test(&root->refcount))
1808	anon_vma_free(root);
1809	}
1810
1811	static struct anon_vma *rmap_walk_anon_lock(struct page *page,
1812	struct rmap_walk_control *rwc)
1813	{
1814	struct anon_vma *anon_vma;
1815
1816	if (rwc->anon_lock)
1817	return rwc->anon_lock(page);
1818
1819	/*
1820	* Note: remove_migration_ptes() cannot use page_lock_anon_vma_read()
1821	* because that depends on page_mapped(); but not all its usages
1822	* are holding mmap_sem. Users without mmap_sem are required to
1823	* take a reference count to prevent the anon_vma disappearing
1824	*/
1825	anon_vma = page_anon_vma(page);
1826	if (!anon_vma)
1827	return NULL;
1828
1829	anon_vma_lock_read(anon_vma);
1830	return anon_vma;
1831	}
1832
1833	/*
1834	* rmap_walk_anon - do something to anonymous page using the object-based
1835	* rmap method
1836	* @page: the page to be handled
1837	* @rwc: control variable according to each walk type
1838	*
1839	* Find all the mappings of a page using the mapping pointer and the vma chains
1840	* contained in the anon_vma struct it points to.
1841	*
1842	* When called from try_to_munlock(), the mmap_sem of the mm containing the vma
1843	* where the page was found will be held for write. So, we won't recheck
1844	* vm_flags for that VMA. That should be OK, because that vma shouldn't be
1845	* LOCKED.
1846	*/
1847	static int rmap_walk_anon(struct page *page, struct rmap_walk_control *rwc,
1848	bool locked)
1849	{
1850	struct anon_vma *anon_vma;
1851	pgoff_t pgoff;
1852	struct anon_vma_chain *avc;
1853	int ret = SWAP_AGAIN;
1854
1855	if (locked) {
1856	anon_vma = page_anon_vma(page);
1857	/* anon_vma disappear under us? */
1858	VM_BUG_ON_PAGE(!anon_vma, page);
1859	} else {
1860	anon_vma = rmap_walk_anon_lock(page, rwc);
1861	}
1862	if (!anon_vma)
1863	return ret;
1864
1865	pgoff = page_to_pgoff(page);
1866	anon_vma_interval_tree_foreach(avc, &anon_vma->rb_root, pgoff, pgoff) {
1867	struct vm_area_struct *vma = avc->vma;
1868	unsigned long address = vma_address(page, vma);
1869
1870	cond_resched();
1871
1872	if (rwc->invalid_vma && rwc->invalid_vma(vma, rwc->arg))
1873	continue;
1874
1875	ret = rwc->rmap_one(page, vma, address, rwc->arg);
1876	if (ret != SWAP_AGAIN)
1877	break;
1878	if (rwc->done && rwc->done(page))
1879	break;
1880	}
1881
1882	if (!locked)
1883	anon_vma_unlock_read(anon_vma);
1884	return ret;
1885	}
1886
1887	/*
1888	* rmap_walk_file - do something to file page using the object-based rmap method
1889	* @page: the page to be handled
1890	* @rwc: control variable according to each walk type
1891	*
1892	* Find all the mappings of a page using the mapping pointer and the vma chains
1893	* contained in the address_space struct it points to.
1894	*
1895	* When called from try_to_munlock(), the mmap_sem of the mm containing the vma
1896	* where the page was found will be held for write. So, we won't recheck
1897	* vm_flags for that VMA. That should be OK, because that vma shouldn't be
1898	* LOCKED.
1899	*/
1900	static int rmap_walk_file(struct page *page, struct rmap_walk_control *rwc,
1901	bool locked)
1902	{
1903	struct address_space *mapping = page_mapping(page);
1904	pgoff_t pgoff;
1905	struct vm_area_struct *vma;
1906	int ret = SWAP_AGAIN;
1907
1908	/*
1909	* The page lock not only makes sure that page->mapping cannot
1910	* suddenly be NULLified by truncation, it makes sure that the
1911	* structure at mapping cannot be freed and reused yet,
1912	* so we can safely take mapping->i_mmap_rwsem.
1913	*/
1914	VM_BUG_ON_PAGE(!PageLocked(page), page);
1915
1916	if (!mapping)
1917	return ret;
1918
1919	pgoff = page_to_pgoff(page);
1920	if (!locked)
1921	i_mmap_lock_read(mapping);
1922	vma_interval_tree_foreach(vma, &mapping->i_mmap, pgoff, pgoff) {
1923	unsigned long address = vma_address(page, vma);
1924
1925	cond_resched();
1926
1927	if (rwc->invalid_vma && rwc->invalid_vma(vma, rwc->arg))
1928	continue;
1929
1930	ret = rwc->rmap_one(page, vma, address, rwc->arg);
1931	if (ret != SWAP_AGAIN)
1932	goto done;
1933	if (rwc->done && rwc->done(page))
1934	goto done;
1935	}
1936
1937	done:
1938	if (!locked)
1939	i_mmap_unlock_read(mapping);
1940	return ret;
1941	}
1942
1943	int rmap_walk(struct page *page, struct rmap_walk_control *rwc)
1944	{
1945	if (unlikely(PageKsm(page)))
1946	return rmap_walk_ksm(page, rwc);
1947	else if (PageAnon(page))
1948	return rmap_walk_anon(page, rwc, false);
1949	else
1950	return rmap_walk_file(page, rwc, false);
1951	}
1952
1953	/* Like rmap_walk, but caller holds relevant rmap lock */
1954	int rmap_walk_locked(struct page *page, struct rmap_walk_control *rwc)
1955	{
1956	/* no ksm support for now */
1957	VM_BUG_ON_PAGE(PageKsm(page), page);
1958	if (PageAnon(page))
1959	return rmap_walk_anon(page, rwc, true);
1960	else
1961	return rmap_walk_file(page, rwc, true);
1962	}
1963
1964	#ifdef CONFIG_HUGETLB_PAGE
1965	/*
1966	* The following three functions are for anonymous (private mapped) hugepages.
1967	* Unlike common anonymous pages, anonymous hugepages have no accounting code
1968	* and no lru code, because we handle hugepages differently from common pages.
1969	*/
1970	static void __hugepage_set_anon_rmap(struct page *page,
1971	struct vm_area_struct *vma, unsigned long address, int exclusive)
1972	{
1973	struct anon_vma *anon_vma = vma->anon_vma;
1974
1975	BUG_ON(!anon_vma);
1976
1977	if (PageAnon(page))
1978	return;
1979	if (!exclusive)
1980	anon_vma = anon_vma->root;
1981
1982	anon_vma = (void *) anon_vma + PAGE_MAPPING_ANON;
1983	page->mapping = (struct address_space *) anon_vma;
1984	page->index = linear_page_index(vma, address);
1985	}
1986
1987	void hugepage_add_anon_rmap(struct page *page,
1988	struct vm_area_struct *vma, unsigned long address)
1989	{
1990	struct anon_vma *anon_vma = vma->anon_vma;
1991	int first;
1992
1993	BUG_ON(!PageLocked(page));
1994	BUG_ON(!anon_vma);
1995	/* address might be in next vma when migration races vma_adjust */
1996	first = atomic_inc_and_test(compound_mapcount_ptr(page));
1997	if (first)
1998	__hugepage_set_anon_rmap(page, vma, address, 0);
1999	}
2000
2001	void hugepage_add_new_anon_rmap(struct page *page,
2002	struct vm_area_struct *vma, unsigned long address)
2003	{
2004	BUG_ON(address < vma->vm_start || address >= vma->vm_end);
2005	atomic_set(compound_mapcount_ptr(page), 0);
2006	__hugepage_set_anon_rmap(page, vma, address, 1);
2007	}
2008	#endif /* CONFIG_HUGETLB_PAGE */
2009