path: root/mm/vmscan.c (plain)
blob: 33b88cc838dad3e284e73c0b42d86e3cfe66228a

1	/*
2	* linux/mm/vmscan.c
3	*
4	* Copyright (C) 1991, 1992, 1993, 1994 Linus Torvalds
5	*
6	* Swap reorganised 29.12.95, Stephen Tweedie.
7	* kswapd added: 7.1.96 sct
8	* Removed kswapd_ctl limits, and swap out as many pages as needed
9	* to bring the system back to freepages.high: 2.4.97, Rik van Riel.
10	* Zone aware kswapd started 02/00, Kanoj Sarcar (kanoj@sgi.com).
11	* Multiqueue VM started 5.8.00, Rik van Riel.
12	*/
13
14	#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
15
16	#include <linux/mm.h>
17	#include <linux/module.h>
18	#include <linux/gfp.h>
19	#include <linux/kernel_stat.h>
20	#include <linux/swap.h>
21	#include <linux/pagemap.h>
22	#include <linux/init.h>
23	#include <linux/highmem.h>
24	#include <linux/vmpressure.h>
25	#include <linux/vmstat.h>
26	#include <linux/file.h>
27	#include <linux/writeback.h>
28	#include <linux/blkdev.h>
29	#include <linux/buffer_head.h> /* for try_to_release_page(),
30	buffer_heads_over_limit */
31	#include <linux/mm_inline.h>
32	#include <linux/backing-dev.h>
33	#include <linux/rmap.h>
34	#include <linux/topology.h>
35	#include <linux/cpu.h>
36	#include <linux/cpuset.h>
37	#include <linux/compaction.h>
38	#include <linux/notifier.h>
39	#include <linux/rwsem.h>
40	#include <linux/delay.h>
41	#include <linux/kthread.h>
42	#include <linux/freezer.h>
43	#include <linux/memcontrol.h>
44	#include <linux/delayacct.h>
45	#include <linux/sysctl.h>
46	#include <linux/oom.h>
47	#include <linux/prefetch.h>
48	#include <linux/printk.h>
49	#include <linux/dax.h>
50	#include <linux/psi.h>
51
52	#include <asm/tlbflush.h>
53	#include <asm/div64.h>
54
55	#include <linux/swapops.h>
56	#include <linux/balloon_compaction.h>
57
58	#include "internal.h"
59
60	#define CREATE_TRACE_POINTS
61	#include <trace/events/vmscan.h>
62
63	struct scan_control {
64	/* How many pages shrink_list() should reclaim */
65	unsigned long nr_to_reclaim;
66
67	/* This context's GFP mask */
68	gfp_t gfp_mask;
69
70	/* Allocation order */
71	int order;
72
73	/*
74	* Nodemask of nodes allowed by the caller. If NULL, all nodes
75	* are scanned.
76	*/
77	nodemask_t *nodemask;
78
79	/*
80	* The memory cgroup that hit its limit and as a result is the
81	* primary target of this reclaim invocation.
82	*/
83	struct mem_cgroup *target_mem_cgroup;
84
85	/* Scan (total_size >> priority) pages at once */
86	int priority;
87
88	/* The highest zone to isolate pages for reclaim from */
89	enum zone_type reclaim_idx;
90
91	unsigned int may_writepage:1;
92
93	/* Can mapped pages be reclaimed? */
94	unsigned int may_unmap:1;
95
96	/* Can pages be swapped as part of reclaim? */
97	unsigned int may_swap:1;
98
99	/* Can cgroups be reclaimed below their normal consumption range? */
100	unsigned int may_thrash:1;
101
102	unsigned int hibernation_mode:1;
103
104	/* One of the zones is ready for compaction */
105	unsigned int compaction_ready:1;
106
107	/* Incremented by the number of inactive pages that were scanned */
108	unsigned long nr_scanned;
109
110	/* Number of pages freed so far during a call to shrink_zones() */
111	unsigned long nr_reclaimed;
112	};
113
114	#ifdef ARCH_HAS_PREFETCH
115	#define prefetch_prev_lru_page(_page, _base, _field) \
116	do { \
117	if ((_page)->lru.prev != _base) { \
118	struct page *prev; \
119	\
120	prev = lru_to_page(&(_page->lru)); \
121	prefetch(&prev->_field); \
122	} \
123	} while (0)
124	#else
125	#define prefetch_prev_lru_page(_page, _base, _field) do { } while (0)
126	#endif
127
128	#ifdef ARCH_HAS_PREFETCHW
129	#define prefetchw_prev_lru_page(_page, _base, _field) \
130	do { \
131	if ((_page)->lru.prev != _base) { \
132	struct page *prev; \
133	\
134	prev = lru_to_page(&(_page->lru)); \
135	prefetchw(&prev->_field); \
136	} \
137	} while (0)
138	#else
139	#define prefetchw_prev_lru_page(_page, _base, _field) do { } while (0)
140	#endif
141
142	/*
143	* From 0 .. 100. Higher means more swappy.
144	*/
145	int vm_swappiness = 60;
146	/*
147	* The total number of pages which are beyond the high watermark within all
148	* zones.
149	*/
150	unsigned long vm_total_pages;
151
152	static LIST_HEAD(shrinker_list);
153	static DECLARE_RWSEM(shrinker_rwsem);
154
155	#ifdef CONFIG_MEMCG
156	static bool global_reclaim(struct scan_control *sc)
157	{
158	return !sc->target_mem_cgroup;
159	}
160
161	/**
162	* sane_reclaim - is the usual dirty throttling mechanism operational?
163	* @sc: scan_control in question
164	*
165	* The normal page dirty throttling mechanism in balance_dirty_pages() is
166	* completely broken with the legacy memcg and direct stalling in
167	* shrink_page_list() is used for throttling instead, which lacks all the
168	* niceties such as fairness, adaptive pausing, bandwidth proportional
169	* allocation and configurability.
170	*
171	* This function tests whether the vmscan currently in progress can assume
172	* that the normal dirty throttling mechanism is operational.
173	*/
174	static bool sane_reclaim(struct scan_control *sc)
175	{
176	struct mem_cgroup *memcg = sc->target_mem_cgroup;
177
178	if (!memcg)
179	return true;
180	#ifdef CONFIG_CGROUP_WRITEBACK
181	if (cgroup_subsys_on_dfl(memory_cgrp_subsys))
182	return true;
183	#endif
184	return false;
185	}
186	#else
187	static bool global_reclaim(struct scan_control *sc)
188	{
189	return true;
190	}
191
192	static bool sane_reclaim(struct scan_control *sc)
193	{
194	return true;
195	}
196	#endif
197
198	/*
199	* This misses isolated pages which are not accounted for to save counters.
200	* As the data only determines if reclaim or compaction continues, it is
201	* not expected that isolated pages will be a dominating factor.
202	*/
203	unsigned long zone_reclaimable_pages(struct zone *zone)
204	{
205	unsigned long nr;
206
207	nr = zone_page_state_snapshot(zone, NR_ZONE_INACTIVE_FILE) +
208	zone_page_state_snapshot(zone, NR_ZONE_ACTIVE_FILE);
209	if (get_nr_swap_pages() > 0)
210	nr += zone_page_state_snapshot(zone, NR_ZONE_INACTIVE_ANON) +
211	zone_page_state_snapshot(zone, NR_ZONE_ACTIVE_ANON);
212
213	return nr;
214	}
215
216	unsigned long pgdat_reclaimable_pages(struct pglist_data *pgdat)
217	{
218	unsigned long nr;
219
220	nr = node_page_state_snapshot(pgdat, NR_ACTIVE_FILE) +
221	node_page_state_snapshot(pgdat, NR_INACTIVE_FILE) +
222	node_page_state_snapshot(pgdat, NR_ISOLATED_FILE);
223
224	if (get_nr_swap_pages() > 0)
225	nr += node_page_state_snapshot(pgdat, NR_ACTIVE_ANON) +
226	node_page_state_snapshot(pgdat, NR_INACTIVE_ANON) +
227	node_page_state_snapshot(pgdat, NR_ISOLATED_ANON);
228
229	return nr;
230	}
231
232	bool pgdat_reclaimable(struct pglist_data *pgdat)
233	{
234	return node_page_state_snapshot(pgdat, NR_PAGES_SCANNED) <
235	pgdat_reclaimable_pages(pgdat) * 6;
236	}
237
238	/**
239	* lruvec_lru_size - Returns the number of pages on the given LRU list.
240	* @lruvec: lru vector
241	* @lru: lru to use
242	* @zone_idx: zones to consider (use MAX_NR_ZONES for the whole LRU list)
243	*/
244	unsigned long lruvec_lru_size(struct lruvec *lruvec, enum lru_list lru, int zone_idx)
245	{
246	unsigned long lru_size;
247	int zid;
248
249	if (!mem_cgroup_disabled())
250	lru_size = mem_cgroup_get_lru_size(lruvec, lru);
251	else
252	lru_size = node_page_state(lruvec_pgdat(lruvec), NR_LRU_BASE + lru);
253
254	for (zid = zone_idx + 1; zid < MAX_NR_ZONES; zid++) {
255	struct zone *zone = &lruvec_pgdat(lruvec)->node_zones[zid];
256	unsigned long size;
257
258	if (!managed_zone(zone))
259	continue;
260
261	if (!mem_cgroup_disabled())
262	size = mem_cgroup_get_zone_lru_size(lruvec, lru, zid);
263	else
264	size = zone_page_state(&lruvec_pgdat(lruvec)->node_zones[zid],
265	NR_ZONE_LRU_BASE + lru);
266	lru_size -= min(size, lru_size);
267	}
268
269	return lru_size;
270
271	}
272
273	/*
274	* Add a shrinker callback to be called from the vm.
275	*/
276	int register_shrinker(struct shrinker *shrinker)
277	{
278	size_t size = sizeof(*shrinker->nr_deferred);
279
280	if (shrinker->flags & SHRINKER_NUMA_AWARE)
281	size *= nr_node_ids;
282
283	shrinker->nr_deferred = kzalloc(size, GFP_KERNEL);
284	if (!shrinker->nr_deferred)
285	return -ENOMEM;
286
287	down_write(&shrinker_rwsem);
288	list_add_tail(&shrinker->list, &shrinker_list);
289	up_write(&shrinker_rwsem);
290	return 0;
291	}
292	EXPORT_SYMBOL(register_shrinker);
293
294	/*
295	* Remove one
296	*/
297	void unregister_shrinker(struct shrinker *shrinker)
298	{
299	if (!shrinker->nr_deferred)
300	return;
301	down_write(&shrinker_rwsem);
302	list_del(&shrinker->list);
303	up_write(&shrinker_rwsem);
304	kfree(shrinker->nr_deferred);
305	shrinker->nr_deferred = NULL;
306	}
307	EXPORT_SYMBOL(unregister_shrinker);
308
309	#define SHRINK_BATCH 128
310
311	static unsigned long do_shrink_slab(struct shrink_control *shrinkctl,
312	struct shrinker *shrinker,
313	unsigned long nr_scanned,
314	unsigned long nr_eligible)
315	{
316	unsigned long freed = 0;
317	unsigned long long delta;
318	long total_scan;
319	long freeable;
320	long nr;
321	long new_nr;
322	int nid = shrinkctl->nid;
323	long batch_size = shrinker->batch ? shrinker->batch
324	: SHRINK_BATCH;
325	long scanned = 0, next_deferred;
326
327	freeable = shrinker->count_objects(shrinker, shrinkctl);
328	if (freeable == 0)
329	return 0;
330
331	/*
332	* copy the current shrinker scan count into a local variable
333	* and zero it so that other concurrent shrinker invocations
334	* don't also do this scanning work.
335	*/
336	nr = atomic_long_xchg(&shrinker->nr_deferred[nid], 0);
337
338	total_scan = nr;
339	delta = (4 * nr_scanned) / shrinker->seeks;
340	delta *= freeable;
341	do_div(delta, nr_eligible + 1);
342	total_scan += delta;
343	if (total_scan < 0) {
344	pr_err("shrink_slab: %pF negative objects to delete nr=%ld\n",
345	shrinker->scan_objects, total_scan);
346	total_scan = freeable;
347	next_deferred = nr;
348	} else
349	next_deferred = total_scan;
350
351	/*
352	* We need to avoid excessive windup on filesystem shrinkers
353	* due to large numbers of GFP_NOFS allocations causing the
354	* shrinkers to return -1 all the time. This results in a large
355	* nr being built up so when a shrink that can do some work
356	* comes along it empties the entire cache due to nr >>>
357	* freeable. This is bad for sustaining a working set in
358	* memory.
359	*
360	* Hence only allow the shrinker to scan the entire cache when
361	* a large delta change is calculated directly.
362	*/
363	if (delta < freeable / 4)
364	total_scan = min(total_scan, freeable / 2);
365
366	/*
367	* Avoid risking looping forever due to too large nr value:
368	* never try to free more than twice the estimate number of
369	* freeable entries.
370	*/
371	if (total_scan > freeable * 2)
372	total_scan = freeable * 2;
373
374	trace_mm_shrink_slab_start(shrinker, shrinkctl, nr,
375	nr_scanned, nr_eligible,
376	freeable, delta, total_scan);
377
378	/*
379	* Normally, we should not scan less than batch_size objects in one
380	* pass to avoid too frequent shrinker calls, but if the slab has less
381	* than batch_size objects in total and we are really tight on memory,
382	* we will try to reclaim all available objects, otherwise we can end
383	* up failing allocations although there are plenty of reclaimable
384	* objects spread over several slabs with usage less than the
385	* batch_size.
386	*
387	* We detect the "tight on memory" situations by looking at the total
388	* number of objects we want to scan (total_scan). If it is greater
389	* than the total number of objects on slab (freeable), we must be
390	* scanning at high prio and therefore should try to reclaim as much as
391	* possible.
392	*/
393	while (total_scan >= batch_size ||
394	total_scan >= freeable) {
395	unsigned long ret;
396	unsigned long nr_to_scan = min(batch_size, total_scan);
397
398	shrinkctl->nr_to_scan = nr_to_scan;
399	ret = shrinker->scan_objects(shrinker, shrinkctl);
400	if (ret == SHRINK_STOP)
401	break;
402	freed += ret;
403
404	count_vm_events(SLABS_SCANNED, nr_to_scan);
405	total_scan -= nr_to_scan;
406	scanned += nr_to_scan;
407
408	cond_resched();
409	}
410
411	if (next_deferred >= scanned)
412	next_deferred -= scanned;
413	else
414	next_deferred = 0;
415	/*
416	* move the unused scan count back into the shrinker in a
417	* manner that handles concurrent updates. If we exhausted the
418	* scan, there is no need to do an update.
419	*/
420	if (next_deferred > 0)
421	new_nr = atomic_long_add_return(next_deferred,
422	&shrinker->nr_deferred[nid]);
423	else
424	new_nr = atomic_long_read(&shrinker->nr_deferred[nid]);
425
426	trace_mm_shrink_slab_end(shrinker, nid, freed, nr, new_nr, total_scan);
427	return freed;
428	}
429
430	/**
431	* shrink_slab - shrink slab caches
432	* @gfp_mask: allocation context
433	* @nid: node whose slab caches to target
434	* @memcg: memory cgroup whose slab caches to target
435	* @nr_scanned: pressure numerator
436	* @nr_eligible: pressure denominator
437	*
438	* Call the shrink functions to age shrinkable caches.
439	*
440	* @nid is passed along to shrinkers with SHRINKER_NUMA_AWARE set,
441	* unaware shrinkers will receive a node id of 0 instead.
442	*
443	* @memcg specifies the memory cgroup to target. If it is not NULL,
444	* only shrinkers with SHRINKER_MEMCG_AWARE set will be called to scan
445	* objects from the memory cgroup specified. Otherwise, only unaware
446	* shrinkers are called.
447	*
448	* @nr_scanned and @nr_eligible form a ratio that indicate how much of
449	* the available objects should be scanned. Page reclaim for example
450	* passes the number of pages scanned and the number of pages on the
451	* LRU lists that it considered on @nid, plus a bias in @nr_scanned
452	* when it encountered mapped pages. The ratio is further biased by
453	* the ->seeks setting of the shrink function, which indicates the
454	* cost to recreate an object relative to that of an LRU page.
455	*
456	* Returns the number of reclaimed slab objects.
457	*/
458	static unsigned long shrink_slab(gfp_t gfp_mask, int nid,
459	struct mem_cgroup *memcg,
460	unsigned long nr_scanned,
461	unsigned long nr_eligible)
462	{
463	struct shrinker *shrinker;
464	unsigned long freed = 0;
465
466	if (memcg && (!memcg_kmem_enabled() || !mem_cgroup_online(memcg)))
467	return 0;
468
469	if (nr_scanned == 0)
470	nr_scanned = SWAP_CLUSTER_MAX;
471
472	if (!down_read_trylock(&shrinker_rwsem)) {
473	/*
474	* If we would return 0, our callers would understand that we
475	* have nothing else to shrink and give up trying. By returning
476	* 1 we keep it going and assume we'll be able to shrink next
477	* time.
478	*/
479	freed = 1;
480	goto out;
481	}
482
483	list_for_each_entry(shrinker, &shrinker_list, list) {
484	struct shrink_control sc = {
485	.gfp_mask = gfp_mask,
486	.nid = nid,
487	.memcg = memcg,
488	};
489
490	/*
491	* If kernel memory accounting is disabled, we ignore
492	* SHRINKER_MEMCG_AWARE flag and call all shrinkers
493	* passing NULL for memcg.
494	*/
495	if (memcg_kmem_enabled() &&
496	!!memcg != !!(shrinker->flags & SHRINKER_MEMCG_AWARE))
497	continue;
498
499	if (!(shrinker->flags & SHRINKER_NUMA_AWARE))
500	sc.nid = 0;
501
502	freed += do_shrink_slab(&sc, shrinker, nr_scanned, nr_eligible);
503	}
504
505	up_read(&shrinker_rwsem);
506	out:
507	cond_resched();
508	return freed;
509	}
510
511	void drop_slab_node(int nid)
512	{
513	unsigned long freed;
514
515	do {
516	struct mem_cgroup *memcg = NULL;
517
518	freed = 0;
519	do {
520	freed += shrink_slab(GFP_KERNEL, nid, memcg,
521	1000, 1000);
522	} while ((memcg = mem_cgroup_iter(NULL, memcg, NULL)) != NULL);
523	} while (freed > 10);
524	}
525
526	void drop_slab(void)
527	{
528	int nid;
529
530	for_each_online_node(nid)
531	drop_slab_node(nid);
532	}
533
534	static inline int is_page_cache_freeable(struct page *page)
535	{
536	/*
537	* A freeable page cache page is referenced only by the caller
538	* that isolated the page, the page cache radix tree and
539	* optional buffer heads at page->private.
540	*/
541	return page_count(page) - page_has_private(page) == 2;
542	}
543
544	static int may_write_to_inode(struct inode *inode, struct scan_control *sc)
545	{
546	if (current->flags & PF_SWAPWRITE)
547	return 1;
548	if (!inode_write_congested(inode))
549	return 1;
550	if (inode_to_bdi(inode) == current->backing_dev_info)
551	return 1;
552	return 0;
553	}
554
555	/*
556	* We detected a synchronous write error writing a page out. Probably
557	* -ENOSPC. We need to propagate that into the address_space for a subsequent
558	* fsync(), msync() or close().
559	*
560	* The tricky part is that after writepage we cannot touch the mapping: nothing
561	* prevents it from being freed up. But we have a ref on the page and once
562	* that page is locked, the mapping is pinned.
563	*
564	* We're allowed to run sleeping lock_page() here because we know the caller has
565	* __GFP_FS.
566	*/
567	static void handle_write_error(struct address_space *mapping,
568	struct page *page, int error)
569	{
570	lock_page(page);
571	if (page_mapping(page) == mapping)
572	mapping_set_error(mapping, error);
573	unlock_page(page);
574	}
575
576	/* possible outcome of pageout() */
577	typedef enum {
578	/* failed to write page out, page is locked */
579	PAGE_KEEP,
580	/* move page to the active list, page is locked */
581	PAGE_ACTIVATE,
582	/* page has been sent to the disk successfully, page is unlocked */
583	PAGE_SUCCESS,
584	/* page is clean and locked */
585	PAGE_CLEAN,
586	} pageout_t;
587
588	/*
589	* pageout is called by shrink_page_list() for each dirty page.
590	* Calls ->writepage().
591	*/
592	static pageout_t pageout(struct page *page, struct address_space *mapping,
593	struct scan_control *sc)
594	{
595	/*
596	* If the page is dirty, only perform writeback if that write
597	* will be non-blocking. To prevent this allocation from being
598	* stalled by pagecache activity. But note that there may be
599	* stalls if we need to run get_block(). We could test
600	* PagePrivate for that.
601	*
602	* If this process is currently in __generic_file_write_iter() against
603	* this page's queue, we can perform writeback even if that
604	* will block.
605	*
606	* If the page is swapcache, write it back even if that would
607	* block, for some throttling. This happens by accident, because
608	* swap_backing_dev_info is bust: it doesn't reflect the
609	* congestion state of the swapdevs. Easy to fix, if needed.
610	*/
611	if (!is_page_cache_freeable(page))
612	return PAGE_KEEP;
613	if (!mapping) {
614	/*
615	* Some data journaling orphaned pages can have
616	* page->mapping == NULL while being dirty with clean buffers.
617	*/
618	if (page_has_private(page)) {
619	if (try_to_free_buffers(page)) {
620	ClearPageDirty(page);
621	pr_info("%s: orphaned page\n", __func__);
622	return PAGE_CLEAN;
623	}
624	}
625	return PAGE_KEEP;
626	}
627	if (mapping->a_ops->writepage == NULL)
628	return PAGE_ACTIVATE;
629	if (!may_write_to_inode(mapping->host, sc))
630	return PAGE_KEEP;
631
632	if (clear_page_dirty_for_io(page)) {
633	int res;
634	struct writeback_control wbc = {
635	.sync_mode = WB_SYNC_NONE,
636	.nr_to_write = SWAP_CLUSTER_MAX,
637	.range_start = 0,
638	.range_end = LLONG_MAX,
639	.for_reclaim = 1,
640	};
641
642	SetPageReclaim(page);
643	res = mapping->a_ops->writepage(page, &wbc);
644	if (res < 0)
645	handle_write_error(mapping, page, res);
646	if (res == AOP_WRITEPAGE_ACTIVATE) {
647	ClearPageReclaim(page);
648	return PAGE_ACTIVATE;
649	}
650
651	if (!PageWriteback(page)) {
652	/* synchronous write or broken a_ops? */
653	ClearPageReclaim(page);
654	}
655	trace_mm_vmscan_writepage(page);
656	inc_node_page_state(page, NR_VMSCAN_WRITE);
657	return PAGE_SUCCESS;
658	}
659
660	return PAGE_CLEAN;
661	}
662
663	/*
664	* Same as remove_mapping, but if the page is removed from the mapping, it
665	* gets returned with a refcount of 0.
666	*/
667	static int __remove_mapping(struct address_space *mapping, struct page *page,
668	bool reclaimed)
669	{
670	unsigned long flags;
671
672	BUG_ON(!PageLocked(page));
673	BUG_ON(mapping != page_mapping(page));
674
675	spin_lock_irqsave(&mapping->tree_lock, flags);
676	/*
677	* The non racy check for a busy page.
678	*
679	* Must be careful with the order of the tests. When someone has
680	* a ref to the page, it may be possible that they dirty it then
681	* drop the reference. So if PageDirty is tested before page_count
682	* here, then the following race may occur:
683	*
684	* get_user_pages(&page);
685	* [user mapping goes away]
686	* write_to(page);
687	* !PageDirty(page) [good]
688	* SetPageDirty(page);
689	* put_page(page);
690	* !page_count(page) [good, discard it]
691	*
692	* [oops, our write_to data is lost]
693	*
694	* Reversing the order of the tests ensures such a situation cannot
695	* escape unnoticed. The smp_rmb is needed to ensure the page->flags
696	* load is not satisfied before that of page->_refcount.
697	*
698	* Note that if SetPageDirty is always performed via set_page_dirty,
699	* and thus under tree_lock, then this ordering is not required.
700	*/
701	if (!page_ref_freeze(page, 2))
702	goto cannot_free;
703	/* note: atomic_cmpxchg in page_freeze_refs provides the smp_rmb */
704	if (unlikely(PageDirty(page))) {
705	page_ref_unfreeze(page, 2);
706	goto cannot_free;
707	}
708
709	if (PageSwapCache(page)) {
710	swp_entry_t swap = { .val = page_private(page) };
711	mem_cgroup_swapout(page, swap);
712	__delete_from_swap_cache(page);
713	spin_unlock_irqrestore(&mapping->tree_lock, flags);
714	swapcache_free(swap);
715	} else {
716	void (*freepage)(struct page *);
717	void *shadow = NULL;
718
719	freepage = mapping->a_ops->freepage;
720	/*
721	* Remember a shadow entry for reclaimed file cache in
722	* order to detect refaults, thus thrashing, later on.
723	*
724	* But don't store shadows in an address space that is
725	* already exiting. This is not just an optizimation,
726	* inode reclaim needs to empty out the radix tree or
727	* the nodes are lost. Don't plant shadows behind its
728	* back.
729	*
730	* We also don't store shadows for DAX mappings because the
731	* only page cache pages found in these are zero pages
732	* covering holes, and because we don't want to mix DAX
733	* exceptional entries and shadow exceptional entries in the
734	* same page_tree.
735	*/
736	if (reclaimed && page_is_file_cache(page) &&
737	!mapping_exiting(mapping) && !dax_mapping(mapping))
738	shadow = workingset_eviction(mapping, page);
739	__delete_from_page_cache(page, shadow);
740	spin_unlock_irqrestore(&mapping->tree_lock, flags);
741
742	if (freepage != NULL)
743	freepage(page);
744	}
745
746	return 1;
747
748	cannot_free:
749	spin_unlock_irqrestore(&mapping->tree_lock, flags);
750	return 0;
751	}
752
753	/*
754	* Attempt to detach a locked page from its ->mapping. If it is dirty or if
755	* someone else has a ref on the page, abort and return 0. If it was
756	* successfully detached, return 1. Assumes the caller has a single ref on
757	* this page.
758	*/
759	int remove_mapping(struct address_space *mapping, struct page *page)
760	{
761	if (__remove_mapping(mapping, page, false)) {
762	/*
763	* Unfreezing the refcount with 1 rather than 2 effectively
764	* drops the pagecache ref for us without requiring another
765	* atomic operation.
766	*/
767	page_ref_unfreeze(page, 1);
768	return 1;
769	}
770	return 0;
771	}
772
773	/**
774	* putback_lru_page - put previously isolated page onto appropriate LRU list
775	* @page: page to be put back to appropriate lru list
776	*
777	* Add previously isolated @page to appropriate LRU list.
778	* Page may still be unevictable for other reasons.
779	*
780	* lru_lock must not be held, interrupts must be enabled.
781	*/
782	void putback_lru_page(struct page *page)
783	{
784	bool is_unevictable;
785	int was_unevictable = PageUnevictable(page);
786
787	VM_BUG_ON_PAGE(PageLRU(page), page);
788
789	redo:
790	ClearPageUnevictable(page);
791
792	if (page_evictable(page)) {
793	/*
794	* For evictable pages, we can use the cache.
795	* In event of a race, worst case is we end up with an
796	* unevictable page on [in]active list.
797	* We know how to handle that.
798	*/
799	is_unevictable = false;
800	lru_cache_add(page);
801	} else {
802	/*
803	* Put unevictable pages directly on zone's unevictable
804	* list.
805	*/
806	is_unevictable = true;
807	add_page_to_unevictable_list(page);
808	/*
809	* When racing with an mlock or AS_UNEVICTABLE clearing
810	* (page is unlocked) make sure that if the other thread
811	* does not observe our setting of PG_lru and fails
812	* isolation/check_move_unevictable_pages,
813	* we see PG_mlocked/AS_UNEVICTABLE cleared below and move
814	* the page back to the evictable list.
815	*
816	* The other side is TestClearPageMlocked() or shmem_lock().
817	*/
818	smp_mb();
819	}
820
821	/*
822	* page's status can change while we move it among lru. If an evictable
823	* page is on unevictable list, it never be freed. To avoid that,
824	* check after we added it to the list, again.
825	*/
826	if (is_unevictable && page_evictable(page)) {
827	if (!isolate_lru_page(page)) {
828	put_page(page);
829	goto redo;
830	}
831	/* This means someone else dropped this page from LRU
832	* So, it will be freed or putback to LRU again. There is
833	* nothing to do here.
834	*/
835	}
836
837	if (was_unevictable && !is_unevictable)
838	count_vm_event(UNEVICTABLE_PGRESCUED);
839	else if (!was_unevictable && is_unevictable)
840	count_vm_event(UNEVICTABLE_PGCULLED);
841
842	put_page(page); /* drop ref from isolate */
843	}
844
845	enum page_references {
846	PAGEREF_RECLAIM,
847	PAGEREF_RECLAIM_CLEAN,
848	PAGEREF_KEEP,
849	PAGEREF_ACTIVATE,
850	};
851
852	static enum page_references page_check_references(struct page *page,
853	struct scan_control *sc)
854	{
855	int referenced_ptes, referenced_page;
856	unsigned long vm_flags;
857
858	referenced_ptes = page_referenced(page, 1, sc->target_mem_cgroup,
859	&vm_flags);
860	referenced_page = TestClearPageReferenced(page);
861
862	/*
863	* Mlock lost the isolation race with us. Let try_to_unmap()
864	* move the page to the unevictable list.
865	*/
866	if (vm_flags & VM_LOCKED)
867	return PAGEREF_RECLAIM;
868
869	if (referenced_ptes) {
870	if (PageSwapBacked(page))
871	return PAGEREF_ACTIVATE;
872	/*
873	* All mapped pages start out with page table
874	* references from the instantiating fault, so we need
875	* to look twice if a mapped file page is used more
876	* than once.
877	*
878	* Mark it and spare it for another trip around the
879	* inactive list. Another page table reference will
880	* lead to its activation.
881	*
882	* Note: the mark is set for activated pages as well
883	* so that recently deactivated but used pages are
884	* quickly recovered.
885	*/
886	SetPageReferenced(page);
887
888	if (referenced_page || referenced_ptes > 1)
889	return PAGEREF_ACTIVATE;
890
891	/*
892	* Activate file-backed executable pages after first usage.
893	*/
894	if (vm_flags & VM_EXEC)
895	return PAGEREF_ACTIVATE;
896
897	return PAGEREF_KEEP;
898	}
899
900	/* Reclaim if clean, defer dirty pages to writeback */
901	if (referenced_page && !PageSwapBacked(page))
902	return PAGEREF_RECLAIM_CLEAN;
903
904	return PAGEREF_RECLAIM;
905	}
906
907	/* Check if a page is dirty or under writeback */
908	static void page_check_dirty_writeback(struct page *page,
909	bool *dirty, bool *writeback)
910	{
911	struct address_space *mapping;
912
913	/*
914	* Anonymous pages are not handled by flushers and must be written
915	* from reclaim context. Do not stall reclaim based on them
916	*/
917	if (!page_is_file_cache(page)) {
918	*dirty = false;
919	*writeback = false;
920	return;
921	}
922
923	/* By default assume that the page flags are accurate */
924	*dirty = PageDirty(page);
925	*writeback = PageWriteback(page);
926
927	/* Verify dirty/writeback state if the filesystem supports it */
928	if (!page_has_private(page))
929	return;
930
931	mapping = page_mapping(page);
932	if (mapping && mapping->a_ops->is_dirty_writeback)
933	mapping->a_ops->is_dirty_writeback(page, dirty, writeback);
934	}
935
936	/*
937	* shrink_page_list() returns the number of reclaimed pages
938	*/
939	static unsigned long shrink_page_list(struct list_head *page_list,
940	struct pglist_data *pgdat,
941	struct scan_control *sc,
942	enum ttu_flags ttu_flags,
943	unsigned long *ret_nr_dirty,
944	unsigned long *ret_nr_unqueued_dirty,
945	unsigned long *ret_nr_congested,
946	unsigned long *ret_nr_writeback,
947	unsigned long *ret_nr_immediate,
948	bool force_reclaim)
949	{
950	LIST_HEAD(ret_pages);
951	LIST_HEAD(free_pages);
952	int pgactivate = 0;
953	unsigned long nr_unqueued_dirty = 0;
954	unsigned long nr_dirty = 0;
955	unsigned long nr_congested = 0;
956	unsigned long nr_reclaimed = 0;
957	unsigned long nr_writeback = 0;
958	unsigned long nr_immediate = 0;
959
960	cond_resched();
961
962	while (!list_empty(page_list)) {
963	struct address_space *mapping;
964	struct page *page;
965	int may_enter_fs;
966	enum page_references references = PAGEREF_RECLAIM_CLEAN;
967	bool dirty, writeback;
968	bool lazyfree = false;
969	int ret = SWAP_SUCCESS;
970
971	cond_resched();
972
973	page = lru_to_page(page_list);
974	list_del(&page->lru);
975
976	if (!trylock_page(page))
977	goto keep;
978
979	VM_BUG_ON_PAGE(PageActive(page), page);
980
981	sc->nr_scanned++;
982
983	if (unlikely(!page_evictable(page)))
984	goto cull_mlocked;
985
986	if (!sc->may_unmap && page_mapped(page))
987	goto keep_locked;
988
989	/* Double the slab pressure for mapped and swapcache pages */
990	if (page_mapped(page) || PageSwapCache(page))
991	sc->nr_scanned++;
992
993	may_enter_fs = (sc->gfp_mask & __GFP_FS) ||
994	(PageSwapCache(page) && (sc->gfp_mask & __GFP_IO));
995
996	/*
997	* The number of dirty pages determines if a zone is marked
998	* reclaim_congested which affects wait_iff_congested. kswapd
999	* will stall and start writing pages if the tail of the LRU
1000	* is all dirty unqueued pages.
1001	*/
1002	page_check_dirty_writeback(page, &dirty, &writeback);
1003	if (dirty || writeback)
1004	nr_dirty++;
1005
1006	if (dirty && !writeback)
1007	nr_unqueued_dirty++;
1008
1009	/*
1010	* Treat this page as congested if the underlying BDI is or if
1011	* pages are cycling through the LRU so quickly that the
1012	* pages marked for immediate reclaim are making it to the
1013	* end of the LRU a second time.
1014	*/
1015	mapping = page_mapping(page);
1016	if (((dirty || writeback) && mapping &&
1017	inode_write_congested(mapping->host)) ||
1018	(writeback && PageReclaim(page)))
1019	nr_congested++;
1020
1021	/*
1022	* If a page at the tail of the LRU is under writeback, there
1023	* are three cases to consider.
1024	*
1025	* 1) If reclaim is encountering an excessive number of pages
1026	* under writeback and this page is both under writeback and
1027	* PageReclaim then it indicates that pages are being queued
1028	* for IO but are being recycled through the LRU before the
1029	* IO can complete. Waiting on the page itself risks an
1030	* indefinite stall if it is impossible to writeback the
1031	* page due to IO error or disconnected storage so instead
1032	* note that the LRU is being scanned too quickly and the
1033	* caller can stall after page list has been processed.
1034	*
1035	* 2) Global or new memcg reclaim encounters a page that is
1036	* not marked for immediate reclaim, or the caller does not
1037	* have __GFP_FS (or __GFP_IO if it's simply going to swap,
1038	* not to fs). In this case mark the page for immediate
1039	* reclaim and continue scanning.
1040	*
1041	* Require may_enter_fs because we would wait on fs, which
1042	* may not have submitted IO yet. And the loop driver might
1043	* enter reclaim, and deadlock if it waits on a page for
1044	* which it is needed to do the write (loop masks off
1045	* __GFP_IO|__GFP_FS for this reason); but more thought
1046	* would probably show more reasons.
1047	*
1048	* 3) Legacy memcg encounters a page that is already marked
1049	* PageReclaim. memcg does not have any dirty pages
1050	* throttling so we could easily OOM just because too many
1051	* pages are in writeback and there is nothing else to
1052	* reclaim. Wait for the writeback to complete.
1053	*/
1054	if (PageWriteback(page)) {
1055	/* Case 1 above */
1056	if (current_is_kswapd() &&
1057	PageReclaim(page) &&
1058	test_bit(PGDAT_WRITEBACK, &pgdat->flags)) {
1059	nr_immediate++;
1060	goto keep_locked;
1061
1062	/* Case 2 above */
1063	} else if (sane_reclaim(sc) ||
1064	!PageReclaim(page) || !may_enter_fs) {
1065	/*
1066	* This is slightly racy - end_page_writeback()
1067	* might have just cleared PageReclaim, then
1068	* setting PageReclaim here end up interpreted
1069	* as PageReadahead - but that does not matter
1070	* enough to care. What we do want is for this
1071	* page to have PageReclaim set next time memcg
1072	* reclaim reaches the tests above, so it will
1073	* then wait_on_page_writeback() to avoid OOM;
1074	* and it's also appropriate in global reclaim.
1075	*/
1076	SetPageReclaim(page);
1077	nr_writeback++;
1078	goto keep_locked;
1079
1080	/* Case 3 above */
1081	} else {
1082	unlock_page(page);
1083	wait_on_page_writeback(page);
1084	/* then go back and try same page again */
1085	list_add_tail(&page->lru, page_list);
1086	continue;
1087	}
1088	}
1089
1090	if (!force_reclaim)
1091	references = page_check_references(page, sc);
1092
1093	switch (references) {
1094	case PAGEREF_ACTIVATE:
1095	goto activate_locked;
1096	case PAGEREF_KEEP:
1097	goto keep_locked;
1098	case PAGEREF_RECLAIM:
1099	case PAGEREF_RECLAIM_CLEAN:
1100	; /* try to reclaim the page below */
1101	}
1102
1103	/*
1104	* Anonymous process memory has backing store?
1105	* Try to allocate it some swap space here.
1106	*/
1107	if (PageAnon(page) && !PageSwapCache(page)) {
1108	if (!(sc->gfp_mask & __GFP_IO))
1109	goto keep_locked;
1110	if (!add_to_swap(page, page_list))
1111	goto activate_locked;
1112	lazyfree = true;
1113	may_enter_fs = 1;
1114
1115	/* Adding to swap updated mapping */
1116	mapping = page_mapping(page);
1117	} else if (unlikely(PageTransHuge(page))) {
1118	/* Split file THP */
1119	if (split_huge_page_to_list(page, page_list))
1120	goto keep_locked;
1121	}
1122
1123	VM_BUG_ON_PAGE(PageTransHuge(page), page);
1124
1125	/*
1126	* The page is mapped into the page tables of one or more
1127	* processes. Try to unmap it here.
1128	*/
1129	if (page_mapped(page) && mapping) {
1130	switch (ret = try_to_unmap(page, lazyfree ?
1131	(ttu_flags | TTU_BATCH_FLUSH | TTU_LZFREE) :
1132	(ttu_flags | TTU_BATCH_FLUSH))) {
1133	case SWAP_FAIL:
1134	goto activate_locked;
1135	case SWAP_AGAIN:
1136	goto keep_locked;
1137	case SWAP_MLOCK:
1138	goto cull_mlocked;
1139	case SWAP_LZFREE:
1140	goto lazyfree;
1141	case SWAP_SUCCESS:
1142	; /* try to free the page below */
1143	}
1144	}
1145
1146	if (PageDirty(page)) {
1147	/*
1148	* Only kswapd can writeback filesystem pages to
1149	* avoid risk of stack overflow but only writeback
1150	* if many dirty pages have been encountered.
1151	*/
1152	if (page_is_file_cache(page) &&
1153	(!current_is_kswapd() ||
1154	!test_bit(PGDAT_DIRTY, &pgdat->flags))) {
1155	/*
1156	* Immediately reclaim when written back.
1157	* Similar in principal to deactivate_page()
1158	* except we already have the page isolated
1159	* and know it's dirty
1160	*/
1161	inc_node_page_state(page, NR_VMSCAN_IMMEDIATE);
1162	SetPageReclaim(page);
1163
1164	goto keep_locked;
1165	}
1166
1167	if (references == PAGEREF_RECLAIM_CLEAN)
1168	goto keep_locked;
1169	if (!may_enter_fs)
1170	goto keep_locked;
1171	if (!sc->may_writepage)
1172	goto keep_locked;
1173
1174	/*
1175	* Page is dirty. Flush the TLB if a writable entry
1176	* potentially exists to avoid CPU writes after IO
1177	* starts and then write it out here.
1178	*/
1179	try_to_unmap_flush_dirty();
1180	switch (pageout(page, mapping, sc)) {
1181	case PAGE_KEEP:
1182	goto keep_locked;
1183	case PAGE_ACTIVATE:
1184	goto activate_locked;
1185	case PAGE_SUCCESS:
1186	if (PageWriteback(page))
1187	goto keep;
1188	if (PageDirty(page))
1189	goto keep;
1190
1191	/*
1192	* A synchronous write - probably a ramdisk. Go
1193	* ahead and try to reclaim the page.
1194	*/
1195	if (!trylock_page(page))
1196	goto keep;
1197	if (PageDirty(page) || PageWriteback(page))
1198	goto keep_locked;
1199	mapping = page_mapping(page);
1200	case PAGE_CLEAN:
1201	; /* try to free the page below */
1202	}
1203	}
1204
1205	/*
1206	* If the page has buffers, try to free the buffer mappings
1207	* associated with this page. If we succeed we try to free
1208	* the page as well.
1209	*
1210	* We do this even if the page is PageDirty().
1211	* try_to_release_page() does not perform I/O, but it is
1212	* possible for a page to have PageDirty set, but it is actually
1213	* clean (all its buffers are clean). This happens if the
1214	* buffers were written out directly, with submit_bh(). ext3
1215	* will do this, as well as the blockdev mapping.
1216	* try_to_release_page() will discover that cleanness and will
1217	* drop the buffers and mark the page clean - it can be freed.
1218	*
1219	* Rarely, pages can have buffers and no ->mapping. These are
1220	* the pages which were not successfully invalidated in
1221	* truncate_complete_page(). We try to drop those buffers here
1222	* and if that worked, and the page is no longer mapped into
1223	* process address space (page_count == 1) it can be freed.
1224	* Otherwise, leave the page on the LRU so it is swappable.
1225	*/
1226	if (page_has_private(page)) {
1227	if (!try_to_release_page(page, sc->gfp_mask))
1228	goto activate_locked;
1229	if (!mapping && page_count(page) == 1) {
1230	unlock_page(page);
1231	if (put_page_testzero(page))
1232	goto free_it;
1233	else {
1234	/*
1235	* rare race with speculative reference.
1236	* the speculative reference will free
1237	* this page shortly, so we may
1238	* increment nr_reclaimed here (and
1239	* leave it off the LRU).
1240	*/
1241	nr_reclaimed++;
1242	continue;
1243	}
1244	}
1245	}
1246
1247	lazyfree:
1248	if (!mapping || !__remove_mapping(mapping, page, true))
1249	goto keep_locked;
1250
1251	/*
1252	* At this point, we have no other references and there is
1253	* no way to pick any more up (removed from LRU, removed
1254	* from pagecache). Can use non-atomic bitops now (and
1255	* we obviously don't have to worry about waking up a process
1256	* waiting on the page lock, because there are no references.
1257	*/
1258	__ClearPageLocked(page);
1259	free_it:
1260	if (ret == SWAP_LZFREE)
1261	count_vm_event(PGLAZYFREED);
1262
1263	nr_reclaimed++;
1264
1265	/*
1266	* Is there need to periodically free_page_list? It would
1267	* appear not as the counts should be low
1268	*/
1269	list_add(&page->lru, &free_pages);
1270	#ifdef CONFIG_AMLOGIC_CMA
1271	if (ttu_flags & TTU_IGNORE_ACCESS)
1272	ClearPageCmaAllocating(page);
1273	#endif
1274	continue;
1275
1276	cull_mlocked:
1277	if (PageSwapCache(page))
1278	try_to_free_swap(page);
1279	unlock_page(page);
1280	list_add(&page->lru, &ret_pages);
1281	continue;
1282
1283	activate_locked:
1284	/* Not a candidate for swapping, so reclaim swap space. */
1285	if (PageSwapCache(page) && mem_cgroup_swap_full(page))
1286	try_to_free_swap(page);
1287	VM_BUG_ON_PAGE(PageActive(page), page);
1288	SetPageActive(page);
1289	pgactivate++;
1290	keep_locked:
1291	unlock_page(page);
1292	keep:
1293	list_add(&page->lru, &ret_pages);
1294	VM_BUG_ON_PAGE(PageLRU(page) || PageUnevictable(page), page);
1295	}
1296
1297	mem_cgroup_uncharge_list(&free_pages);
1298	try_to_unmap_flush();
1299	free_hot_cold_page_list(&free_pages, true);
1300
1301	list_splice(&ret_pages, page_list);
1302	count_vm_events(PGACTIVATE, pgactivate);
1303
1304	*ret_nr_dirty += nr_dirty;
1305	*ret_nr_congested += nr_congested;
1306	*ret_nr_unqueued_dirty += nr_unqueued_dirty;
1307	*ret_nr_writeback += nr_writeback;
1308	*ret_nr_immediate += nr_immediate;
1309	return nr_reclaimed;
1310	}
1311
1312	#ifdef CONFIG_AMLOGIC_CMA
1313	#define ACTIVE_MIGRATE 3
1314	#define INACTIVE_MIGRATE (ACTIVE_MIGRATE * 4)
1315	static int filecache_need_migrate(struct page *page)
1316	{
1317	if (PageActive(page) && page_mapcount(page) >= ACTIVE_MIGRATE)
1318	return 1;
1319
1320	if (!PageActive(page) && page_mapcount(page) >= INACTIVE_MIGRATE)
1321	return 1;
1322
1323	if (PageUnevictable(page))
1324	return 0;
1325
1326	return 0;
1327	}
1328	#endif
1329	unsigned long reclaim_clean_pages_from_list(struct zone *zone,
1330	struct list_head *page_list)
1331	{
1332	struct scan_control sc = {
1333	.gfp_mask = GFP_KERNEL,
1334	.priority = DEF_PRIORITY,
1335	.may_unmap = 1,
1336	};
1337	unsigned long ret, dummy1, dummy2, dummy3, dummy4, dummy5;
1338	struct page *page, *next;
1339	LIST_HEAD(clean_pages);
1340	#ifdef CONFIG_AMLOGIC_CMA
1341	LIST_HEAD(high_active_pages);
1342	unsigned long nr_high_active = 0;
1343	unsigned long nr_normal_pages = 0;
1344	#endif
1345
1346	list_for_each_entry_safe(page, next, page_list, lru) {
1347	if (page_is_file_cache(page) && !PageDirty(page) &&
1348	!__PageMovable(page)) {
1349	#ifdef CONFIG_AMLOGIC_CMA
1350	if (filecache_need_migrate(page)) {
1351	/*
1352	* leaving pages with high map count to migrate
1353	* instead of reclaimed. This can help to avoid
1354	* file cache jolt if reclaim large cma size
1355	*/
1356	list_move(&page->lru, &high_active_pages);
1357	nr_high_active++;
1358	} else {
1359	ClearPageActive(page);
1360	list_move(&page->lru, &clean_pages);
1361	nr_normal_pages++;
1362	}
1363	#else
1364	ClearPageActive(page);
1365	list_move(&page->lru, &clean_pages);
1366	#endif
1367	}
1368	}
1369
1370	ret = shrink_page_list(&clean_pages, zone->zone_pgdat, &sc,
1371	TTU_UNMAP|TTU_IGNORE_ACCESS,
1372	&dummy1, &dummy2, &dummy3, &dummy4, &dummy5, true);
1373	list_splice(&clean_pages, page_list);
1374	#ifdef CONFIG_AMLOGIC_CMA
1375	list_splice(&high_active_pages, page_list);
1376	pr_debug("high_active:%4ld, normal:%4ld, reclaimed:%4ld\n",
1377	nr_high_active, nr_normal_pages, ret);
1378	#endif
1379	mod_node_page_state(zone->zone_pgdat, NR_ISOLATED_FILE, -ret);
1380	return ret;
1381	}
1382
1383	/*
1384	* Attempt to remove the specified page from its LRU. Only take this page
1385	* if it is of the appropriate PageActive status. Pages which are being
1386	* freed elsewhere are also ignored.
1387	*
1388	* page: page to consider
1389	* mode: one of the LRU isolation modes defined above
1390	*
1391	* returns 0 on success, -ve errno on failure.
1392	*/
1393	int __isolate_lru_page(struct page *page, isolate_mode_t mode)
1394	{
1395	int ret = -EINVAL;
1396
1397	/* Only take pages on the LRU. */
1398	if (!PageLRU(page))
1399	return ret;
1400
1401	/* Compaction should not handle unevictable pages but CMA can do so */
1402	if (PageUnevictable(page) && !(mode & ISOLATE_UNEVICTABLE))
1403	return ret;
1404
1405	ret = -EBUSY;
1406
1407	/*
1408	* To minimise LRU disruption, the caller can indicate that it only
1409	* wants to isolate pages it will be able to operate on without
1410	* blocking - clean pages for the most part.
1411	*
1412	* ISOLATE_CLEAN means that only clean pages should be isolated. This
1413	* is used by reclaim when it is cannot write to backing storage
1414	*
1415	* ISOLATE_ASYNC_MIGRATE is used to indicate that it only wants to pages
1416	* that it is possible to migrate without blocking
1417	*/
1418	if (mode & (ISOLATE_CLEAN|ISOLATE_ASYNC_MIGRATE)) {
1419	/* All the caller can do on PageWriteback is block */
1420	if (PageWriteback(page))
1421	return ret;
1422
1423	if (PageDirty(page)) {
1424	struct address_space *mapping;
1425	bool migrate_dirty;
1426
1427	/* ISOLATE_CLEAN means only clean pages */
1428	if (mode & ISOLATE_CLEAN)
1429	return ret;
1430
1431	/*
1432	* Only pages without mappings or that have a
1433	* ->migratepage callback are possible to migrate
1434	* without blocking. However, we can be racing with
1435	* truncation so it's necessary to lock the page
1436	* to stabilise the mapping as truncation holds
1437	* the page lock until after the page is removed
1438	* from the page cache.
1439	*/
1440	if (!trylock_page(page))
1441	return ret;
1442
1443	mapping = page_mapping(page);
1444	migrate_dirty = !mapping || mapping->a_ops->migratepage;
1445	unlock_page(page);
1446	if (!migrate_dirty)
1447	return ret;
1448	}
1449	}
1450
1451	if ((mode & ISOLATE_UNMAPPED) && page_mapped(page))
1452	return ret;
1453
1454	if (likely(get_page_unless_zero(page))) {
1455	/*
1456	* Be careful not to clear PageLRU until after we're
1457	* sure the page is not being freed elsewhere -- the
1458	* page release code relies on it.
1459	*/
1460	ClearPageLRU(page);
1461	ret = 0;
1462	}
1463
1464	return ret;
1465	}
1466
1467
1468	/*
1469	* Update LRU sizes after isolating pages. The LRU size updates must
1470	* be complete before mem_cgroup_update_lru_size due to a santity check.
1471	*/
1472	static __always_inline void update_lru_sizes(struct lruvec *lruvec,
1473	enum lru_list lru, unsigned long *nr_zone_taken)
1474	{
1475	int zid;
1476
1477	for (zid = 0; zid < MAX_NR_ZONES; zid++) {
1478	if (!nr_zone_taken[zid])
1479	continue;
1480
1481	__update_lru_size(lruvec, lru, zid, -nr_zone_taken[zid]);
1482	#ifdef CONFIG_MEMCG
1483	mem_cgroup_update_lru_size(lruvec, lru, zid, -nr_zone_taken[zid]);
1484	#endif
1485	}
1486
1487	}
1488
1489	/*
1490	* zone_lru_lock is heavily contended. Some of the functions that
1491	* shrink the lists perform better by taking out a batch of pages
1492	* and working on them outside the LRU lock.
1493	*
1494	* For pagecache intensive workloads, this function is the hottest
1495	* spot in the kernel (apart from copy_*_user functions).
1496	*
1497	* Appropriate locks must be held before calling this function.
1498	*
1499	* @nr_to_scan: The number of pages to look through on the list.
1500	* @lruvec: The LRU vector to pull pages from.
1501	* @dst: The temp list to put pages on to.
1502	* @nr_scanned: The number of pages that were scanned.
1503	* @sc: The scan_control struct for this reclaim session
1504	* @mode: One of the LRU isolation modes
1505	* @lru: LRU list id for isolating
1506	*
1507	* returns how many pages were moved onto *@dst.
1508	*/
1509	static unsigned long isolate_lru_pages(unsigned long nr_to_scan,
1510	struct lruvec *lruvec, struct list_head *dst,
1511	unsigned long *nr_scanned, struct scan_control *sc,
1512	isolate_mode_t mode, enum lru_list lru)
1513	{
1514	struct list_head *src = &lruvec->lists[lru];
1515	unsigned long nr_taken = 0;
1516	unsigned long nr_zone_taken[MAX_NR_ZONES] = { 0 };
1517	unsigned long nr_skipped[MAX_NR_ZONES] = { 0, };
1518	unsigned long scan, nr_pages;
1519	LIST_HEAD(pages_skipped);
1520	#ifdef CONFIG_AMLOGIC_MODIFY
1521	int num = NR_INACTIVE_ANON_CMA - NR_INACTIVE_ANON;
1522	int migrate_type = 0;
1523	#endif /* CONFIG_AMLOGIC_MODIFY */
1524
1525	for (scan = 0; scan < nr_to_scan && nr_taken < nr_to_scan &&
1526	!list_empty(src);) {
1527	struct page *page;
1528
1529	page = lru_to_page(src);
1530	prefetchw_prev_lru_page(page, src, flags);
1531
1532	VM_BUG_ON_PAGE(!PageLRU(page), page);
1533
1534	if (page_zonenum(page) > sc->reclaim_idx) {
1535	list_move(&page->lru, &pages_skipped);
1536	nr_skipped[page_zonenum(page)]++;
1537	continue;
1538	}
1539
1540	/*
1541	* Account for scanned and skipped separetly to avoid the pgdat
1542	* being prematurely marked unreclaimable by pgdat_reclaimable.
1543	*/
1544	scan++;
1545
1546	switch (__isolate_lru_page(page, mode)) {
1547	case 0:
1548	nr_pages = hpage_nr_pages(page);
1549	nr_taken += nr_pages;
1550	nr_zone_taken[page_zonenum(page)] += nr_pages;
1551	list_move(&page->lru, dst);
1552	#ifdef CONFIG_AMLOGIC_MODIFY
1553	migrate_type = get_pageblock_migratetype(page);
1554	if (is_migrate_cma(migrate_type) ||
1555	is_migrate_isolate(migrate_type))
1556	__mod_zone_page_state(page_zone(page),
1557	NR_LRU_BASE + lru + num,
1558	-nr_pages);
1559	#endif /* CONFIG_AMLOGIC_MODIFY */
1560	break;
1561
1562	case -EBUSY:
1563	/* else it is being freed elsewhere */
1564	list_move(&page->lru, src);
1565	continue;
1566
1567	default:
1568	BUG();
1569	}
1570	}
1571
1572	/*
1573	* Splice any skipped pages to the start of the LRU list. Note that
1574	* this disrupts the LRU order when reclaiming for lower zones but
1575	* we cannot splice to the tail. If we did then the SWAP_CLUSTER_MAX
1576	* scanning would soon rescan the same pages to skip and put the
1577	* system at risk of premature OOM.
1578	*/
1579	if (!list_empty(&pages_skipped)) {
1580	int zid;
1581	unsigned long total_skipped = 0;
1582
1583	for (zid = 0; zid < MAX_NR_ZONES; zid++) {
1584	if (!nr_skipped[zid])
1585	continue;
1586
1587	__count_zid_vm_events(PGSCAN_SKIP, zid, nr_skipped[zid]);
1588	total_skipped += nr_skipped[zid];
1589	}
1590
1591	/*
1592	* Account skipped pages as a partial scan as the pgdat may be
1593	* close to unreclaimable. If the LRU list is empty, account
1594	* skipped pages as a full scan.
1595	*/
1596	scan += list_empty(src) ? total_skipped : total_skipped >> 2;
1597
1598	list_splice(&pages_skipped, src);
1599	}
1600	*nr_scanned = scan;
1601	trace_mm_vmscan_lru_isolate(sc->reclaim_idx, sc->order, nr_to_scan, scan,
1602	nr_taken, mode, is_file_lru(lru));
1603	update_lru_sizes(lruvec, lru, nr_zone_taken);
1604	return nr_taken;
1605	}
1606
1607	/**
1608	* isolate_lru_page - tries to isolate a page from its LRU list
1609	* @page: page to isolate from its LRU list
1610	*
1611	* Isolates a @page from an LRU list, clears PageLRU and adjusts the
1612	* vmstat statistic corresponding to whatever LRU list the page was on.
1613	*
1614	* Returns 0 if the page was removed from an LRU list.
1615	* Returns -EBUSY if the page was not on an LRU list.
1616	*
1617	* The returned page will have PageLRU() cleared. If it was found on
1618	* the active list, it will have PageActive set. If it was found on
1619	* the unevictable list, it will have the PageUnevictable bit set. That flag
1620	* may need to be cleared by the caller before letting the page go.
1621	*
1622	* The vmstat statistic corresponding to the list on which the page was
1623	* found will be decremented.
1624	*
1625	* Restrictions:
1626	* (1) Must be called with an elevated refcount on the page. This is a
1627	* fundamentnal difference from isolate_lru_pages (which is called
1628	* without a stable reference).
1629	* (2) the lru_lock must not be held.
1630	* (3) interrupts must be enabled.
1631	*/
1632	int isolate_lru_page(struct page *page)
1633	{
1634	int ret = -EBUSY;
1635
1636	VM_BUG_ON_PAGE(!page_count(page), page);
1637	WARN_RATELIMIT(PageTail(page), "trying to isolate tail page");
1638
1639	if (PageLRU(page)) {
1640	struct zone *zone = page_zone(page);
1641	struct lruvec *lruvec;
1642
1643	spin_lock_irq(zone_lru_lock(zone));
1644	lruvec = mem_cgroup_page_lruvec(page, zone->zone_pgdat);
1645	if (PageLRU(page)) {
1646	int lru = page_lru(page);
1647	get_page(page);
1648	ClearPageLRU(page);
1649	del_page_from_lru_list(page, lruvec, lru);
1650	ret = 0;
1651	}
1652	spin_unlock_irq(zone_lru_lock(zone));
1653	}
1654	return ret;
1655	}
1656
1657	/*
1658	* A direct reclaimer may isolate SWAP_CLUSTER_MAX pages from the LRU list and
1659	* then get resheduled. When there are massive number of tasks doing page
1660	* allocation, such sleeping direct reclaimers may keep piling up on each CPU,
1661	* the LRU list will go small and be scanned faster than necessary, leading to
1662	* unnecessary swapping, thrashing and OOM.
1663	*/
1664	static int too_many_isolated(struct pglist_data *pgdat, int file,
1665	struct scan_control *sc)
1666	{
1667	#ifdef CONFIG_AMLOGIC_CMA
1668	signed long inactive, isolated;
1669	#else
1670	unsigned long inactive, isolated;
1671	#endif /* CONFIG_AMLOGIC_CMA */
1672
1673	if (current_is_kswapd())
1674	return 0;
1675
1676	if (!sane_reclaim(sc))
1677	return 0;
1678
1679	if (file) {
1680	inactive = node_page_state(pgdat, NR_INACTIVE_FILE);
1681	isolated = node_page_state(pgdat, NR_ISOLATED_FILE);
1682	} else {
1683	inactive = node_page_state(pgdat, NR_INACTIVE_ANON);
1684	isolated = node_page_state(pgdat, NR_ISOLATED_ANON);
1685	}
1686
1687	#ifdef CONFIG_AMLOGIC_CMA
1688	isolated -= node_page_state(pgdat, NR_CMA_ISOLATED);
1689	#endif /* CONFIG_AMLOGIC_CMA */
1690	/*
1691	* GFP_NOIO/GFP_NOFS callers are allowed to isolate more pages, so they
1692	* won't get blocked by normal direct-reclaimers, forming a circular
1693	* deadlock.
1694	*/
1695	#ifndef CONFIG_AMLOGIC_MODIFY
1696	if ((sc->gfp_mask & (__GFP_IO | __GFP_FS)) == (__GFP_IO | __GFP_FS))
1697	inactive >>= 3;
1698	#endif
1699	#ifdef CONFIG_AMLOGIC_CMA
1700	WARN_ONCE(isolated > inactive,
1701	"isolated:%ld, cma:%ld, inactive:%ld, mask:%x, file:%d\n",
1702	isolated, node_page_state(pgdat, NR_CMA_ISOLATED),
1703	inactive, sc->gfp_mask, file);
1704	#endif /* CONFIG_AMLOGIC_CMA */
1705	return isolated > inactive;
1706	}
1707
1708	static noinline_for_stack void
1709	putback_inactive_pages(struct lruvec *lruvec, struct list_head *page_list)
1710	{
1711	struct zone_reclaim_stat *reclaim_stat = &lruvec->reclaim_stat;
1712	struct pglist_data *pgdat = lruvec_pgdat(lruvec);
1713	LIST_HEAD(pages_to_free);
1714
1715	/*
1716	* Put back any unfreeable pages.
1717	*/
1718	while (!list_empty(page_list)) {
1719	struct page *page = lru_to_page(page_list);
1720	int lru;
1721
1722	VM_BUG_ON_PAGE(PageLRU(page), page);
1723	list_del(&page->lru);
1724	if (unlikely(!page_evictable(page))) {
1725	spin_unlock_irq(&pgdat->lru_lock);
1726	putback_lru_page(page);
1727	spin_lock_irq(&pgdat->lru_lock);
1728	continue;
1729	}
1730
1731	lruvec = mem_cgroup_page_lruvec(page, pgdat);
1732
1733	SetPageLRU(page);
1734	lru = page_lru(page);
1735	add_page_to_lru_list(page, lruvec, lru);
1736
1737	if (is_active_lru(lru)) {
1738	int file = is_file_lru(lru);
1739	int numpages = hpage_nr_pages(page);
1740	reclaim_stat->recent_rotated[file] += numpages;
1741	}
1742	if (put_page_testzero(page)) {
1743	__ClearPageLRU(page);
1744	__ClearPageActive(page);
1745	del_page_from_lru_list(page, lruvec, lru);
1746
1747	if (unlikely(PageCompound(page))) {
1748	spin_unlock_irq(&pgdat->lru_lock);
1749	mem_cgroup_uncharge(page);
1750	(*get_compound_page_dtor(page))(page);
1751	spin_lock_irq(&pgdat->lru_lock);
1752	} else
1753	list_add(&page->lru, &pages_to_free);
1754	}
1755	}
1756
1757	/*
1758	* To save our caller's stack, now use input list for pages to free.
1759	*/
1760	list_splice(&pages_to_free, page_list);
1761	}
1762
1763	/*
1764	* If a kernel thread (such as nfsd for loop-back mounts) services
1765	* a backing device by writing to the page cache it sets PF_LESS_THROTTLE.
1766	* In that case we should only throttle if the backing device it is
1767	* writing to is congested. In other cases it is safe to throttle.
1768	*/
1769	static int current_may_throttle(void)
1770	{
1771	return !(current->flags & PF_LESS_THROTTLE) ||
1772	current->backing_dev_info == NULL ||
1773	bdi_write_congested(current->backing_dev_info);
1774	}
1775
1776	static bool inactive_reclaimable_pages(struct lruvec *lruvec,
1777	struct scan_control *sc, enum lru_list lru)
1778	{
1779	int zid;
1780	struct zone *zone;
1781	int file = is_file_lru(lru);
1782	struct pglist_data *pgdat = lruvec_pgdat(lruvec);
1783
1784	if (!global_reclaim(sc))
1785	return true;
1786
1787	for (zid = sc->reclaim_idx; zid >= 0; zid--) {
1788	zone = &pgdat->node_zones[zid];
1789	if (!managed_zone(zone))
1790	continue;
1791
1792	if (zone_page_state_snapshot(zone, NR_ZONE_LRU_BASE +
1793	LRU_FILE * file) >= SWAP_CLUSTER_MAX)
1794	return true;
1795	}
1796
1797	return false;
1798	}
1799
1800	/*
1801	* shrink_inactive_list() is a helper for shrink_node(). It returns the number
1802	* of reclaimed pages
1803	*/
1804	static noinline_for_stack unsigned long
1805	shrink_inactive_list(unsigned long nr_to_scan, struct lruvec *lruvec,
1806	struct scan_control *sc, enum lru_list lru)
1807	{
1808	LIST_HEAD(page_list);
1809	unsigned long nr_scanned;
1810	unsigned long nr_reclaimed = 0;
1811	unsigned long nr_taken;
1812	unsigned long nr_dirty = 0;
1813	unsigned long nr_congested = 0;
1814	unsigned long nr_unqueued_dirty = 0;
1815	unsigned long nr_writeback = 0;
1816	unsigned long nr_immediate = 0;
1817	isolate_mode_t isolate_mode = 0;
1818	int file = is_file_lru(lru);
1819	struct pglist_data *pgdat = lruvec_pgdat(lruvec);
1820	struct zone_reclaim_stat *reclaim_stat = &lruvec->reclaim_stat;
1821
1822	if (!inactive_reclaimable_pages(lruvec, sc, lru))
1823	return 0;
1824
1825	while (unlikely(too_many_isolated(pgdat, file, sc))) {
1826	congestion_wait(BLK_RW_ASYNC, HZ/10);
1827
1828	/* We are about to die and free our memory. Return now. */
1829	if (fatal_signal_pending(current))
1830	return SWAP_CLUSTER_MAX;
1831	}
1832
1833	lru_add_drain();
1834
1835	if (!sc->may_unmap)
1836	isolate_mode |= ISOLATE_UNMAPPED;
1837	if (!sc->may_writepage)
1838	isolate_mode |= ISOLATE_CLEAN;
1839
1840	spin_lock_irq(&pgdat->lru_lock);
1841
1842	nr_taken = isolate_lru_pages(nr_to_scan, lruvec, &page_list,
1843	&nr_scanned, sc, isolate_mode, lru);
1844
1845	__mod_node_page_state(pgdat, NR_ISOLATED_ANON + file, nr_taken);
1846	reclaim_stat->recent_scanned[file] += nr_taken;
1847
1848	if (global_reclaim(sc)) {
1849	__mod_node_page_state(pgdat, NR_PAGES_SCANNED, nr_scanned);
1850	if (current_is_kswapd())
1851	__count_vm_events(PGSCAN_KSWAPD, nr_scanned);
1852	else
1853	__count_vm_events(PGSCAN_DIRECT, nr_scanned);
1854	}
1855	spin_unlock_irq(&pgdat->lru_lock);
1856
1857	if (nr_taken == 0)
1858	return 0;
1859
1860	nr_reclaimed = shrink_page_list(&page_list, pgdat, sc, TTU_UNMAP,
1861	&nr_dirty, &nr_unqueued_dirty, &nr_congested,
1862	&nr_writeback, &nr_immediate,
1863	false);
1864
1865	spin_lock_irq(&pgdat->lru_lock);
1866
1867	if (global_reclaim(sc)) {
1868	if (current_is_kswapd())
1869	__count_vm_events(PGSTEAL_KSWAPD, nr_reclaimed);
1870	else
1871	__count_vm_events(PGSTEAL_DIRECT, nr_reclaimed);
1872	}
1873
1874	putback_inactive_pages(lruvec, &page_list);
1875
1876	__mod_node_page_state(pgdat, NR_ISOLATED_ANON + file, -nr_taken);
1877
1878	spin_unlock_irq(&pgdat->lru_lock);
1879
1880	mem_cgroup_uncharge_list(&page_list);
1881	free_hot_cold_page_list(&page_list, true);
1882
1883	/*
1884	* If reclaim is isolating dirty pages under writeback, it implies
1885	* that the long-lived page allocation rate is exceeding the page
1886	* laundering rate. Either the global limits are not being effective
1887	* at throttling processes due to the page distribution throughout
1888	* zones or there is heavy usage of a slow backing device. The
1889	* only option is to throttle from reclaim context which is not ideal
1890	* as there is no guarantee the dirtying process is throttled in the
1891	* same way balance_dirty_pages() manages.
1892	*
1893	* Once a zone is flagged ZONE_WRITEBACK, kswapd will count the number
1894	* of pages under pages flagged for immediate reclaim and stall if any
1895	* are encountered in the nr_immediate check below.
1896	*/
1897	if (nr_writeback && nr_writeback == nr_taken)
1898	set_bit(PGDAT_WRITEBACK, &pgdat->flags);
1899
1900	/*
1901	* Legacy memcg will stall in page writeback so avoid forcibly
1902	* stalling here.
1903	*/
1904	if (sane_reclaim(sc)) {
1905	/*
1906	* Tag a zone as congested if all the dirty pages scanned were
1907	* backed by a congested BDI and wait_iff_congested will stall.
1908	*/
1909	if (nr_dirty && nr_dirty == nr_congested)
1910	set_bit(PGDAT_CONGESTED, &pgdat->flags);
1911
1912	/*
1913	* If dirty pages are scanned that are not queued for IO, it
1914	* implies that flushers are not keeping up. In this case, flag
1915	* the pgdat PGDAT_DIRTY and kswapd will start writing pages from
1916	* reclaim context.
1917	*/
1918	if (nr_unqueued_dirty == nr_taken)
1919	set_bit(PGDAT_DIRTY, &pgdat->flags);
1920
1921	/*
1922	* If kswapd scans pages marked marked for immediate
1923	* reclaim and under writeback (nr_immediate), it implies
1924	* that pages are cycling through the LRU faster than
1925	* they are written so also forcibly stall.
1926	*/
1927	if (nr_immediate && current_may_throttle())
1928	congestion_wait(BLK_RW_ASYNC, HZ/10);
1929	}
1930
1931	/*
1932	* Stall direct reclaim for IO completions if underlying BDIs or zone
1933	* is congested. Allow kswapd to continue until it starts encountering
1934	* unqueued dirty pages or cycling through the LRU too quickly.
1935	*/
1936	if (!sc->hibernation_mode && !current_is_kswapd() &&
1937	current_may_throttle())
1938	wait_iff_congested(pgdat, BLK_RW_ASYNC, HZ/10);
1939
1940	trace_mm_vmscan_lru_shrink_inactive(pgdat->node_id,
1941	nr_scanned, nr_reclaimed,
1942	sc->priority, file);
1943	return nr_reclaimed;
1944	}
1945
1946	/*
1947	* This moves pages from the active list to the inactive list.
1948	*
1949	* We move them the other way if the page is referenced by one or more
1950	* processes, from rmap.
1951	*
1952	* If the pages are mostly unmapped, the processing is fast and it is
1953	* appropriate to hold zone_lru_lock across the whole operation. But if
1954	* the pages are mapped, the processing is slow (page_referenced()) so we
1955	* should drop zone_lru_lock around each page. It's impossible to balance
1956	* this, so instead we remove the pages from the LRU while processing them.
1957	* It is safe to rely on PG_active against the non-LRU pages in here because
1958	* nobody will play with that bit on a non-LRU page.
1959	*
1960	* The downside is that we have to touch page->_refcount against each page.
1961	* But we had to alter page->flags anyway.
1962	*/
1963
1964	static void move_active_pages_to_lru(struct lruvec *lruvec,
1965	struct list_head *list,
1966	struct list_head *pages_to_free,
1967	enum lru_list lru)
1968	{
1969	struct pglist_data *pgdat = lruvec_pgdat(lruvec);
1970	unsigned long pgmoved = 0;
1971	struct page *page;
1972	int nr_pages;
1973	#ifdef CONFIG_AMLOGIC_MODIFY
1974	int num = NR_INACTIVE_ANON_CMA - NR_INACTIVE_ANON;
1975	int migrate_type = 0;
1976	#endif /* CONFIG_AMLOGIC_MODIFY */
1977
1978	while (!list_empty(list)) {
1979	page = lru_to_page(list);
1980	lruvec = mem_cgroup_page_lruvec(page, pgdat);
1981
1982	VM_BUG_ON_PAGE(PageLRU(page), page);
1983	SetPageLRU(page);
1984
1985	nr_pages = hpage_nr_pages(page);
1986	update_lru_size(lruvec, lru, page_zonenum(page), nr_pages);
1987	list_move(&page->lru, &lruvec->lists[lru]);
1988	pgmoved += nr_pages;
1989	#ifdef CONFIG_AMLOGIC_MODIFY
1990	migrate_type = get_pageblock_migratetype(page);
1991	if (is_migrate_cma(migrate_type) ||
1992	is_migrate_isolate(migrate_type))
1993	__mod_zone_page_state(page_zone(page),
1994	NR_LRU_BASE + lru + num,
1995	nr_pages);
1996	#endif /* CONFIG_AMLOGIC_MODIFY */
1997
1998	if (put_page_testzero(page)) {
1999	__ClearPageLRU(page);
2000	__ClearPageActive(page);
2001	del_page_from_lru_list(page, lruvec, lru);
2002
2003	if (unlikely(PageCompound(page))) {
2004	spin_unlock_irq(&pgdat->lru_lock);
2005	mem_cgroup_uncharge(page);
2006	(*get_compound_page_dtor(page))(page);
2007	spin_lock_irq(&pgdat->lru_lock);
2008	} else
2009	list_add(&page->lru, pages_to_free);
2010	}
2011	}
2012
2013	if (!is_active_lru(lru))
2014	__count_vm_events(PGDEACTIVATE, pgmoved);
2015	}
2016
2017	static void shrink_active_list(unsigned long nr_to_scan,
2018	struct lruvec *lruvec,
2019	struct scan_control *sc,
2020	enum lru_list lru)
2021	{
2022	unsigned long nr_taken;
2023	unsigned long nr_scanned;
2024	unsigned long vm_flags;
2025	LIST_HEAD(l_hold); /* The pages which were snipped off */
2026	LIST_HEAD(l_active);
2027	LIST_HEAD(l_inactive);
2028	struct page *page;
2029	struct zone_reclaim_stat *reclaim_stat = &lruvec->reclaim_stat;
2030	unsigned long nr_rotated = 0;
2031	isolate_mode_t isolate_mode = 0;
2032	int file = is_file_lru(lru);
2033	struct pglist_data *pgdat = lruvec_pgdat(lruvec);
2034
2035	lru_add_drain();
2036
2037	if (!sc->may_unmap)
2038	isolate_mode |= ISOLATE_UNMAPPED;
2039	if (!sc->may_writepage)
2040	isolate_mode |= ISOLATE_CLEAN;
2041
2042	spin_lock_irq(&pgdat->lru_lock);
2043
2044	nr_taken = isolate_lru_pages(nr_to_scan, lruvec, &l_hold,
2045	&nr_scanned, sc, isolate_mode, lru);
2046
2047	__mod_node_page_state(pgdat, NR_ISOLATED_ANON + file, nr_taken);
2048	reclaim_stat->recent_scanned[file] += nr_taken;
2049
2050	if (global_reclaim(sc))
2051	__mod_node_page_state(pgdat, NR_PAGES_SCANNED, nr_scanned);
2052	__count_vm_events(PGREFILL, nr_scanned);
2053
2054	spin_unlock_irq(&pgdat->lru_lock);
2055
2056	while (!list_empty(&l_hold)) {
2057	cond_resched();
2058	page = lru_to_page(&l_hold);
2059	list_del(&page->lru);
2060
2061	if (unlikely(!page_evictable(page))) {
2062	putback_lru_page(page);
2063	continue;
2064	}
2065
2066	if (unlikely(buffer_heads_over_limit)) {
2067	if (page_has_private(page) && trylock_page(page)) {
2068	if (page_has_private(page))
2069	try_to_release_page(page, 0);
2070	unlock_page(page);
2071	}
2072	}
2073
2074	if (page_referenced(page, 0, sc->target_mem_cgroup,
2075	&vm_flags)) {
2076	nr_rotated += hpage_nr_pages(page);
2077	/*
2078	* Identify referenced, file-backed active pages and
2079	* give them one more trip around the active list. So
2080	* that executable code get better chances to stay in
2081	* memory under moderate memory pressure. Anon pages
2082	* are not likely to be evicted by use-once streaming
2083	* IO, plus JVM can create lots of anon VM_EXEC pages,
2084	* so we ignore them here.
2085	*/
2086	if ((vm_flags & VM_EXEC) && page_is_file_cache(page)) {
2087	list_add(&page->lru, &l_active);
2088	continue;
2089	}
2090	}
2091
2092	ClearPageActive(page); /* we are de-activating */
2093	SetPageWorkingset(page);
2094	list_add(&page->lru, &l_inactive);
2095	}
2096
2097	/*
2098	* Move pages back to the lru list.
2099	*/
2100	spin_lock_irq(&pgdat->lru_lock);
2101	/*
2102	* Count referenced pages from currently used mappings as rotated,
2103	* even though only some of them are actually re-activated. This
2104	* helps balance scan pressure between file and anonymous pages in
2105	* get_scan_count.
2106	*/
2107	reclaim_stat->recent_rotated[file] += nr_rotated;
2108
2109	move_active_pages_to_lru(lruvec, &l_active, &l_hold, lru);
2110	move_active_pages_to_lru(lruvec, &l_inactive, &l_hold, lru - LRU_ACTIVE);
2111	__mod_node_page_state(pgdat, NR_ISOLATED_ANON + file, -nr_taken);
2112	spin_unlock_irq(&pgdat->lru_lock);
2113
2114	mem_cgroup_uncharge_list(&l_hold);
2115	free_hot_cold_page_list(&l_hold, true);
2116	}
2117
2118	/*
2119	* The inactive anon list should be small enough that the VM never has
2120	* to do too much work.
2121	*
2122	* The inactive file list should be small enough to leave most memory
2123	* to the established workingset on the scan-resistant active list,
2124	* but large enough to avoid thrashing the aggregate readahead window.
2125	*
2126	* Both inactive lists should also be large enough that each inactive
2127	* page has a chance to be referenced again before it is reclaimed.
2128	*
2129	* The inactive_ratio is the target ratio of ACTIVE to INACTIVE pages
2130	* on this LRU, maintained by the pageout code. A zone->inactive_ratio
2131	* of 3 means 3:1 or 25% of the pages are kept on the inactive list.
2132	*
2133	* total target max
2134	* memory ratio inactive
2135	* -------------------------------------
2136	* 10MB 1 5MB
2137	* 100MB 1 50MB
2138	* 1GB 3 250MB
2139	* 10GB 10 0.9GB
2140	* 100GB 31 3GB
2141	* 1TB 101 10GB
2142	* 10TB 320 32GB
2143	*/
2144	static bool inactive_list_is_low(struct lruvec *lruvec, bool file,
2145	struct scan_control *sc)
2146	{
2147	unsigned long inactive_ratio;
2148	unsigned long inactive, active;
2149	enum lru_list inactive_lru = file * LRU_FILE;
2150	enum lru_list active_lru = file * LRU_FILE + LRU_ACTIVE;
2151	unsigned long gb;
2152
2153	/*
2154	* If we don't have swap space, anonymous page deactivation
2155	* is pointless.
2156	*/
2157	if (!file && !total_swap_pages)
2158	return false;
2159
2160	inactive = lruvec_lru_size(lruvec, inactive_lru, sc->reclaim_idx);
2161	active = lruvec_lru_size(lruvec, active_lru, sc->reclaim_idx);
2162
2163	gb = (inactive + active) >> (30 - PAGE_SHIFT);
2164	if (gb)
2165	inactive_ratio = int_sqrt(10 * gb);
2166	#ifdef CONFIG_AMLOGIC_MEMORY_EXTEND
2167	else if (!file && (totalram_pages >> (20 - PAGE_SHIFT)) >= 512)
2168	inactive_ratio = 2;
2169	#endif /* CONFIG_AMLOGIC_MEMORY_EXTEND */
2170	else
2171	inactive_ratio = 1;
2172
2173	return inactive * inactive_ratio < active;
2174	}
2175
2176	static unsigned long shrink_list(enum lru_list lru, unsigned long nr_to_scan,
2177	struct lruvec *lruvec, struct scan_control *sc)
2178	{
2179	if (is_active_lru(lru)) {
2180	if (inactive_list_is_low(lruvec, is_file_lru(lru), sc))
2181	shrink_active_list(nr_to_scan, lruvec, sc, lru);
2182	return 0;
2183	}
2184
2185	return shrink_inactive_list(nr_to_scan, lruvec, sc, lru);
2186	}
2187
2188	enum scan_balance {
2189	SCAN_EQUAL,
2190	SCAN_FRACT,
2191	SCAN_ANON,
2192	SCAN_FILE,
2193	};
2194
2195	/*
2196	* Determine how aggressively the anon and file LRU lists should be
2197	* scanned. The relative value of each set of LRU lists is determined
2198	* by looking at the fraction of the pages scanned we did rotate back
2199	* onto the active list instead of evict.
2200	*
2201	* nr[0] = anon inactive pages to scan; nr[1] = anon active pages to scan
2202	* nr[2] = file inactive pages to scan; nr[3] = file active pages to scan
2203	*/
2204	static void get_scan_count(struct lruvec *lruvec, struct mem_cgroup *memcg,
2205	struct scan_control *sc, unsigned long *nr,
2206	unsigned long *lru_pages)
2207	{
2208	int swappiness = mem_cgroup_swappiness(memcg);
2209	struct zone_reclaim_stat *reclaim_stat = &lruvec->reclaim_stat;
2210	u64 fraction[2];
2211	u64 denominator = 0; /* gcc */
2212	struct pglist_data *pgdat = lruvec_pgdat(lruvec);
2213	unsigned long anon_prio, file_prio;
2214	enum scan_balance scan_balance;
2215	unsigned long anon, file;
2216	bool force_scan = false;
2217	unsigned long ap, fp;
2218	enum lru_list lru;
2219	bool some_scanned;
2220	int pass;
2221
2222	/*
2223	* If the zone or memcg is small, nr[l] can be 0. This
2224	* results in no scanning on this priority and a potential
2225	* priority drop. Global direct reclaim can go to the next
2226	* zone and tends to have no problems. Global kswapd is for
2227	* zone balancing and it needs to scan a minimum amount. When
2228	* reclaiming for a memcg, a priority drop can cause high
2229	* latencies, so it's better to scan a minimum amount there as
2230	* well.
2231	*/
2232	if (current_is_kswapd()) {
2233	if (!pgdat_reclaimable(pgdat))
2234	force_scan = true;
2235	if (!mem_cgroup_online(memcg))
2236	force_scan = true;
2237	}
2238	if (!global_reclaim(sc))
2239	force_scan = true;
2240
2241	/* If we have no swap space, do not bother scanning anon pages. */
2242	if (!sc->may_swap || mem_cgroup_get_nr_swap_pages(memcg) <= 0) {
2243	scan_balance = SCAN_FILE;
2244	goto out;
2245	}
2246
2247	/*
2248	* Global reclaim will swap to prevent OOM even with no
2249	* swappiness, but memcg users want to use this knob to
2250	* disable swapping for individual groups completely when
2251	* using the memory controller's swap limit feature would be
2252	* too expensive.
2253	*/
2254	if (!global_reclaim(sc) && !swappiness) {
2255	scan_balance = SCAN_FILE;
2256	goto out;
2257	}
2258
2259	/*
2260	* Do not apply any pressure balancing cleverness when the
2261	* system is close to OOM, scan both anon and file equally
2262	* (unless the swappiness setting disagrees with swapping).
2263	*/
2264	if (!sc->priority && swappiness) {
2265	scan_balance = SCAN_EQUAL;
2266	goto out;
2267	}
2268
2269	/*
2270	* Prevent the reclaimer from falling into the cache trap: as
2271	* cache pages start out inactive, every cache fault will tip
2272	* the scan balance towards the file LRU. And as the file LRU
2273	* shrinks, so does the window for rotation from references.
2274	* This means we have a runaway feedback loop where a tiny
2275	* thrashing file LRU becomes infinitely more attractive than
2276	* anon pages. Try to detect this based on file LRU size.
2277	*/
2278	if (global_reclaim(sc)) {
2279	unsigned long pgdatfile;
2280	unsigned long pgdatfree;
2281	int z;
2282	unsigned long total_high_wmark = 0;
2283
2284	pgdatfree = sum_zone_node_page_state(pgdat->node_id, NR_FREE_PAGES);
2285	pgdatfile = node_page_state(pgdat, NR_ACTIVE_FILE) +
2286	node_page_state(pgdat, NR_INACTIVE_FILE);
2287
2288	for (z = 0; z < MAX_NR_ZONES; z++) {
2289	struct zone *zone = &pgdat->node_zones[z];
2290	if (!managed_zone(zone))
2291	continue;
2292
2293	total_high_wmark += high_wmark_pages(zone);
2294	}
2295
2296	if (unlikely(pgdatfile + pgdatfree <= total_high_wmark)) {
2297	scan_balance = SCAN_ANON;
2298	goto out;
2299	}
2300	}
2301
2302	/*
2303	* If there is enough inactive page cache, i.e. if the size of the
2304	* inactive list is greater than that of the active list *and* the
2305	* inactive list actually has some pages to scan on this priority, we
2306	* do not reclaim anything from the anonymous working set right now.
2307	* Without the second condition we could end up never scanning an
2308	* lruvec even if it has plenty of old anonymous pages unless the
2309	* system is under heavy pressure.
2310	*/
2311	if (!inactive_list_is_low(lruvec, true, sc) &&
2312	lruvec_lru_size(lruvec, LRU_INACTIVE_FILE, sc->reclaim_idx) >> sc->priority) {
2313	scan_balance = SCAN_FILE;
2314	goto out;
2315	}
2316
2317	scan_balance = SCAN_FRACT;
2318
2319	/*
2320	* With swappiness at 100, anonymous and file have the same priority.
2321	* This scanning priority is essentially the inverse of IO cost.
2322	*/
2323	anon_prio = swappiness;
2324	#ifdef CONFIG_AMLOGIC_MEMORY_EXTEND
2325	if (get_nr_swap_pages() * 3 < total_swap_pages)
2326	anon_prio >>= 1;
2327	#endif /* CONFIG_AMLOGIC_MEMORY_EXTEND */
2328	file_prio = 200 - anon_prio;
2329
2330	/*
2331	* OK, so we have swap space and a fair amount of page cache
2332	* pages. We use the recently rotated / recently scanned
2333	* ratios to determine how valuable each cache is.
2334	*
2335	* Because workloads change over time (and to avoid overflow)
2336	* we keep these statistics as a floating average, which ends
2337	* up weighing recent references more than old ones.
2338	*
2339	* anon in [0], file in [1]
2340	*/
2341
2342	anon = lruvec_lru_size(lruvec, LRU_ACTIVE_ANON, MAX_NR_ZONES) +
2343	lruvec_lru_size(lruvec, LRU_INACTIVE_ANON, MAX_NR_ZONES);
2344	file = lruvec_lru_size(lruvec, LRU_ACTIVE_FILE, MAX_NR_ZONES) +
2345	lruvec_lru_size(lruvec, LRU_INACTIVE_FILE, MAX_NR_ZONES);
2346
2347	spin_lock_irq(&pgdat->lru_lock);
2348	if (unlikely(reclaim_stat->recent_scanned[0] > anon / 4)) {
2349	reclaim_stat->recent_scanned[0] /= 2;
2350	reclaim_stat->recent_rotated[0] /= 2;
2351	}
2352
2353	if (unlikely(reclaim_stat->recent_scanned[1] > file / 4)) {
2354	reclaim_stat->recent_scanned[1] /= 2;
2355	reclaim_stat->recent_rotated[1] /= 2;
2356	}
2357
2358	/*
2359	* The amount of pressure on anon vs file pages is inversely
2360	* proportional to the fraction of recently scanned pages on
2361	* each list that were recently referenced and in active use.
2362	*/
2363	ap = anon_prio * (reclaim_stat->recent_scanned[0] + 1);
2364	ap /= reclaim_stat->recent_rotated[0] + 1;
2365
2366	fp = file_prio * (reclaim_stat->recent_scanned[1] + 1);
2367	fp /= reclaim_stat->recent_rotated[1] + 1;
2368	spin_unlock_irq(&pgdat->lru_lock);
2369
2370	fraction[0] = ap;
2371	fraction[1] = fp;
2372	denominator = ap + fp + 1;
2373	out:
2374	some_scanned = false;
2375	/* Only use force_scan on second pass. */
2376	for (pass = 0; !some_scanned && pass < 2; pass++) {
2377	*lru_pages = 0;
2378	for_each_evictable_lru(lru) {
2379	int file = is_file_lru(lru);
2380	unsigned long size;
2381	unsigned long scan;
2382
2383	size = lruvec_lru_size(lruvec, lru, sc->reclaim_idx);
2384	scan = size >> sc->priority;
2385
2386	if (!scan && pass && force_scan)
2387	scan = min(size, SWAP_CLUSTER_MAX);
2388
2389	switch (scan_balance) {
2390	case SCAN_EQUAL:
2391	/* Scan lists relative to size */
2392	break;
2393	case SCAN_FRACT:
2394	/*
2395	* Scan types proportional to swappiness and
2396	* their relative recent reclaim efficiency.
2397	*/
2398	scan = div64_u64(scan * fraction[file],
2399	denominator);
2400	break;
2401	case SCAN_FILE:
2402	case SCAN_ANON:
2403	/* Scan one type exclusively */
2404	if ((scan_balance == SCAN_FILE) != file) {
2405	size = 0;
2406	scan = 0;
2407	}
2408	break;
2409	default:
2410	/* Look ma, no brain */
2411	BUG();
2412	}
2413
2414	*lru_pages += size;
2415	nr[lru] = scan;
2416
2417	/*
2418	* Skip the second pass and don't force_scan,
2419	* if we found something to scan.
2420	*/
2421	some_scanned |= !!scan;
2422	}
2423	}
2424	}
2425
2426	/*
2427	* This is a basic per-node page freer. Used by both kswapd and direct reclaim.
2428	*/
2429	static void shrink_node_memcg(struct pglist_data *pgdat, struct mem_cgroup *memcg,
2430	struct scan_control *sc, unsigned long *lru_pages)
2431	{
2432	struct lruvec *lruvec = mem_cgroup_lruvec(pgdat, memcg);
2433	unsigned long nr[NR_LRU_LISTS];
2434	unsigned long targets[NR_LRU_LISTS];
2435	unsigned long nr_to_scan;
2436	enum lru_list lru;
2437	unsigned long nr_reclaimed = 0;
2438	unsigned long nr_to_reclaim = sc->nr_to_reclaim;
2439	struct blk_plug plug;
2440	bool scan_adjusted;
2441
2442	get_scan_count(lruvec, memcg, sc, nr, lru_pages);
2443
2444	/* Record the original scan target for proportional adjustments later */
2445	memcpy(targets, nr, sizeof(nr));
2446
2447	/*
2448	* Global reclaiming within direct reclaim at DEF_PRIORITY is a normal
2449	* event that can occur when there is little memory pressure e.g.
2450	* multiple streaming readers/writers. Hence, we do not abort scanning
2451	* when the requested number of pages are reclaimed when scanning at
2452	* DEF_PRIORITY on the assumption that the fact we are direct
2453	* reclaiming implies that kswapd is not keeping up and it is best to
2454	* do a batch of work at once. For memcg reclaim one check is made to
2455	* abort proportional reclaim if either the file or anon lru has already
2456	* dropped to zero at the first pass.
2457	*/
2458	scan_adjusted = (global_reclaim(sc) && !current_is_kswapd() &&
2459	sc->priority == DEF_PRIORITY);
2460
2461	blk_start_plug(&plug);
2462	while (nr[LRU_INACTIVE_ANON] || nr[LRU_ACTIVE_FILE] ||
2463	nr[LRU_INACTIVE_FILE]) {
2464	unsigned long nr_anon, nr_file, percentage;
2465	unsigned long nr_scanned;
2466
2467	for_each_evictable_lru(lru) {
2468	if (nr[lru]) {
2469	nr_to_scan = min(nr[lru], SWAP_CLUSTER_MAX);
2470	nr[lru] -= nr_to_scan;
2471
2472	nr_reclaimed += shrink_list(lru, nr_to_scan,
2473	lruvec, sc);
2474	}
2475	}
2476
2477	cond_resched();
2478
2479	if (nr_reclaimed < nr_to_reclaim || scan_adjusted)
2480	continue;
2481
2482	/*
2483	* For kswapd and memcg, reclaim at least the number of pages
2484	* requested. Ensure that the anon and file LRUs are scanned
2485	* proportionally what was requested by get_scan_count(). We
2486	* stop reclaiming one LRU and reduce the amount scanning
2487	* proportional to the original scan target.
2488	*/
2489	nr_file = nr[LRU_INACTIVE_FILE] + nr[LRU_ACTIVE_FILE];
2490	nr_anon = nr[LRU_INACTIVE_ANON] + nr[LRU_ACTIVE_ANON];
2491
2492	/*
2493	* It's just vindictive to attack the larger once the smaller
2494	* has gone to zero. And given the way we stop scanning the
2495	* smaller below, this makes sure that we only make one nudge
2496	* towards proportionality once we've got nr_to_reclaim.
2497	*/
2498	if (!nr_file || !nr_anon)
2499	break;
2500
2501	if (nr_file > nr_anon) {
2502	unsigned long scan_target = targets[LRU_INACTIVE_ANON] +
2503	targets[LRU_ACTIVE_ANON] + 1;
2504	lru = LRU_BASE;
2505	percentage = nr_anon * 100 / scan_target;
2506	} else {
2507	unsigned long scan_target = targets[LRU_INACTIVE_FILE] +
2508	targets[LRU_ACTIVE_FILE] + 1;
2509	lru = LRU_FILE;
2510	percentage = nr_file * 100 / scan_target;
2511	}
2512
2513	/* Stop scanning the smaller of the LRU */
2514	nr[lru] = 0;
2515	nr[lru + LRU_ACTIVE] = 0;
2516
2517	/*
2518	* Recalculate the other LRU scan count based on its original
2519	* scan target and the percentage scanning already complete
2520	*/
2521	lru = (lru == LRU_FILE) ? LRU_BASE : LRU_FILE;
2522	nr_scanned = targets[lru] - nr[lru];
2523	nr[lru] = targets[lru] * (100 - percentage) / 100;
2524	nr[lru] -= min(nr[lru], nr_scanned);
2525
2526	lru += LRU_ACTIVE;
2527	nr_scanned = targets[lru] - nr[lru];
2528	nr[lru] = targets[lru] * (100 - percentage) / 100;
2529	nr[lru] -= min(nr[lru], nr_scanned);
2530
2531	scan_adjusted = true;
2532	}
2533	blk_finish_plug(&plug);
2534	sc->nr_reclaimed += nr_reclaimed;
2535
2536	/*
2537	* Even if we did not try to evict anon pages at all, we want to
2538	* rebalance the anon lru active/inactive ratio.
2539	*/
2540	if (inactive_list_is_low(lruvec, false, sc))
2541	shrink_active_list(SWAP_CLUSTER_MAX, lruvec,
2542	sc, LRU_ACTIVE_ANON);
2543	}
2544
2545	/* Use reclaim/compaction for costly allocs or under memory pressure */
2546	static bool in_reclaim_compaction(struct scan_control *sc)
2547	{
2548	if (IS_ENABLED(CONFIG_COMPACTION) && sc->order &&
2549	(sc->order > PAGE_ALLOC_COSTLY_ORDER ||
2550	sc->priority < DEF_PRIORITY - 2))
2551	return true;
2552
2553	return false;
2554	}
2555
2556	/*
2557	* Reclaim/compaction is used for high-order allocation requests. It reclaims
2558	* order-0 pages before compacting the zone. should_continue_reclaim() returns
2559	* true if more pages should be reclaimed such that when the page allocator
2560	* calls try_to_compact_zone() that it will have enough free pages to succeed.
2561	* It will give up earlier than that if there is difficulty reclaiming pages.
2562	*/
2563	static inline bool should_continue_reclaim(struct pglist_data *pgdat,
2564	unsigned long nr_reclaimed,
2565	unsigned long nr_scanned,
2566	struct scan_control *sc)
2567	{
2568	unsigned long pages_for_compaction;
2569	unsigned long inactive_lru_pages;
2570	int z;
2571
2572	/* If not in reclaim/compaction mode, stop */
2573	if (!in_reclaim_compaction(sc))
2574	return false;
2575
2576	/* Consider stopping depending on scan and reclaim activity */
2577	if (sc->gfp_mask & __GFP_REPEAT) {
2578	/*
2579	* For __GFP_REPEAT allocations, stop reclaiming if the
2580	* full LRU list has been scanned and we are still failing
2581	* to reclaim pages. This full LRU scan is potentially
2582	* expensive but a __GFP_REPEAT caller really wants to succeed
2583	*/
2584	if (!nr_reclaimed && !nr_scanned)
2585	return false;
2586	} else {
2587	/*
2588	* For non-__GFP_REPEAT allocations which can presumably
2589	* fail without consequence, stop if we failed to reclaim
2590	* any pages from the last SWAP_CLUSTER_MAX number of
2591	* pages that were scanned. This will return to the
2592	* caller faster at the risk reclaim/compaction and
2593	* the resulting allocation attempt fails
2594	*/
2595	if (!nr_reclaimed)
2596	return false;
2597	}
2598
2599	/*
2600	* If we have not reclaimed enough pages for compaction and the
2601	* inactive lists are large enough, continue reclaiming
2602	*/
2603	pages_for_compaction = compact_gap(sc->order);
2604	inactive_lru_pages = node_page_state(pgdat, NR_INACTIVE_FILE);
2605	if (get_nr_swap_pages() > 0)
2606	inactive_lru_pages += node_page_state(pgdat, NR_INACTIVE_ANON);
2607	if (sc->nr_reclaimed < pages_for_compaction &&
2608	inactive_lru_pages > pages_for_compaction)
2609	return true;
2610
2611	/* If compaction would go ahead or the allocation would succeed, stop */
2612	for (z = 0; z <= sc->reclaim_idx; z++) {
2613	struct zone *zone = &pgdat->node_zones[z];
2614	if (!managed_zone(zone))
2615	continue;
2616
2617	switch (compaction_suitable(zone, sc->order, 0, sc->reclaim_idx)) {
2618	case COMPACT_SUCCESS:
2619	case COMPACT_CONTINUE:
2620	return false;
2621	default:
2622	/* check next zone */
2623	;
2624	}
2625	}
2626	return true;
2627	}
2628
2629	static bool shrink_node(pg_data_t *pgdat, struct scan_control *sc)
2630	{
2631	struct reclaim_state *reclaim_state = current->reclaim_state;
2632	unsigned long nr_reclaimed, nr_scanned;
2633	bool reclaimable = false;
2634
2635	do {
2636	struct mem_cgroup *root = sc->target_mem_cgroup;
2637	struct mem_cgroup_reclaim_cookie reclaim = {
2638	.pgdat = pgdat,
2639	.priority = sc->priority,
2640	};
2641	unsigned long node_lru_pages = 0;
2642	struct mem_cgroup *memcg;
2643
2644	nr_reclaimed = sc->nr_reclaimed;
2645	nr_scanned = sc->nr_scanned;
2646
2647	memcg = mem_cgroup_iter(root, NULL, &reclaim);
2648	do {
2649	unsigned long lru_pages;
2650	unsigned long reclaimed;
2651	unsigned long scanned;
2652
2653	if (mem_cgroup_low(root, memcg)) {
2654	if (!sc->may_thrash)
2655	continue;
2656	mem_cgroup_events(memcg, MEMCG_LOW, 1);
2657	}
2658
2659	reclaimed = sc->nr_reclaimed;
2660	scanned = sc->nr_scanned;
2661
2662	shrink_node_memcg(pgdat, memcg, sc, &lru_pages);
2663	node_lru_pages += lru_pages;
2664
2665	if (memcg)
2666	shrink_slab(sc->gfp_mask, pgdat->node_id,
2667	memcg, sc->nr_scanned - scanned,
2668	lru_pages);
2669
2670	/* Record the group's reclaim efficiency */
2671	vmpressure(sc->gfp_mask, memcg, false,
2672	sc->nr_scanned - scanned,
2673	sc->nr_reclaimed - reclaimed);
2674
2675	/*
2676	* Direct reclaim and kswapd have to scan all memory
2677	* cgroups to fulfill the overall scan target for the
2678	* node.
2679	*
2680	* Limit reclaim, on the other hand, only cares about
2681	* nr_to_reclaim pages to be reclaimed and it will
2682	* retry with decreasing priority if one round over the
2683	* whole hierarchy is not sufficient.
2684	*/
2685	if (!global_reclaim(sc) &&
2686	sc->nr_reclaimed >= sc->nr_to_reclaim) {
2687	mem_cgroup_iter_break(root, memcg);
2688	break;
2689	}
2690	} while ((memcg = mem_cgroup_iter(root, memcg, &reclaim)));
2691
2692	/*
2693	* Shrink the slab caches in the same proportion that
2694	* the eligible LRU pages were scanned.
2695	*/
2696	if (global_reclaim(sc))
2697	shrink_slab(sc->gfp_mask, pgdat->node_id, NULL,
2698	sc->nr_scanned - nr_scanned,
2699	node_lru_pages);
2700
2701	if (reclaim_state) {
2702	sc->nr_reclaimed += reclaim_state->reclaimed_slab;
2703	reclaim_state->reclaimed_slab = 0;
2704	}
2705
2706	/* Record the subtree's reclaim efficiency */
2707	vmpressure(sc->gfp_mask, sc->target_mem_cgroup, true,
2708	sc->nr_scanned - nr_scanned,
2709	sc->nr_reclaimed - nr_reclaimed);
2710
2711	if (sc->nr_reclaimed - nr_reclaimed)
2712	reclaimable = true;
2713
2714	} while (should_continue_reclaim(pgdat, sc->nr_reclaimed - nr_reclaimed,
2715	sc->nr_scanned - nr_scanned, sc));
2716
2717	/*
2718	* Kswapd gives up on balancing particular nodes after too
2719	* many failures to reclaim anything from them and goes to
2720	* sleep. On reclaim progress, reset the failure counter. A
2721	* successful direct reclaim run will revive a dormant kswapd.
2722	*/
2723	if (reclaimable)
2724	pgdat->kswapd_failures = 0;
2725
2726	return reclaimable;
2727	}
2728
2729	/*
2730	* Returns true if compaction should go ahead for a costly-order request, or
2731	* the allocation would already succeed without compaction. Return false if we
2732	* should reclaim first.
2733	*/
2734	static inline bool compaction_ready(struct zone *zone, struct scan_control *sc)
2735	{
2736	unsigned long watermark;
2737	enum compact_result suitable;
2738
2739	suitable = compaction_suitable(zone, sc->order, 0, sc->reclaim_idx);
2740	if (suitable == COMPACT_SUCCESS)
2741	/* Allocation should succeed already. Don't reclaim. */
2742	return true;
2743	if (suitable == COMPACT_SKIPPED)
2744	/* Compaction cannot yet proceed. Do reclaim. */
2745	return false;
2746
2747	/*
2748	* Compaction is already possible, but it takes time to run and there
2749	* are potentially other callers using the pages just freed. So proceed
2750	* with reclaim to make a buffer of free pages available to give
2751	* compaction a reasonable chance of completing and allocating the page.
2752	* Note that we won't actually reclaim the whole buffer in one attempt
2753	* as the target watermark in should_continue_reclaim() is lower. But if
2754	* we are already above the high+gap watermark, don't reclaim at all.
2755	*/
2756	watermark = high_wmark_pages(zone) + compact_gap(sc->order);
2757
2758	return zone_watermark_ok_safe(zone, 0, watermark, sc->reclaim_idx);
2759	}
2760
2761	/*
2762	* This is the direct reclaim path, for page-allocating processes. We only
2763	* try to reclaim pages from zones which will satisfy the caller's allocation
2764	* request.
2765	*
2766	* If a zone is deemed to be full of pinned pages then just give it a light
2767	* scan then give up on it.
2768	*/
2769	static void shrink_zones(struct zonelist *zonelist, struct scan_control *sc)
2770	{
2771	struct zoneref *z;
2772	struct zone *zone;
2773	unsigned long nr_soft_reclaimed;
2774	unsigned long nr_soft_scanned;
2775	gfp_t orig_mask;
2776	pg_data_t *last_pgdat = NULL;
2777
2778	/*
2779	* If the number of buffer_heads in the machine exceeds the maximum
2780	* allowed level, force direct reclaim to scan the highmem zone as
2781	* highmem pages could be pinning lowmem pages storing buffer_heads
2782	*/
2783	orig_mask = sc->gfp_mask;
2784	if (buffer_heads_over_limit) {
2785	sc->gfp_mask |= __GFP_HIGHMEM;
2786	sc->reclaim_idx = gfp_zone(sc->gfp_mask);
2787	}
2788
2789	for_each_zone_zonelist_nodemask(zone, z, zonelist,
2790	sc->reclaim_idx, sc->nodemask) {
2791	/*
2792	* Take care memory controller reclaiming has small influence
2793	* to global LRU.
2794	*/
2795	if (global_reclaim(sc)) {
2796	if (!cpuset_zone_allowed(zone,
2797	GFP_KERNEL | __GFP_HARDWALL))
2798	continue;
2799
2800	/*
2801	* If we already have plenty of memory free for
2802	* compaction in this zone, don't free any more.
2803	* Even though compaction is invoked for any
2804	* non-zero order, only frequent costly order
2805	* reclamation is disruptive enough to become a
2806	* noticeable problem, like transparent huge
2807	* page allocations.
2808	*/
2809	if (IS_ENABLED(CONFIG_COMPACTION) &&
2810	sc->order > PAGE_ALLOC_COSTLY_ORDER &&
2811	compaction_ready(zone, sc)) {
2812	sc->compaction_ready = true;
2813	continue;
2814	}
2815
2816	/*
2817	* Shrink each node in the zonelist once. If the
2818	* zonelist is ordered by zone (not the default) then a
2819	* node may be shrunk multiple times but in that case
2820	* the user prefers lower zones being preserved.
2821	*/
2822	if (zone->zone_pgdat == last_pgdat)
2823	continue;
2824
2825	/*
2826	* This steals pages from memory cgroups over softlimit
2827	* and returns the number of reclaimed pages and
2828	* scanned pages. This works for global memory pressure
2829	* and balancing, not for a memcg's limit.
2830	*/
2831	nr_soft_scanned = 0;
2832	nr_soft_reclaimed = mem_cgroup_soft_limit_reclaim(zone->zone_pgdat,
2833	sc->order, sc->gfp_mask,
2834	&nr_soft_scanned);
2835	sc->nr_reclaimed += nr_soft_reclaimed;
2836	sc->nr_scanned += nr_soft_scanned;
2837	/* need some check for avoid more shrink_zone() */
2838	}
2839
2840	/* See comment about same check for global reclaim above */
2841	if (zone->zone_pgdat == last_pgdat)
2842	continue;
2843	last_pgdat = zone->zone_pgdat;
2844	shrink_node(zone->zone_pgdat, sc);
2845	}
2846
2847	/*
2848	* Restore to original mask to avoid the impact on the caller if we
2849	* promoted it to __GFP_HIGHMEM.
2850	*/
2851	sc->gfp_mask = orig_mask;
2852	}
2853
2854	/*
2855	* This is the main entry point to direct page reclaim.
2856	*
2857	* If a full scan of the inactive list fails to free enough memory then we
2858	* are "out of memory" and something needs to be killed.
2859	*
2860	* If the caller is !__GFP_FS then the probability of a failure is reasonably
2861	* high - the zone may be full of dirty or under-writeback pages, which this
2862	* caller can't do much about. We kick the writeback threads and take explicit
2863	* naps in the hope that some of these pages can be written. But if the
2864	* allocating task holds filesystem locks which prevent writeout this might not
2865	* work, and the allocation attempt will fail.
2866	*
2867	* returns: 0, if no pages reclaimed
2868	* else, the number of pages reclaimed
2869	*/
2870	static unsigned long do_try_to_free_pages(struct zonelist *zonelist,
2871	struct scan_control *sc)
2872	{
2873	int initial_priority = sc->priority;
2874	unsigned long total_scanned = 0;
2875	unsigned long writeback_threshold;
2876	retry:
2877	delayacct_freepages_start();
2878
2879	if (global_reclaim(sc))
2880	__count_zid_vm_events(ALLOCSTALL, sc->reclaim_idx, 1);
2881
2882	do {
2883	vmpressure_prio(sc->gfp_mask, sc->target_mem_cgroup,
2884	sc->priority);
2885	sc->nr_scanned = 0;
2886	shrink_zones(zonelist, sc);
2887
2888	total_scanned += sc->nr_scanned;
2889	if (sc->nr_reclaimed >= sc->nr_to_reclaim)
2890	break;
2891
2892	if (sc->compaction_ready)
2893	break;
2894
2895	/*
2896	* If we're getting trouble reclaiming, start doing
2897	* writepage even in laptop mode.
2898	*/
2899	if (sc->priority < DEF_PRIORITY - 2)
2900	sc->may_writepage = 1;
2901
2902	/*
2903	* Try to write back as many pages as we just scanned. This
2904	* tends to cause slow streaming writers to write data to the
2905	* disk smoothly, at the dirtying rate, which is nice. But
2906	* that's undesirable in laptop mode, where we *want* lumpy
2907	* writeout. So in laptop mode, write out the whole world.
2908	*/
2909	writeback_threshold = sc->nr_to_reclaim + sc->nr_to_reclaim / 2;
2910	if (total_scanned > writeback_threshold) {
2911	wakeup_flusher_threads(laptop_mode ? 0 : total_scanned,
2912	WB_REASON_TRY_TO_FREE_PAGES);
2913	sc->may_writepage = 1;
2914	}
2915	} while (--sc->priority >= 0);
2916
2917	delayacct_freepages_end();
2918
2919	if (sc->nr_reclaimed)
2920	return sc->nr_reclaimed;
2921
2922	/* Aborted reclaim to try compaction? don't OOM, then */
2923	if (sc->compaction_ready)
2924	return 1;
2925
2926	/* Untapped cgroup reserves? Don't OOM, retry. */
2927	if (!sc->may_thrash) {
2928	sc->priority = initial_priority;
2929	sc->may_thrash = 1;
2930	goto retry;
2931	}
2932
2933	return 0;
2934	}
2935
2936	static bool allow_direct_reclaim(pg_data_t *pgdat)
2937	{
2938	struct zone *zone;
2939	unsigned long pfmemalloc_reserve = 0;
2940	unsigned long free_pages = 0;
2941	int i;
2942	bool wmark_ok;
2943
2944	if (pgdat->kswapd_failures >= MAX_RECLAIM_RETRIES)
2945	return true;
2946
2947	for (i = 0; i <= ZONE_NORMAL; i++) {
2948	zone = &pgdat->node_zones[i];
2949	if (!managed_zone(zone))
2950	continue;
2951
2952	if (!zone_reclaimable_pages(zone))
2953	continue;
2954
2955	pfmemalloc_reserve += min_wmark_pages(zone);
2956	free_pages += zone_page_state(zone, NR_FREE_PAGES);
2957	}
2958
2959	/* If there are no reserves (unexpected config) then do not throttle */
2960	if (!pfmemalloc_reserve)
2961	return true;
2962
2963	wmark_ok = free_pages > pfmemalloc_reserve / 2;
2964
2965	/* kswapd must be awake if processes are being throttled */
2966	if (!wmark_ok && waitqueue_active(&pgdat->kswapd_wait)) {
2967	pgdat->kswapd_classzone_idx = min(pgdat->kswapd_classzone_idx,
2968	(enum zone_type)ZONE_NORMAL);
2969	wake_up_interruptible(&pgdat->kswapd_wait);
2970	}
2971
2972	return wmark_ok;
2973	}
2974
2975	/*
2976	* Throttle direct reclaimers if backing storage is backed by the network
2977	* and the PFMEMALLOC reserve for the preferred node is getting dangerously
2978	* depleted. kswapd will continue to make progress and wake the processes
2979	* when the low watermark is reached.
2980	*
2981	* Returns true if a fatal signal was delivered during throttling. If this
2982	* happens, the page allocator should not consider triggering the OOM killer.
2983	*/
2984	static bool throttle_direct_reclaim(gfp_t gfp_mask, struct zonelist *zonelist,
2985	nodemask_t *nodemask)
2986	{
2987	struct zoneref *z;
2988	struct zone *zone;
2989	pg_data_t *pgdat = NULL;
2990
2991	/*
2992	* Kernel threads should not be throttled as they may be indirectly
2993	* responsible for cleaning pages necessary for reclaim to make forward
2994	* progress. kjournald for example may enter direct reclaim while
2995	* committing a transaction where throttling it could forcing other
2996	* processes to block on log_wait_commit().
2997	*/
2998	if (current->flags & PF_KTHREAD)
2999	goto out;
3000
3001	/*
3002	* If a fatal signal is pending, this process should not throttle.
3003	* It should return quickly so it can exit and free its memory
3004	*/
3005	if (fatal_signal_pending(current))
3006	goto out;
3007
3008	/*
3009	* Check if the pfmemalloc reserves are ok by finding the first node
3010	* with a usable ZONE_NORMAL or lower zone. The expectation is that
3011	* GFP_KERNEL will be required for allocating network buffers when
3012	* swapping over the network so ZONE_HIGHMEM is unusable.
3013	*
3014	* Throttling is based on the first usable node and throttled processes
3015	* wait on a queue until kswapd makes progress and wakes them. There
3016	* is an affinity then between processes waking up and where reclaim
3017	* progress has been made assuming the process wakes on the same node.
3018	* More importantly, processes running on remote nodes will not compete
3019	* for remote pfmemalloc reserves and processes on different nodes
3020	* should make reasonable progress.
3021	*/
3022	for_each_zone_zonelist_nodemask(zone, z, zonelist,
3023	gfp_zone(gfp_mask), nodemask) {
3024	if (zone_idx(zone) > ZONE_NORMAL)
3025	continue;
3026
3027	/* Throttle based on the first usable node */
3028	pgdat = zone->zone_pgdat;
3029	if (allow_direct_reclaim(pgdat))
3030	goto out;
3031	break;
3032	}
3033
3034	/* If no zone was usable by the allocation flags then do not throttle */
3035	if (!pgdat)
3036	goto out;
3037
3038	/* Account for the throttling */
3039	count_vm_event(PGSCAN_DIRECT_THROTTLE);
3040
3041	/*
3042	* If the caller cannot enter the filesystem, it's possible that it
3043	* is due to the caller holding an FS lock or performing a journal
3044	* transaction in the case of a filesystem like ext[3|4]. In this case,
3045	* it is not safe to block on pfmemalloc_wait as kswapd could be
3046	* blocked waiting on the same lock. Instead, throttle for up to a
3047	* second before continuing.
3048	*/
3049	if (!(gfp_mask & __GFP_FS)) {
3050	wait_event_interruptible_timeout(pgdat->pfmemalloc_wait,
3051	allow_direct_reclaim(pgdat), HZ);
3052
3053	goto check_pending;
3054	}
3055
3056	/* Throttle until kswapd wakes the process */
3057	wait_event_killable(zone->zone_pgdat->pfmemalloc_wait,
3058	allow_direct_reclaim(pgdat));
3059
3060	check_pending:
3061	if (fatal_signal_pending(current))
3062	return true;
3063
3064	out:
3065	return false;
3066	}
3067
3068	unsigned long try_to_free_pages(struct zonelist *zonelist, int order,
3069	gfp_t gfp_mask, nodemask_t *nodemask)
3070	{
3071	unsigned long nr_reclaimed;
3072	struct scan_control sc = {
3073	.nr_to_reclaim = SWAP_CLUSTER_MAX,
3074	.gfp_mask = memalloc_noio_flags(gfp_mask),
3075	.reclaim_idx = gfp_zone(gfp_mask),
3076	.order = order,
3077	.nodemask = nodemask,
3078	.priority = DEF_PRIORITY,
3079	.may_writepage = !laptop_mode,
3080	.may_unmap = 1,
3081	.may_swap = 1,
3082	};
3083
3084	/*
3085	* Do not enter reclaim if fatal signal was delivered while throttled.
3086	* 1 is returned so that the page allocator does not OOM kill at this
3087	* point.
3088	*/
3089	if (throttle_direct_reclaim(sc.gfp_mask, zonelist, nodemask))
3090	return 1;
3091
3092	trace_mm_vmscan_direct_reclaim_begin(order,
3093	sc.may_writepage,
3094	sc.gfp_mask,
3095	sc.reclaim_idx);
3096
3097	nr_reclaimed = do_try_to_free_pages(zonelist, &sc);
3098
3099	trace_mm_vmscan_direct_reclaim_end(nr_reclaimed);
3100
3101	return nr_reclaimed;
3102	}
3103
3104	#ifdef CONFIG_MEMCG
3105
3106	unsigned long mem_cgroup_shrink_node(struct mem_cgroup *memcg,
3107	gfp_t gfp_mask, bool noswap,
3108	pg_data_t *pgdat,
3109	unsigned long *nr_scanned)
3110	{
3111	struct scan_control sc = {
3112	.nr_to_reclaim = SWAP_CLUSTER_MAX,
3113	.target_mem_cgroup = memcg,
3114	.may_writepage = !laptop_mode,
3115	.may_unmap = 1,
3116	.reclaim_idx = MAX_NR_ZONES - 1,
3117	.may_swap = !noswap,
3118	};
3119	unsigned long lru_pages;
3120
3121	sc.gfp_mask = (gfp_mask & GFP_RECLAIM_MASK) |
3122	(GFP_HIGHUSER_MOVABLE & ~GFP_RECLAIM_MASK);
3123
3124	trace_mm_vmscan_memcg_softlimit_reclaim_begin(sc.order,
3125	sc.may_writepage,
3126	sc.gfp_mask,
3127	sc.reclaim_idx);
3128
3129	/*
3130	* NOTE: Although we can get the priority field, using it
3131	* here is not a good idea, since it limits the pages we can scan.
3132	* if we don't reclaim here, the shrink_node from balance_pgdat
3133	* will pick up pages from other mem cgroup's as well. We hack
3134	* the priority and make it zero.
3135	*/
3136	shrink_node_memcg(pgdat, memcg, &sc, &lru_pages);
3137
3138	trace_mm_vmscan_memcg_softlimit_reclaim_end(sc.nr_reclaimed);
3139
3140	*nr_scanned = sc.nr_scanned;
3141	return sc.nr_reclaimed;
3142	}
3143
3144	unsigned long try_to_free_mem_cgroup_pages(struct mem_cgroup *memcg,
3145	unsigned long nr_pages,
3146	gfp_t gfp_mask,
3147	bool may_swap)
3148	{
3149	struct zonelist *zonelist;
3150	unsigned long nr_reclaimed;
3151	unsigned long pflags;
3152	int nid;
3153	struct scan_control sc = {
3154	.nr_to_reclaim = max(nr_pages, SWAP_CLUSTER_MAX),
3155	.gfp_mask = (gfp_mask & GFP_RECLAIM_MASK) |
3156	(GFP_HIGHUSER_MOVABLE & ~GFP_RECLAIM_MASK),
3157	.reclaim_idx = MAX_NR_ZONES - 1,
3158	.target_mem_cgroup = memcg,
3159	.priority = DEF_PRIORITY,
3160	.may_writepage = !laptop_mode,
3161	.may_unmap = 1,
3162	.may_swap = may_swap,
3163	};
3164
3165	/*
3166	* Unlike direct reclaim via alloc_pages(), memcg's reclaim doesn't
3167	* take care of from where we get pages. So the node where we start the
3168	* scan does not need to be the current node.
3169	*/
3170	nid = mem_cgroup_select_victim_node(memcg);
3171
3172	zonelist = &NODE_DATA(nid)->node_zonelists[ZONELIST_FALLBACK];
3173
3174	trace_mm_vmscan_memcg_reclaim_begin(0,
3175	sc.may_writepage,
3176	sc.gfp_mask,
3177	sc.reclaim_idx);
3178
3179	psi_memstall_enter(&pflags);
3180	current->flags |= PF_MEMALLOC;
3181
3182	nr_reclaimed = do_try_to_free_pages(zonelist, &sc);
3183
3184	current->flags &= ~PF_MEMALLOC;
3185	psi_memstall_leave(&pflags);
3186
3187	trace_mm_vmscan_memcg_reclaim_end(nr_reclaimed);
3188
3189	return nr_reclaimed;
3190	}
3191	#endif
3192
3193	static void age_active_anon(struct pglist_data *pgdat,
3194	struct scan_control *sc)
3195	{
3196	struct mem_cgroup *memcg;
3197
3198	if (!total_swap_pages)
3199	return;
3200
3201	memcg = mem_cgroup_iter(NULL, NULL, NULL);
3202	do {
3203	struct lruvec *lruvec = mem_cgroup_lruvec(pgdat, memcg);
3204
3205	if (inactive_list_is_low(lruvec, false, sc))
3206	shrink_active_list(SWAP_CLUSTER_MAX, lruvec,
3207	sc, LRU_ACTIVE_ANON);
3208
3209	memcg = mem_cgroup_iter(NULL, memcg, NULL);
3210	} while (memcg);
3211	}
3212
3213	static bool zone_balanced(struct zone *zone, int order, int classzone_idx)
3214	{
3215	unsigned long mark = high_wmark_pages(zone);
3216
3217	if (!zone_watermark_ok_safe(zone, order, mark, classzone_idx))
3218	return false;
3219
3220	/*
3221	* If any eligible zone is balanced then the node is not considered
3222	* to be congested or dirty
3223	*/
3224	clear_bit(PGDAT_CONGESTED, &zone->zone_pgdat->flags);
3225	clear_bit(PGDAT_DIRTY, &zone->zone_pgdat->flags);
3226	clear_bit(PGDAT_WRITEBACK, &zone->zone_pgdat->flags);
3227
3228	return true;
3229	}
3230
3231	/*
3232	* Prepare kswapd for sleeping. This verifies that there are no processes
3233	* waiting in throttle_direct_reclaim() and that watermarks have been met.
3234	*
3235	* Returns true if kswapd is ready to sleep
3236	*/
3237	static bool prepare_kswapd_sleep(pg_data_t *pgdat, int order, int classzone_idx)
3238	{
3239	int i;
3240
3241	/*
3242	* The throttled processes are normally woken up in balance_pgdat() as
3243	* soon as allow_direct_reclaim() is true. But there is a potential
3244	* race between when kswapd checks the watermarks and a process gets
3245	* throttled. There is also a potential race if processes get
3246	* throttled, kswapd wakes, a large process exits thereby balancing the
3247	* zones, which causes kswapd to exit balance_pgdat() before reaching
3248	* the wake up checks. If kswapd is going to sleep, no process should
3249	* be sleeping on pfmemalloc_wait, so wake them now if necessary. If
3250	* the wake up is premature, processes will wake kswapd and get
3251	* throttled again. The difference from wake ups in balance_pgdat() is
3252	* that here we are under prepare_to_wait().
3253	*/
3254	if (waitqueue_active(&pgdat->pfmemalloc_wait))
3255	wake_up_all(&pgdat->pfmemalloc_wait);
3256
3257	/* Hopeless node, leave it to direct reclaim */
3258	if (pgdat->kswapd_failures >= MAX_RECLAIM_RETRIES)
3259	return true;
3260
3261	for (i = 0; i <= classzone_idx; i++) {
3262	struct zone *zone = pgdat->node_zones + i;
3263
3264	if (!managed_zone(zone))
3265	continue;
3266
3267	if (!zone_balanced(zone, order, classzone_idx))
3268	return false;
3269	}
3270
3271	return true;
3272	}
3273
3274	/*
3275	* kswapd shrinks a node of pages that are at or below the highest usable
3276	* zone that is currently unbalanced.
3277	*
3278	* Returns true if kswapd scanned at least the requested number of pages to
3279	* reclaim or if the lack of progress was due to pages under writeback.
3280	* This is used to determine if the scanning priority needs to be raised.
3281	*/
3282	static bool kswapd_shrink_node(pg_data_t *pgdat,
3283	struct scan_control *sc)
3284	{
3285	struct zone *zone;
3286	int z;
3287
3288	/* Reclaim a number of pages proportional to the number of zones */
3289	sc->nr_to_reclaim = 0;
3290	for (z = 0; z <= sc->reclaim_idx; z++) {
3291	zone = pgdat->node_zones + z;
3292	if (!managed_zone(zone))
3293	continue;
3294
3295	sc->nr_to_reclaim += max(high_wmark_pages(zone), SWAP_CLUSTER_MAX);
3296	}
3297
3298	/*
3299	* Historically care was taken to put equal pressure on all zones but
3300	* now pressure is applied based on node LRU order.
3301	*/
3302	shrink_node(pgdat, sc);
3303
3304	/*
3305	* Fragmentation may mean that the system cannot be rebalanced for
3306	* high-order allocations. If twice the allocation size has been
3307	* reclaimed then recheck watermarks only at order-0 to prevent
3308	* excessive reclaim. Assume that a process requested a high-order
3309	* can direct reclaim/compact.
3310	*/
3311	if (sc->order && sc->nr_reclaimed >= compact_gap(sc->order))
3312	sc->order = 0;
3313
3314	return sc->nr_scanned >= sc->nr_to_reclaim;
3315	}
3316
3317	/*
3318	* For kswapd, balance_pgdat() will reclaim pages across a node from zones
3319	* that are eligible for use by the caller until at least one zone is
3320	* balanced.
3321	*
3322	* Returns the order kswapd finished reclaiming at.
3323	*
3324	* kswapd scans the zones in the highmem->normal->dma direction. It skips
3325	* zones which have free_pages > high_wmark_pages(zone), but once a zone is
3326	* found to have free_pages <= high_wmark_pages(zone), any page is that zone
3327	* or lower is eligible for reclaim until at least one usable zone is
3328	* balanced.
3329	*/
3330	static int balance_pgdat(pg_data_t *pgdat, int order, int classzone_idx)
3331	{
3332	int i;
3333	unsigned long nr_soft_reclaimed;
3334	unsigned long nr_soft_scanned;
3335	unsigned long pflags;
3336	struct zone *zone;
3337	struct scan_control sc = {
3338	.gfp_mask = GFP_KERNEL,
3339	.order = order,
3340	.priority = DEF_PRIORITY,
3341	.may_writepage = !laptop_mode,
3342	.may_unmap = 1,
3343	.may_swap = 1,
3344	};
3345
3346	psi_memstall_enter(&pflags);
3347	count_vm_event(PAGEOUTRUN);
3348
3349	do {
3350	unsigned long nr_reclaimed = sc.nr_reclaimed;
3351	bool raise_priority = true;
3352
3353	sc.reclaim_idx = classzone_idx;
3354
3355	/*
3356	* If the number of buffer_heads exceeds the maximum allowed
3357	* then consider reclaiming from all zones. This has a dual
3358	* purpose -- on 64-bit systems it is expected that
3359	* buffer_heads are stripped during active rotation. On 32-bit
3360	* systems, highmem pages can pin lowmem memory and shrinking
3361	* buffers can relieve lowmem pressure. Reclaim may still not
3362	* go ahead if all eligible zones for the original allocation
3363	* request are balanced to avoid excessive reclaim from kswapd.
3364	*/
3365	if (buffer_heads_over_limit) {
3366	for (i = MAX_NR_ZONES - 1; i >= 0; i--) {
3367	zone = pgdat->node_zones + i;
3368	if (!managed_zone(zone))
3369	continue;
3370
3371	sc.reclaim_idx = i;
3372	break;
3373	}
3374	}
3375
3376	/*
3377	* Only reclaim if there are no eligible zones. Check from
3378	* high to low zone as allocations prefer higher zones.
3379	* Scanning from low to high zone would allow congestion to be
3380	* cleared during a very small window when a small low
3381	* zone was balanced even under extreme pressure when the
3382	* overall node may be congested. Note that sc.reclaim_idx
3383	* is not used as buffer_heads_over_limit may have adjusted
3384	* it.
3385	*/
3386	for (i = classzone_idx; i >= 0; i--) {
3387	zone = pgdat->node_zones + i;
3388	if (!managed_zone(zone))
3389	continue;
3390
3391	if (zone_balanced(zone, sc.order, classzone_idx))
3392	goto out;
3393	}
3394
3395	/*
3396	* Do some background aging of the anon list, to give
3397	* pages a chance to be referenced before reclaiming. All
3398	* pages are rotated regardless of classzone as this is
3399	* about consistent aging.
3400	*/
3401	age_active_anon(pgdat, &sc);
3402
3403	/*
3404	* If we're getting trouble reclaiming, start doing writepage
3405	* even in laptop mode.
3406	*/
3407	if (sc.priority < DEF_PRIORITY - 2)
3408	sc.may_writepage = 1;
3409
3410	/* Call soft limit reclaim before calling shrink_node. */
3411	sc.nr_scanned = 0;
3412	nr_soft_scanned = 0;
3413	nr_soft_reclaimed = mem_cgroup_soft_limit_reclaim(pgdat, sc.order,
3414	sc.gfp_mask, &nr_soft_scanned);
3415	sc.nr_reclaimed += nr_soft_reclaimed;
3416
3417	/*
3418	* There should be no need to raise the scanning priority if
3419	* enough pages are already being scanned that that high
3420	* watermark would be met at 100% efficiency.
3421	*/
3422	if (kswapd_shrink_node(pgdat, &sc))
3423	raise_priority = false;
3424
3425	/*
3426	* If the low watermark is met there is no need for processes
3427	* to be throttled on pfmemalloc_wait as they should not be
3428	* able to safely make forward progress. Wake them
3429	*/
3430	if (waitqueue_active(&pgdat->pfmemalloc_wait) &&
3431	allow_direct_reclaim(pgdat))
3432	wake_up_all(&pgdat->pfmemalloc_wait);
3433
3434	/* Check if kswapd should be suspending */
3435	if (try_to_freeze() || kthread_should_stop())
3436	break;
3437
3438	/*
3439	* Raise priority if scanning rate is too low or there was no
3440	* progress in reclaiming pages
3441	*/
3442	nr_reclaimed = sc.nr_reclaimed - nr_reclaimed;
3443	if (raise_priority || !nr_reclaimed)
3444	sc.priority--;
3445	} while (sc.priority >= 1);
3446
3447	if (!sc.nr_reclaimed)
3448	pgdat->kswapd_failures++;
3449
3450	out:
3451	psi_memstall_leave(&pflags);
3452	/*
3453	* Return the order kswapd stopped reclaiming at as
3454	* prepare_kswapd_sleep() takes it into account. If another caller
3455	* entered the allocator slow path while kswapd was awake, order will
3456	* remain at the higher level.
3457	*/
3458	return sc.order;
3459	}
3460
3461	static void kswapd_try_to_sleep(pg_data_t *pgdat, int alloc_order, int reclaim_order,
3462	unsigned int classzone_idx)
3463	{
3464	long remaining = 0;
3465	DEFINE_WAIT(wait);
3466
3467	if (freezing(current) || kthread_should_stop())
3468	return;
3469
3470	prepare_to_wait(&pgdat->kswapd_wait, &wait, TASK_INTERRUPTIBLE);
3471
3472	/* Try to sleep for a short interval */
3473	if (prepare_kswapd_sleep(pgdat, reclaim_order, classzone_idx)) {
3474	/*
3475	* Compaction records what page blocks it recently failed to
3476	* isolate pages from and skips them in the future scanning.
3477	* When kswapd is going to sleep, it is reasonable to assume
3478	* that pages and compaction may succeed so reset the cache.
3479	*/
3480	reset_isolation_suitable(pgdat);
3481
3482	/*
3483	* We have freed the memory, now we should compact it to make
3484	* allocation of the requested order possible.
3485	*/
3486	wakeup_kcompactd(pgdat, alloc_order, classzone_idx);
3487
3488	remaining = schedule_timeout(HZ/10);
3489
3490	/*
3491	* If woken prematurely then reset kswapd_classzone_idx and
3492	* order. The values will either be from a wakeup request or
3493	* the previous request that slept prematurely.
3494	*/
3495	if (remaining) {
3496	pgdat->kswapd_classzone_idx = max(pgdat->kswapd_classzone_idx, classzone_idx);
3497	pgdat->kswapd_order = max(pgdat->kswapd_order, reclaim_order);
3498	}
3499
3500	finish_wait(&pgdat->kswapd_wait, &wait);
3501	prepare_to_wait(&pgdat->kswapd_wait, &wait, TASK_INTERRUPTIBLE);
3502	}
3503
3504	/*
3505	* After a short sleep, check if it was a premature sleep. If not, then
3506	* go fully to sleep until explicitly woken up.
3507	*/
3508	if (!remaining &&
3509	prepare_kswapd_sleep(pgdat, reclaim_order, classzone_idx)) {
3510	trace_mm_vmscan_kswapd_sleep(pgdat->node_id);
3511
3512	/*
3513	* vmstat counters are not perfectly accurate and the estimated
3514	* value for counters such as NR_FREE_PAGES can deviate from the
3515	* true value by nr_online_cpus * threshold. To avoid the zone
3516	* watermarks being breached while under pressure, we reduce the
3517	* per-cpu vmstat threshold while kswapd is awake and restore
3518	* them before going back to sleep.
3519	*/
3520	set_pgdat_percpu_threshold(pgdat, calculate_normal_threshold);
3521
3522	if (!kthread_should_stop())
3523	schedule();
3524
3525	set_pgdat_percpu_threshold(pgdat, calculate_pressure_threshold);
3526	} else {
3527	if (remaining)
3528	count_vm_event(KSWAPD_LOW_WMARK_HIT_QUICKLY);
3529	else
3530	count_vm_event(KSWAPD_HIGH_WMARK_HIT_QUICKLY);
3531	}
3532	finish_wait(&pgdat->kswapd_wait, &wait);
3533	}
3534
3535	/*
3536	* The background pageout daemon, started as a kernel thread
3537	* from the init process.
3538	*
3539	* This basically trickles out pages so that we have _some_
3540	* free memory available even if there is no other activity
3541	* that frees anything up. This is needed for things like routing
3542	* etc, where we otherwise might have all activity going on in
3543	* asynchronous contexts that cannot page things out.
3544	*
3545	* If there are applications that are active memory-allocators
3546	* (most normal use), this basically shouldn't matter.
3547	*/
3548	static int kswapd(void *p)
3549	{
3550	unsigned int alloc_order, reclaim_order, classzone_idx;
3551	pg_data_t *pgdat = (pg_data_t*)p;
3552	struct task_struct *tsk = current;
3553
3554	struct reclaim_state reclaim_state = {
3555	.reclaimed_slab = 0,
3556	};
3557	const struct cpumask *cpumask = cpumask_of_node(pgdat->node_id);
3558
3559	lockdep_set_current_reclaim_state(GFP_KERNEL);
3560
3561	if (!cpumask_empty(cpumask))
3562	set_cpus_allowed_ptr(tsk, cpumask);
3563	current->reclaim_state = &reclaim_state;
3564
3565	/*
3566	* Tell the memory management that we're a "memory allocator",
3567	* and that if we need more memory we should get access to it
3568	* regardless (see "__alloc_pages()"). "kswapd" should
3569	* never get caught in the normal page freeing logic.
3570	*
3571	* (Kswapd normally doesn't need memory anyway, but sometimes
3572	* you need a small amount of memory in order to be able to
3573	* page out something else, and this flag essentially protects
3574	* us from recursively trying to free more memory as we're
3575	* trying to free the first piece of memory in the first place).
3576	*/
3577	tsk->flags |= PF_MEMALLOC | PF_SWAPWRITE | PF_KSWAPD;
3578	set_freezable();
3579
3580	pgdat->kswapd_order = alloc_order = reclaim_order = 0;
3581	pgdat->kswapd_classzone_idx = classzone_idx = 0;
3582	#ifdef CONFIG_AMLOGIC_CMA
3583	set_user_nice(current, -5);
3584	#endif /* CONFIG_AMLOGIC_CMA */
3585	for ( ; ; ) {
3586	bool ret;
3587
3588	kswapd_try_sleep:
3589	kswapd_try_to_sleep(pgdat, alloc_order, reclaim_order,
3590	classzone_idx);
3591
3592	/* Read the new order and classzone_idx */
3593	alloc_order = reclaim_order = pgdat->kswapd_order;
3594	classzone_idx = pgdat->kswapd_classzone_idx;
3595	pgdat->kswapd_order = 0;
3596	pgdat->kswapd_classzone_idx = 0;
3597
3598	ret = try_to_freeze();
3599	if (kthread_should_stop())
3600	break;
3601
3602	/*
3603	* We can speed up thawing tasks if we don't call balance_pgdat
3604	* after returning from the refrigerator
3605	*/
3606	if (ret)
3607	continue;
3608
3609	/*
3610	* Reclaim begins at the requested order but if a high-order
3611	* reclaim fails then kswapd falls back to reclaiming for
3612	* order-0. If that happens, kswapd will consider sleeping
3613	* for the order it finished reclaiming at (reclaim_order)
3614	* but kcompactd is woken to compact for the original
3615	* request (alloc_order).
3616	*/
3617	trace_mm_vmscan_kswapd_wake(pgdat->node_id, classzone_idx,
3618	alloc_order);
3619	reclaim_order = balance_pgdat(pgdat, alloc_order, classzone_idx);
3620	if (reclaim_order < alloc_order)
3621	goto kswapd_try_sleep;
3622
3623	alloc_order = reclaim_order = pgdat->kswapd_order;
3624	classzone_idx = pgdat->kswapd_classzone_idx;
3625	}
3626
3627	tsk->flags &= ~(PF_MEMALLOC | PF_SWAPWRITE | PF_KSWAPD);
3628	current->reclaim_state = NULL;
3629	lockdep_clear_current_reclaim_state();
3630
3631	return 0;
3632	}
3633
3634	/*
3635	* A zone is low on free memory, so wake its kswapd task to service it.
3636	*/
3637	void wakeup_kswapd(struct zone *zone, int order, enum zone_type classzone_idx)
3638	{
3639	pg_data_t *pgdat;
3640	int z;
3641
3642	if (!managed_zone(zone))
3643	return;
3644
3645	if (!cpuset_zone_allowed(zone, GFP_KERNEL | __GFP_HARDWALL))
3646	return;
3647	pgdat = zone->zone_pgdat;
3648	pgdat->kswapd_classzone_idx = max(pgdat->kswapd_classzone_idx, classzone_idx);
3649	pgdat->kswapd_order = max(pgdat->kswapd_order, order);
3650	if (!waitqueue_active(&pgdat->kswapd_wait))
3651	return;
3652
3653	/* Hopeless node, leave it to direct reclaim */
3654	if (pgdat->kswapd_failures >= MAX_RECLAIM_RETRIES)
3655	return;
3656
3657	/* Only wake kswapd if all zones are unbalanced */
3658	for (z = 0; z <= classzone_idx; z++) {
3659	zone = pgdat->node_zones + z;
3660	if (!managed_zone(zone))
3661	continue;
3662
3663	if (zone_balanced(zone, order, classzone_idx))
3664	return;
3665	}
3666
3667	trace_mm_vmscan_wakeup_kswapd(pgdat->node_id, zone_idx(zone), order);
3668	wake_up_interruptible(&pgdat->kswapd_wait);
3669	}
3670
3671	#ifdef CONFIG_HIBERNATION
3672	/*
3673	* Try to free `nr_to_reclaim' of memory, system-wide, and return the number of
3674	* freed pages.
3675	*
3676	* Rather than trying to age LRUs the aim is to preserve the overall
3677	* LRU order by reclaiming preferentially
3678	* inactive > active > active referenced > active mapped
3679	*/
3680	unsigned long shrink_all_memory(unsigned long nr_to_reclaim)
3681	{
3682	struct reclaim_state reclaim_state;
3683	struct scan_control sc = {
3684	.nr_to_reclaim = nr_to_reclaim,
3685	.gfp_mask = GFP_HIGHUSER_MOVABLE,
3686	.reclaim_idx = MAX_NR_ZONES - 1,
3687	.priority = DEF_PRIORITY,
3688	.may_writepage = 1,
3689	.may_unmap = 1,
3690	.may_swap = 1,
3691	.hibernation_mode = 1,
3692	};
3693	struct zonelist *zonelist = node_zonelist(numa_node_id(), sc.gfp_mask);
3694	struct task_struct *p = current;
3695	unsigned long nr_reclaimed;
3696
3697	p->flags |= PF_MEMALLOC;
3698	lockdep_set_current_reclaim_state(sc.gfp_mask);
3699	reclaim_state.reclaimed_slab = 0;
3700	p->reclaim_state = &reclaim_state;
3701
3702	nr_reclaimed = do_try_to_free_pages(zonelist, &sc);
3703
3704	p->reclaim_state = NULL;
3705	lockdep_clear_current_reclaim_state();
3706	p->flags &= ~PF_MEMALLOC;
3707
3708	return nr_reclaimed;
3709	}
3710	#endif /* CONFIG_HIBERNATION */
3711
3712	/* It's optimal to keep kswapds on the same CPUs as their memory, but
3713	not required for correctness. So if the last cpu in a node goes
3714	away, we get changed to run anywhere: as the first one comes back,
3715	restore their cpu bindings. */
3716	static int cpu_callback(struct notifier_block *nfb, unsigned long action,
3717	void *hcpu)
3718	{
3719	int nid;
3720
3721	if (action == CPU_ONLINE || action == CPU_ONLINE_FROZEN) {
3722	for_each_node_state(nid, N_MEMORY) {
3723	pg_data_t *pgdat = NODE_DATA(nid);
3724	const struct cpumask *mask;
3725
3726	mask = cpumask_of_node(pgdat->node_id);
3727
3728	if (cpumask_any_and(cpu_online_mask, mask) < nr_cpu_ids)
3729	/* One of our CPUs online: restore mask */
3730	set_cpus_allowed_ptr(pgdat->kswapd, mask);
3731	}
3732	}
3733	return NOTIFY_OK;
3734	}
3735
3736	/*
3737	* This kswapd start function will be called by init and node-hot-add.
3738	* On node-hot-add, kswapd will moved to proper cpus if cpus are hot-added.
3739	*/
3740	int kswapd_run(int nid)
3741	{
3742	pg_data_t *pgdat = NODE_DATA(nid);
3743	int ret = 0;
3744
3745	if (pgdat->kswapd)
3746	return 0;
3747
3748	pgdat->kswapd = kthread_run(kswapd, pgdat, "kswapd%d", nid);
3749	if (IS_ERR(pgdat->kswapd)) {
3750	/* failure at boot is fatal */
3751	BUG_ON(system_state == SYSTEM_BOOTING);
3752	pr_err("Failed to start kswapd on node %d\n", nid);
3753	ret = PTR_ERR(pgdat->kswapd);
3754	pgdat->kswapd = NULL;
3755	}
3756	return ret;
3757	}
3758
3759	/*
3760	* Called by memory hotplug when all memory in a node is offlined. Caller must
3761	* hold mem_hotplug_begin/end().
3762	*/
3763	void kswapd_stop(int nid)
3764	{
3765	struct task_struct *kswapd = NODE_DATA(nid)->kswapd;
3766
3767	if (kswapd) {
3768	kthread_stop(kswapd);
3769	NODE_DATA(nid)->kswapd = NULL;
3770	}
3771	}
3772
3773	static int __init kswapd_init(void)
3774	{
3775	int nid;
3776
3777	swap_setup();
3778	for_each_node_state(nid, N_MEMORY)
3779	kswapd_run(nid);
3780	hotcpu_notifier(cpu_callback, 0);
3781	return 0;
3782	}
3783
3784	module_init(kswapd_init)
3785
3786	#ifdef CONFIG_NUMA
3787	/*
3788	* Node reclaim mode
3789	*
3790	* If non-zero call node_reclaim when the number of free pages falls below
3791	* the watermarks.
3792	*/
3793	int node_reclaim_mode __read_mostly;
3794
3795	#define RECLAIM_OFF 0
3796	#define RECLAIM_ZONE (1<<0) /* Run shrink_inactive_list on the zone */
3797	#define RECLAIM_WRITE (1<<1) /* Writeout pages during reclaim */
3798	#define RECLAIM_UNMAP (1<<2) /* Unmap pages during reclaim */
3799
3800	/*
3801	* Priority for NODE_RECLAIM. This determines the fraction of pages
3802	* of a node considered for each zone_reclaim. 4 scans 1/16th of
3803	* a zone.
3804	*/
3805	#define NODE_RECLAIM_PRIORITY 4
3806
3807	/*
3808	* Percentage of pages in a zone that must be unmapped for node_reclaim to
3809	* occur.
3810	*/
3811	int sysctl_min_unmapped_ratio = 1;
3812
3813	/*
3814	* If the number of slab pages in a zone grows beyond this percentage then
3815	* slab reclaim needs to occur.
3816	*/
3817	int sysctl_min_slab_ratio = 5;
3818
3819	static inline unsigned long node_unmapped_file_pages(struct pglist_data *pgdat)
3820	{
3821	unsigned long file_mapped = node_page_state(pgdat, NR_FILE_MAPPED);
3822	unsigned long file_lru = node_page_state(pgdat, NR_INACTIVE_FILE) +
3823	node_page_state(pgdat, NR_ACTIVE_FILE);
3824
3825	/*
3826	* It's possible for there to be more file mapped pages than
3827	* accounted for by the pages on the file LRU lists because
3828	* tmpfs pages accounted for as ANON can also be FILE_MAPPED
3829	*/
3830	return (file_lru > file_mapped) ? (file_lru - file_mapped) : 0;
3831	}
3832
3833	/* Work out how many page cache pages we can reclaim in this reclaim_mode */
3834	static unsigned long node_pagecache_reclaimable(struct pglist_data *pgdat)
3835	{
3836	unsigned long nr_pagecache_reclaimable;
3837	unsigned long delta = 0;
3838
3839	/*
3840	* If RECLAIM_UNMAP is set, then all file pages are considered
3841	* potentially reclaimable. Otherwise, we have to worry about
3842	* pages like swapcache and node_unmapped_file_pages() provides
3843	* a better estimate
3844	*/
3845	if (node_reclaim_mode & RECLAIM_UNMAP)
3846	nr_pagecache_reclaimable = node_page_state(pgdat, NR_FILE_PAGES);
3847	else
3848	nr_pagecache_reclaimable = node_unmapped_file_pages(pgdat);
3849
3850	/* If we can't clean pages, remove dirty pages from consideration */
3851	if (!(node_reclaim_mode & RECLAIM_WRITE))
3852	delta += node_page_state(pgdat, NR_FILE_DIRTY);
3853
3854	/* Watch for any possible underflows due to delta */
3855	if (unlikely(delta > nr_pagecache_reclaimable))
3856	delta = nr_pagecache_reclaimable;
3857
3858	return nr_pagecache_reclaimable - delta;
3859	}
3860
3861	/*
3862	* Try to free up some pages from this node through reclaim.
3863	*/
3864	static int __node_reclaim(struct pglist_data *pgdat, gfp_t gfp_mask, unsigned int order)
3865	{
3866	/* Minimum pages needed in order to stay on node */
3867	const unsigned long nr_pages = 1 << order;
3868	struct task_struct *p = current;
3869	struct reclaim_state reclaim_state;
3870	struct scan_control sc = {
3871	.nr_to_reclaim = max(nr_pages, SWAP_CLUSTER_MAX),
3872	.gfp_mask = memalloc_noio_flags(gfp_mask),
3873	.order = order,
3874	.priority = NODE_RECLAIM_PRIORITY,
3875	.may_writepage = !!(node_reclaim_mode & RECLAIM_WRITE),
3876	.may_unmap = !!(node_reclaim_mode & RECLAIM_UNMAP),
3877	.may_swap = 1,
3878	.reclaim_idx = gfp_zone(gfp_mask),
3879	};
3880
3881	cond_resched();
3882	/*
3883	* We need to be able to allocate from the reserves for RECLAIM_UNMAP
3884	* and we also need to be able to write out pages for RECLAIM_WRITE
3885	* and RECLAIM_UNMAP.
3886	*/
3887	p->flags |= PF_MEMALLOC | PF_SWAPWRITE;
3888	lockdep_set_current_reclaim_state(sc.gfp_mask);
3889	reclaim_state.reclaimed_slab = 0;
3890	p->reclaim_state = &reclaim_state;
3891
3892	if (node_pagecache_reclaimable(pgdat) > pgdat->min_unmapped_pages) {
3893	/*
3894	* Free memory by calling shrink zone with increasing
3895	* priorities until we have enough memory freed.
3896	*/
3897	do {
3898	shrink_node(pgdat, &sc);
3899	} while (sc.nr_reclaimed < nr_pages && --sc.priority >= 0);
3900	}
3901
3902	p->reclaim_state = NULL;
3903	current->flags &= ~(PF_MEMALLOC | PF_SWAPWRITE);
3904	lockdep_clear_current_reclaim_state();
3905	return sc.nr_reclaimed >= nr_pages;
3906	}
3907
3908	int node_reclaim(struct pglist_data *pgdat, gfp_t gfp_mask, unsigned int order)
3909	{
3910	int ret;
3911
3912	/*
3913	* Node reclaim reclaims unmapped file backed pages and
3914	* slab pages if we are over the defined limits.
3915	*
3916	* A small portion of unmapped file backed pages is needed for
3917	* file I/O otherwise pages read by file I/O will be immediately
3918	* thrown out if the node is overallocated. So we do not reclaim
3919	* if less than a specified percentage of the node is used by
3920	* unmapped file backed pages.
3921	*/
3922	if (node_pagecache_reclaimable(pgdat) <= pgdat->min_unmapped_pages &&
3923	sum_zone_node_page_state(pgdat->node_id, NR_SLAB_RECLAIMABLE) <= pgdat->min_slab_pages)
3924	return NODE_RECLAIM_FULL;
3925
3926	/*
3927	* Do not scan if the allocation should not be delayed.
3928	*/
3929	if (!gfpflags_allow_blocking(gfp_mask) || (current->flags & PF_MEMALLOC))
3930	return NODE_RECLAIM_NOSCAN;
3931
3932	/*
3933	* Only run node reclaim on the local node or on nodes that do not
3934	* have associated processors. This will favor the local processor
3935	* over remote processors and spread off node memory allocations
3936	* as wide as possible.
3937	*/
3938	if (node_state(pgdat->node_id, N_CPU) && pgdat->node_id != numa_node_id())
3939	return NODE_RECLAIM_NOSCAN;
3940
3941	if (test_and_set_bit(PGDAT_RECLAIM_LOCKED, &pgdat->flags))
3942	return NODE_RECLAIM_NOSCAN;
3943
3944	ret = __node_reclaim(pgdat, gfp_mask, order);
3945	clear_bit(PGDAT_RECLAIM_LOCKED, &pgdat->flags);
3946
3947	if (!ret)
3948	count_vm_event(PGSCAN_ZONE_RECLAIM_FAILED);
3949
3950	return ret;
3951	}
3952	#endif
3953
3954	/*
3955	* page_evictable - test whether a page is evictable
3956	* @page: the page to test
3957	*
3958	* Test whether page is evictable--i.e., should be placed on active/inactive
3959	* lists vs unevictable list.
3960	*
3961	* Reasons page might not be evictable:
3962	* (1) page's mapping marked unevictable
3963	* (2) page is part of an mlocked VMA
3964	*
3965	*/
3966	int page_evictable(struct page *page)
3967	{
3968	int ret;
3969
3970	/* Prevent address_space of inode and swap cache from being freed */
3971	rcu_read_lock();
3972	ret = !mapping_unevictable(page_mapping(page)) && !PageMlocked(page);
3973	rcu_read_unlock();
3974	return ret;
3975	}
3976
3977	#ifdef CONFIG_SHMEM
3978	/**
3979	* check_move_unevictable_pages - check pages for evictability and move to appropriate zone lru list
3980	* @pages: array of pages to check
3981	* @nr_pages: number of pages to check
3982	*
3983	* Checks pages for evictability and moves them to the appropriate lru list.
3984	*
3985	* This function is only used for SysV IPC SHM_UNLOCK.
3986	*/
3987	void check_move_unevictable_pages(struct page **pages, int nr_pages)
3988	{
3989	struct lruvec *lruvec;
3990	struct pglist_data *pgdat = NULL;
3991	int pgscanned = 0;
3992	int pgrescued = 0;
3993	int i;
3994
3995	for (i = 0; i < nr_pages; i++) {
3996	struct page *page = pages[i];
3997	struct pglist_data *pagepgdat = page_pgdat(page);
3998
3999	pgscanned++;
4000	if (pagepgdat != pgdat) {
4001	if (pgdat)
4002	spin_unlock_irq(&pgdat->lru_lock);
4003	pgdat = pagepgdat;
4004	spin_lock_irq(&pgdat->lru_lock);
4005	}
4006	lruvec = mem_cgroup_page_lruvec(page, pgdat);
4007
4008	if (!PageLRU(page) || !PageUnevictable(page))
4009	continue;
4010
4011	if (page_evictable(page)) {
4012	enum lru_list lru = page_lru_base_type(page);
4013
4014	VM_BUG_ON_PAGE(PageActive(page), page);
4015	ClearPageUnevictable(page);
4016	del_page_from_lru_list(page, lruvec, LRU_UNEVICTABLE);
4017	add_page_to_lru_list(page, lruvec, lru);
4018	pgrescued++;
4019	}
4020	}
4021
4022	if (pgdat) {
4023	__count_vm_events(UNEVICTABLE_PGRESCUED, pgrescued);
4024	__count_vm_events(UNEVICTABLE_PGSCANNED, pgscanned);
4025	spin_unlock_irq(&pgdat->lru_lock);
4026	}
4027	}
4028	#endif /* CONFIG_SHMEM */
4029