path: root/mm/compaction.c (plain)
blob: 0bc730ca90a6604b855e7440709dfd018127c355

1	/*
2	* linux/mm/compaction.c
3	*
4	* Memory compaction for the reduction of external fragmentation. Note that
5	* this heavily depends upon page migration to do all the real heavy
6	* lifting
7	*
8	* Copyright IBM Corp. 2007-2010 Mel Gorman <mel@csn.ul.ie>
9	*/
10	#include <linux/cpu.h>
11	#include <linux/swap.h>
12	#include <linux/migrate.h>
13	#include <linux/compaction.h>
14	#include <linux/mm_inline.h>
15	#include <linux/backing-dev.h>
16	#include <linux/sysctl.h>
17	#include <linux/sysfs.h>
18	#include <linux/page-isolation.h>
19	#include <linux/kasan.h>
20	#include <linux/kthread.h>
21	#include <linux/freezer.h>
22	#include <linux/page_owner.h>
23	#include <linux/psi.h>
24	#include "internal.h"
25	#ifdef CONFIG_AMLOGIC_PAGE_TRACE
26	#include <linux/amlogic/page_trace.h>
27	#endif
28
29	#ifdef CONFIG_COMPACTION
30	static inline void count_compact_event(enum vm_event_item item)
31	{
32	count_vm_event(item);
33	}
34
35	static inline void count_compact_events(enum vm_event_item item, long delta)
36	{
37	count_vm_events(item, delta);
38	}
39	#else
40	#define count_compact_event(item) do { } while (0)
41	#define count_compact_events(item, delta) do { } while (0)
42	#endif
43
44	#if defined CONFIG_COMPACTION || defined CONFIG_CMA
45
46	#define CREATE_TRACE_POINTS
47	#include <trace/events/compaction.h>
48
49	#define block_start_pfn(pfn, order) round_down(pfn, 1UL << (order))
50	#define block_end_pfn(pfn, order) ALIGN((pfn) + 1, 1UL << (order))
51	#define pageblock_start_pfn(pfn) block_start_pfn(pfn, pageblock_order)
52	#define pageblock_end_pfn(pfn) block_end_pfn(pfn, pageblock_order)
53
54	static unsigned long release_freepages(struct list_head *freelist)
55	{
56	struct page *page, *next;
57	unsigned long high_pfn = 0;
58
59	list_for_each_entry_safe(page, next, freelist, lru) {
60	unsigned long pfn = page_to_pfn(page);
61	list_del(&page->lru);
62	__free_page(page);
63	if (pfn > high_pfn)
64	high_pfn = pfn;
65	}
66
67	return high_pfn;
68	}
69
70	static void map_pages(struct list_head *list)
71	{
72	unsigned int i, order, nr_pages;
73	struct page *page, *next;
74	LIST_HEAD(tmp_list);
75
76	list_for_each_entry_safe(page, next, list, lru) {
77	list_del(&page->lru);
78
79	order = page_private(page);
80	nr_pages = 1 << order;
81
82	post_alloc_hook(page, order, __GFP_MOVABLE);
83	if (order)
84	split_page(page, order);
85
86	for (i = 0; i < nr_pages; i++) {
87	list_add(&page->lru, &tmp_list);
88	page++;
89	}
90	}
91
92	list_splice(&tmp_list, list);
93	}
94
95	static inline bool migrate_async_suitable(int migratetype)
96	{
97	return is_migrate_cma(migratetype) || migratetype == MIGRATE_MOVABLE;
98	}
99
100	#ifdef CONFIG_COMPACTION
101
102	int PageMovable(struct page *page)
103	{
104	struct address_space *mapping;
105
106	VM_BUG_ON_PAGE(!PageLocked(page), page);
107	if (!__PageMovable(page))
108	return 0;
109
110	mapping = page_mapping(page);
111	if (mapping && mapping->a_ops && mapping->a_ops->isolate_page)
112	return 1;
113
114	return 0;
115	}
116	EXPORT_SYMBOL(PageMovable);
117
118	void __SetPageMovable(struct page *page, struct address_space *mapping)
119	{
120	VM_BUG_ON_PAGE(!PageLocked(page), page);
121	VM_BUG_ON_PAGE((unsigned long)mapping & PAGE_MAPPING_MOVABLE, page);
122	page->mapping = (void *)((unsigned long)mapping | PAGE_MAPPING_MOVABLE);
123	}
124	EXPORT_SYMBOL(__SetPageMovable);
125
126	void __ClearPageMovable(struct page *page)
127	{
128	VM_BUG_ON_PAGE(!PageLocked(page), page);
129	VM_BUG_ON_PAGE(!PageMovable(page), page);
130	/*
131	* Clear registered address_space val with keeping PAGE_MAPPING_MOVABLE
132	* flag so that VM can catch up released page by driver after isolation.
133	* With it, VM migration doesn't try to put it back.
134	*/
135	page->mapping = (void *)((unsigned long)page->mapping &
136	PAGE_MAPPING_MOVABLE);
137	}
138	EXPORT_SYMBOL(__ClearPageMovable);
139
140	/* Do not skip compaction more than 64 times */
141	#define COMPACT_MAX_DEFER_SHIFT 6
142
143	/*
144	* Compaction is deferred when compaction fails to result in a page
145	* allocation success. 1 << compact_defer_limit compactions are skipped up
146	* to a limit of 1 << COMPACT_MAX_DEFER_SHIFT
147	*/
148	void defer_compaction(struct zone *zone, int order)
149	{
150	zone->compact_considered = 0;
151	zone->compact_defer_shift++;
152
153	if (order < zone->compact_order_failed)
154	zone->compact_order_failed = order;
155
156	if (zone->compact_defer_shift > COMPACT_MAX_DEFER_SHIFT)
157	zone->compact_defer_shift = COMPACT_MAX_DEFER_SHIFT;
158
159	trace_mm_compaction_defer_compaction(zone, order);
160	}
161
162	/* Returns true if compaction should be skipped this time */
163	bool compaction_deferred(struct zone *zone, int order)
164	{
165	unsigned long defer_limit = 1UL << zone->compact_defer_shift;
166
167	if (order < zone->compact_order_failed)
168	return false;
169
170	/* Avoid possible overflow */
171	if (++zone->compact_considered > defer_limit)
172	zone->compact_considered = defer_limit;
173
174	if (zone->compact_considered >= defer_limit)
175	return false;
176
177	trace_mm_compaction_deferred(zone, order);
178
179	return true;
180	}
181
182	/*
183	* Update defer tracking counters after successful compaction of given order,
184	* which means an allocation either succeeded (alloc_success == true) or is
185	* expected to succeed.
186	*/
187	void compaction_defer_reset(struct zone *zone, int order,
188	bool alloc_success)
189	{
190	if (alloc_success) {
191	zone->compact_considered = 0;
192	zone->compact_defer_shift = 0;
193	}
194	if (order >= zone->compact_order_failed)
195	zone->compact_order_failed = order + 1;
196
197	trace_mm_compaction_defer_reset(zone, order);
198	}
199
200	/* Returns true if restarting compaction after many failures */
201	bool compaction_restarting(struct zone *zone, int order)
202	{
203	if (order < zone->compact_order_failed)
204	return false;
205
206	return zone->compact_defer_shift == COMPACT_MAX_DEFER_SHIFT &&
207	zone->compact_considered >= 1UL << zone->compact_defer_shift;
208	}
209
210	/* Returns true if the pageblock should be scanned for pages to isolate. */
211	static inline bool isolation_suitable(struct compact_control *cc,
212	struct page *page)
213	{
214	if (cc->ignore_skip_hint)
215	return true;
216
217	return !get_pageblock_skip(page);
218	}
219
220	static void reset_cached_positions(struct zone *zone)
221	{
222	zone->compact_cached_migrate_pfn[0] = zone->zone_start_pfn;
223	zone->compact_cached_migrate_pfn[1] = zone->zone_start_pfn;
224	zone->compact_cached_free_pfn =
225	pageblock_start_pfn(zone_end_pfn(zone) - 1);
226	}
227
228	/*
229	* This function is called to clear all cached information on pageblocks that
230	* should be skipped for page isolation when the migrate and free page scanner
231	* meet.
232	*/
233	static void __reset_isolation_suitable(struct zone *zone)
234	{
235	unsigned long start_pfn = zone->zone_start_pfn;
236	unsigned long end_pfn = zone_end_pfn(zone);
237	unsigned long pfn;
238
239	zone->compact_blockskip_flush = false;
240
241	/* Walk the zone and mark every pageblock as suitable for isolation */
242	for (pfn = start_pfn; pfn < end_pfn; pfn += pageblock_nr_pages) {
243	struct page *page;
244
245	cond_resched();
246
247	if (!pfn_valid(pfn))
248	continue;
249
250	page = pfn_to_page(pfn);
251	if (zone != page_zone(page))
252	continue;
253
254	clear_pageblock_skip(page);
255	}
256
257	reset_cached_positions(zone);
258	}
259
260	void reset_isolation_suitable(pg_data_t *pgdat)
261	{
262	int zoneid;
263
264	for (zoneid = 0; zoneid < MAX_NR_ZONES; zoneid++) {
265	struct zone *zone = &pgdat->node_zones[zoneid];
266	if (!populated_zone(zone))
267	continue;
268
269	/* Only flush if a full compaction finished recently */
270	if (zone->compact_blockskip_flush)
271	__reset_isolation_suitable(zone);
272	}
273	}
274
275	/*
276	* If no pages were isolated then mark this pageblock to be skipped in the
277	* future. The information is later cleared by __reset_isolation_suitable().
278	*/
279	static void update_pageblock_skip(struct compact_control *cc,
280	struct page *page, unsigned long nr_isolated,
281	bool migrate_scanner)
282	{
283	struct zone *zone = cc->zone;
284	unsigned long pfn;
285
286	if (cc->ignore_skip_hint)
287	return;
288
289	if (!page)
290	return;
291
292	if (nr_isolated)
293	return;
294
295	set_pageblock_skip(page);
296
297	pfn = page_to_pfn(page);
298
299	/* Update where async and sync compaction should restart */
300	if (migrate_scanner) {
301	if (pfn > zone->compact_cached_migrate_pfn[0])
302	zone->compact_cached_migrate_pfn[0] = pfn;
303	if (cc->mode != MIGRATE_ASYNC &&
304	pfn > zone->compact_cached_migrate_pfn[1])
305	zone->compact_cached_migrate_pfn[1] = pfn;
306	} else {
307	if (pfn < zone->compact_cached_free_pfn)
308	zone->compact_cached_free_pfn = pfn;
309	}
310	}
311	#else
312	static inline bool isolation_suitable(struct compact_control *cc,
313	struct page *page)
314	{
315	return true;
316	}
317
318	static void update_pageblock_skip(struct compact_control *cc,
319	struct page *page, unsigned long nr_isolated,
320	bool migrate_scanner)
321	{
322	}
323	#endif /* CONFIG_COMPACTION */
324
325	/*
326	* Compaction requires the taking of some coarse locks that are potentially
327	* very heavily contended. For async compaction, back out if the lock cannot
328	* be taken immediately. For sync compaction, spin on the lock if needed.
329	*
330	* Returns true if the lock is held
331	* Returns false if the lock is not held and compaction should abort
332	*/
333	static bool compact_trylock_irqsave(spinlock_t *lock, unsigned long *flags,
334	struct compact_control *cc)
335	{
336	if (cc->mode == MIGRATE_ASYNC) {
337	if (!spin_trylock_irqsave(lock, *flags)) {
338	cc->contended = true;
339	return false;
340	}
341	} else {
342	spin_lock_irqsave(lock, *flags);
343	}
344
345	return true;
346	}
347
348	/*
349	* Compaction requires the taking of some coarse locks that are potentially
350	* very heavily contended. The lock should be periodically unlocked to avoid
351	* having disabled IRQs for a long time, even when there is nobody waiting on
352	* the lock. It might also be that allowing the IRQs will result in
353	* need_resched() becoming true. If scheduling is needed, async compaction
354	* aborts. Sync compaction schedules.
355	* Either compaction type will also abort if a fatal signal is pending.
356	* In either case if the lock was locked, it is dropped and not regained.
357	*
358	* Returns true if compaction should abort due to fatal signal pending, or
359	* async compaction due to need_resched()
360	* Returns false when compaction can continue (sync compaction might have
361	* scheduled)
362	*/
363	static bool compact_unlock_should_abort(spinlock_t *lock,
364	unsigned long flags, bool *locked, struct compact_control *cc)
365	{
366	if (*locked) {
367	spin_unlock_irqrestore(lock, flags);
368	*locked = false;
369	}
370
371	if (fatal_signal_pending(current)) {
372	cc->contended = true;
373	return true;
374	}
375
376	if (need_resched()) {
377	if (cc->mode == MIGRATE_ASYNC) {
378	cc->contended = true;
379	return true;
380	}
381	cond_resched();
382	}
383
384	return false;
385	}
386
387	/*
388	* Aside from avoiding lock contention, compaction also periodically checks
389	* need_resched() and either schedules in sync compaction or aborts async
390	* compaction. This is similar to what compact_unlock_should_abort() does, but
391	* is used where no lock is concerned.
392	*
393	* Returns false when no scheduling was needed, or sync compaction scheduled.
394	* Returns true when async compaction should abort.
395	*/
396	static inline bool compact_should_abort(struct compact_control *cc)
397	{
398	/* async compaction aborts if contended */
399	if (need_resched()) {
400	if (cc->mode == MIGRATE_ASYNC) {
401	cc->contended = true;
402	return true;
403	}
404
405	cond_resched();
406	}
407
408	return false;
409	}
410
411	/*
412	* Isolate free pages onto a private freelist. If @strict is true, will abort
413	* returning 0 on any invalid PFNs or non-free pages inside of the pageblock
414	* (even though it may still end up isolating some pages).
415	*/
416	static unsigned long isolate_freepages_block(struct compact_control *cc,
417	unsigned long *start_pfn,
418	unsigned long end_pfn,
419	struct list_head *freelist,
420	bool strict)
421	{
422	int nr_scanned = 0, total_isolated = 0;
423	struct page *cursor, *valid_page = NULL;
424	unsigned long flags = 0;
425	bool locked = false;
426	unsigned long blockpfn = *start_pfn;
427	unsigned int order;
428
429	cursor = pfn_to_page(blockpfn);
430
431	/* Isolate free pages. */
432	for (; blockpfn < end_pfn; blockpfn++, cursor++) {
433	int isolated;
434	struct page *page = cursor;
435
436	/*
437	* Periodically drop the lock (if held) regardless of its
438	* contention, to give chance to IRQs. Abort if fatal signal
439	* pending or async compaction detects need_resched()
440	*/
441	#ifdef CONFIG_AMLOGIC_CMA
442	if (!(blockpfn % SWAP_CLUSTER_MAX)
443	&& compact_unlock_should_abort(&cc->zone->lock, flags,
444	&locked, cc)) {
445	if (strict)
446	cma_debug(1, page, "abort, blk:%lx, swap:%ld\n",
447	blockpfn, SWAP_CLUSTER_MAX);
448	break;
449	}
450	#else
451	if (!(blockpfn % SWAP_CLUSTER_MAX)
452	&& compact_unlock_should_abort(&cc->zone->lock, flags,
453	&locked, cc))
454	break;
455	#endif
456
457	nr_scanned++;
458	#ifdef CONFIG_AMLOGIC_CMA
459	if (!pfn_valid_within(blockpfn)) {
460	if (strict)
461	cma_debug(1, page, "invalid pfn:%lx\n",
462	blockpfn);
463	goto isolate_fail;
464	}
465	#else
466	if (!pfn_valid_within(blockpfn))
467	goto isolate_fail;
468	#endif
469
470	if (!valid_page)
471	valid_page = page;
472
473	/*
474	* For compound pages such as THP and hugetlbfs, we can save
475	* potentially a lot of iterations if we skip them at once.
476	* The check is racy, but we can consider only valid values
477	* and the only danger is skipping too much.
478	*/
479	if (PageCompound(page)) {
480	unsigned int comp_order = compound_order(page);
481
482	if (likely(comp_order < MAX_ORDER)) {
483	blockpfn += (1UL << comp_order) - 1;
484	cursor += (1UL << comp_order) - 1;
485	}
486	#ifdef CONFIG_AMLOGIC_CMA
487	if (strict)
488	cma_debug(1, page, "compound page:%lx\n",
489	page_to_pfn(page));
490	#endif
491	goto isolate_fail;
492	}
493
494	#ifdef CONFIG_AMLOGIC_CMA
495	if (!PageBuddy(page)) {
496	if (strict)
497	cma_debug(1, page, " NO buddy page1:%lx\n",
498	page_to_pfn(page));
499	goto isolate_fail;
500	}
501	#else
502	if (!PageBuddy(page))
503	goto isolate_fail;
504	#endif
505
506	/*
507	* If we already hold the lock, we can skip some rechecking.
508	* Note that if we hold the lock now, checked_pageblock was
509	* already set in some previous iteration (or strict is true),
510	* so it is correct to skip the suitable migration target
511	* recheck as well.
512	*/
513	if (!locked) {
514	/*
515	* The zone lock must be held to isolate freepages.
516	* Unfortunately this is a very coarse lock and can be
517	* heavily contended if there are parallel allocations
518	* or parallel compactions. For async compaction do not
519	* spin on the lock and we acquire the lock as late as
520	* possible.
521	*/
522	locked = compact_trylock_irqsave(&cc->zone->lock,
523	&flags, cc);
524	#ifdef CONFIG_AMLOGIC_CMA
525	if (!locked) {
526	if (strict)
527	cma_debug(1, page, " lock failed:%lx\n",
528	page_to_pfn(page));
529	break;
530	}
531	/* Recheck this is a buddy page under lock */
532	if (!PageBuddy(page)) {
533	if (strict)
534	cma_debug(1, page, " No buddy2:%lx\n",
535	page_to_pfn(page));
536	goto isolate_fail;
537	}
538	#else
539	if (!locked)
540	break;
541
542	/* Recheck this is a buddy page under lock */
543	if (!PageBuddy(page))
544	goto isolate_fail;
545	#endif
546	}
547
548	/* Found a free page, will break it into order-0 pages */
549	order = page_order(page);
550	isolated = __isolate_free_page(page, order);
551	#ifdef CONFIG_AMLOGIC_CMA
552	if (!isolated) {
553	if (strict)
554	cma_debug(1, page, "iso free fail:%lx, o:%d\n",
555	page_to_pfn(page), order);
556	break;
557	}
558	#else
559	if (!isolated)
560	break;
561	#endif
562	set_page_private(page, order);
563
564	total_isolated += isolated;
565	cc->nr_freepages += isolated;
566	list_add_tail(&page->lru, freelist);
567
568	if (!strict && cc->nr_migratepages <= cc->nr_freepages) {
569	blockpfn += isolated;
570	break;
571	}
572	/* Advance to the end of split page */
573	blockpfn += isolated - 1;
574	cursor += isolated - 1;
575	continue;
576
577	isolate_fail:
578	if (strict)
579	break;
580	else
581	continue;
582
583	}
584
585	if (locked)
586	spin_unlock_irqrestore(&cc->zone->lock, flags);
587
588	/*
589	* There is a tiny chance that we have read bogus compound_order(),
590	* so be careful to not go outside of the pageblock.
591	*/
592	if (unlikely(blockpfn > end_pfn))
593	blockpfn = end_pfn;
594
595	trace_mm_compaction_isolate_freepages(*start_pfn, blockpfn,
596	nr_scanned, total_isolated);
597
598	/* Record how far we have got within the block */
599	*start_pfn = blockpfn;
600
601	/*
602	* If strict isolation is requested by CMA then check that all the
603	* pages requested were isolated. If there were any failures, 0 is
604	* returned and CMA will fail.
605	*/
606	if (strict && blockpfn < end_pfn)
607	total_isolated = 0;
608
609	/* Update the pageblock-skip if the whole pageblock was scanned */
610	if (blockpfn == end_pfn)
611	update_pageblock_skip(cc, valid_page, total_isolated, false);
612
613	count_compact_events(COMPACTFREE_SCANNED, nr_scanned);
614	if (total_isolated)
615	count_compact_events(COMPACTISOLATED, total_isolated);
616	return total_isolated;
617	}
618
619	/**
620	* isolate_freepages_range() - isolate free pages.
621	* @start_pfn: The first PFN to start isolating.
622	* @end_pfn: The one-past-last PFN.
623	*
624	* Non-free pages, invalid PFNs, or zone boundaries within the
625	* [start_pfn, end_pfn) range are considered errors, cause function to
626	* undo its actions and return zero.
627	*
628	* Otherwise, function returns one-past-the-last PFN of isolated page
629	* (which may be greater then end_pfn if end fell in a middle of
630	* a free page).
631	*/
632	unsigned long
633	isolate_freepages_range(struct compact_control *cc,
634	unsigned long start_pfn, unsigned long end_pfn)
635	{
636	unsigned long isolated, pfn, block_start_pfn, block_end_pfn;
637	LIST_HEAD(freelist);
638
639	pfn = start_pfn;
640	block_start_pfn = pageblock_start_pfn(pfn);
641	if (block_start_pfn < cc->zone->zone_start_pfn)
642	block_start_pfn = cc->zone->zone_start_pfn;
643	block_end_pfn = pageblock_end_pfn(pfn);
644
645	for (; pfn < end_pfn; pfn += isolated,
646	block_start_pfn = block_end_pfn,
647	block_end_pfn += pageblock_nr_pages) {
648	/* Protect pfn from changing by isolate_freepages_block */
649	unsigned long isolate_start_pfn = pfn;
650
651	block_end_pfn = min(block_end_pfn, end_pfn);
652
653	/*
654	* pfn could pass the block_end_pfn if isolated freepage
655	* is more than pageblock order. In this case, we adjust
656	* scanning range to right one.
657	*/
658	if (pfn >= block_end_pfn) {
659	block_start_pfn = pageblock_start_pfn(pfn);
660	block_end_pfn = pageblock_end_pfn(pfn);
661	block_end_pfn = min(block_end_pfn, end_pfn);
662	}
663
664	#ifdef CONFIG_AMLOGIC_CMA
665	if (!pageblock_pfn_to_page(block_start_pfn,
666	block_end_pfn, cc->zone)) {
667	cma_debug(1, NULL, " no page block\n");
668	break;
669	}
670	#else
671	if (!pageblock_pfn_to_page(block_start_pfn,
672	block_end_pfn, cc->zone))
673	break;
674	#endif
675
676	isolated = isolate_freepages_block(cc, &isolate_start_pfn,
677	block_end_pfn, &freelist, true);
678
679	/*
680	* In strict mode, isolate_freepages_block() returns 0 if
681	* there are any holes in the block (ie. invalid PFNs or
682	* non-free pages).
683	*/
684	#ifdef CONFIG_AMLOGIC_CMA
685	if (!isolated) {
686	cma_debug(1, NULL, " isolate free page failed\n");
687	break;
688	}
689	#else
690	if (!isolated)
691	break;
692	#endif
693
694	/*
695	* If we managed to isolate pages, it is always (1 << n) *
696	* pageblock_nr_pages for some non-negative n. (Max order
697	* page may span two pageblocks).
698	*/
699	}
700
701	/* __isolate_free_page() does not map the pages */
702	map_pages(&freelist);
703
704	if (pfn < end_pfn) {
705	/* Loop terminated early, cleanup. */
706	#ifdef CONFIG_AMLOGIC_CMA
707	cma_debug(1, NULL, "pfn:%lx, end:%lx, start:%lx\n",
708	pfn, end_pfn, start_pfn);
709	#endif
710	release_freepages(&freelist);
711	return 0;
712	}
713
714	/* We don't use freelists for anything. */
715	return pfn;
716	}
717
718	/* Similar to reclaim, but different enough that they don't share logic */
719	static bool too_many_isolated(struct zone *zone)
720	{
721	#ifdef CONFIG_AMLOGIC_CMA
722	signed long active, inactive, isolated;
723	#else
724	unsigned long active, inactive, isolated;
725	#endif
726
727	inactive = node_page_state(zone->zone_pgdat, NR_INACTIVE_FILE) +
728	node_page_state(zone->zone_pgdat, NR_INACTIVE_ANON);
729	active = node_page_state(zone->zone_pgdat, NR_ACTIVE_FILE) +
730	node_page_state(zone->zone_pgdat, NR_ACTIVE_ANON);
731	isolated = node_page_state(zone->zone_pgdat, NR_ISOLATED_FILE) +
732	node_page_state(zone->zone_pgdat, NR_ISOLATED_ANON);
733
734	#ifdef CONFIG_AMLOGIC_CMA
735	isolated -= global_page_state(NR_CMA_ISOLATED);
736	WARN_ONCE(isolated > (inactive + active) / 2,
737	"isolated:%ld, cma:%ld, inactive:%ld, active:%ld\n",
738	isolated, global_page_state(NR_CMA_ISOLATED),
739	inactive, active);
740	#endif /* CONFIG_AMLOGIC_CMA */
741	return isolated > (inactive + active) / 2;
742	}
743	#ifdef CONFIG_AMLOGIC_CMA
744	static void check_page_to_cma(struct compact_control *cc, struct page *page)
745	{
746	struct address_space *mapping;
747
748	if (cc->forbid_to_cma) /* no need check once it is true */
749	return;
750
751	mapping = page_mapping(page);
752	if ((unsigned long)mapping & PAGE_MAPPING_ANON)
753	mapping = NULL;
754
755	if (PageKsm(page) && !PageSlab(page))
756	cc->forbid_to_cma = true;
757
758	if (mapping && cma_forbidden_mask(mapping_gfp_mask(mapping)))
759	cc->forbid_to_cma = true;
760	}
761	#endif
762
763	/**
764	* isolate_migratepages_block() - isolate all migrate-able pages within
765	* a single pageblock
766	* @cc: Compaction control structure.
767	* @low_pfn: The first PFN to isolate
768	* @end_pfn: The one-past-the-last PFN to isolate, within same pageblock
769	* @isolate_mode: Isolation mode to be used.
770	*
771	* Isolate all pages that can be migrated from the range specified by
772	* [low_pfn, end_pfn). The range is expected to be within same pageblock.
773	* Returns zero if there is a fatal signal pending, otherwise PFN of the
774	* first page that was not scanned (which may be both less, equal to or more
775	* than end_pfn).
776	*
777	* The pages are isolated on cc->migratepages list (not required to be empty),
778	* and cc->nr_migratepages is updated accordingly. The cc->migrate_pfn field
779	* is neither read nor updated.
780	*/
781	static unsigned long
782	isolate_migratepages_block(struct compact_control *cc, unsigned long low_pfn,
783	unsigned long end_pfn, isolate_mode_t isolate_mode)
784	{
785	struct zone *zone = cc->zone;
786	unsigned long nr_scanned = 0, nr_isolated = 0;
787	struct lruvec *lruvec;
788	unsigned long flags = 0;
789	bool locked = false;
790	struct page *page = NULL, *valid_page = NULL;
791	unsigned long start_pfn = low_pfn;
792	bool skip_on_failure = false;
793	unsigned long next_skip_pfn = 0;
794
795	/*
796	* Ensure that there are not too many pages isolated from the LRU
797	* list by either parallel reclaimers or compaction. If there are,
798	* delay for some time until fewer pages are isolated
799	*/
800	while (unlikely(too_many_isolated(zone))) {
801	/* async migration should just abort */
802	if (cc->mode == MIGRATE_ASYNC)
803	return 0;
804
805	congestion_wait(BLK_RW_ASYNC, HZ/10);
806
807	if (fatal_signal_pending(current))
808	return 0;
809	}
810
811	if (compact_should_abort(cc))
812	return 0;
813
814	if (cc->direct_compaction && (cc->mode == MIGRATE_ASYNC)) {
815	skip_on_failure = true;
816	next_skip_pfn = block_end_pfn(low_pfn, cc->order);
817	}
818
819	/* Time to isolate some pages for migration */
820	for (; low_pfn < end_pfn; low_pfn++) {
821
822	if (skip_on_failure && low_pfn >= next_skip_pfn) {
823	/*
824	* We have isolated all migration candidates in the
825	* previous order-aligned block, and did not skip it due
826	* to failure. We should migrate the pages now and
827	* hopefully succeed compaction.
828	*/
829	if (nr_isolated)
830	break;
831
832	/*
833	* We failed to isolate in the previous order-aligned
834	* block. Set the new boundary to the end of the
835	* current block. Note we can't simply increase
836	* next_skip_pfn by 1 << order, as low_pfn might have
837	* been incremented by a higher number due to skipping
838	* a compound or a high-order buddy page in the
839	* previous loop iteration.
840	*/
841	next_skip_pfn = block_end_pfn(low_pfn, cc->order);
842	}
843
844	/*
845	* Periodically drop the lock (if held) regardless of its
846	* contention, to give chance to IRQs. Abort async compaction
847	* if contended.
848	*/
849	if (!(low_pfn % SWAP_CLUSTER_MAX)
850	&& compact_unlock_should_abort(zone_lru_lock(zone), flags,
851	&locked, cc))
852	break;
853
854	if (!pfn_valid_within(low_pfn))
855	goto isolate_fail;
856	nr_scanned++;
857
858	page = pfn_to_page(low_pfn);
859
860	#ifdef CONFIG_AMLOGIC_CMA
861	check_page_to_cma(cc, page);
862	#endif
863	if (!valid_page)
864	valid_page = page;
865
866	/*
867	* Skip if free. We read page order here without zone lock
868	* which is generally unsafe, but the race window is small and
869	* the worst thing that can happen is that we skip some
870	* potential isolation targets.
871	*/
872	if (PageBuddy(page)) {
873	unsigned long freepage_order = page_order_unsafe(page);
874
875	/*
876	* Without lock, we cannot be sure that what we got is
877	* a valid page order. Consider only values in the
878	* valid order range to prevent low_pfn overflow.
879	*/
880	if (freepage_order > 0 && freepage_order < MAX_ORDER)
881	low_pfn += (1UL << freepage_order) - 1;
882	continue;
883	}
884
885	/*
886	* Regardless of being on LRU, compound pages such as THP and
887	* hugetlbfs are not to be compacted. We can potentially save
888	* a lot of iterations if we skip them at once. The check is
889	* racy, but we can consider only valid values and the only
890	* danger is skipping too much.
891	*/
892	if (PageCompound(page)) {
893	unsigned int comp_order = compound_order(page);
894
895	if (likely(comp_order < MAX_ORDER))
896	low_pfn += (1UL << comp_order) - 1;
897
898	goto isolate_fail;
899	}
900
901	/*
902	* Check may be lockless but that's ok as we recheck later.
903	* It's possible to migrate LRU and non-lru movable pages.
904	* Skip any other type of page
905	*/
906	if (!PageLRU(page)) {
907	/*
908	* __PageMovable can return false positive so we need
909	* to verify it under page_lock.
910	*/
911	if (unlikely(__PageMovable(page)) &&
912	!PageIsolated(page)) {
913	if (locked) {
914	spin_unlock_irqrestore(zone_lru_lock(zone),
915	flags);
916	locked = false;
917	}
918
919	if (isolate_movable_page(page, isolate_mode))
920	goto isolate_success;
921	}
922
923	goto isolate_fail;
924	}
925
926	/*
927	* Migration will fail if an anonymous page is pinned in memory,
928	* so avoid taking lru_lock and isolating it unnecessarily in an
929	* admittedly racy check.
930	*/
931	#ifndef CONFIG_AMLOGIC_CMA
932	if (!page_mapping(page) &&
933	page_count(page) > page_mapcount(page))
934	goto isolate_fail;
935	#endif /* !CONFIG_AMLOGIC_CMA */
936
937	/* If we already hold the lock, we can skip some rechecking */
938	if (!locked) {
939	locked = compact_trylock_irqsave(zone_lru_lock(zone),
940	&flags, cc);
941	if (!locked)
942	break;
943
944	/* Recheck PageLRU and PageCompound under lock */
945	if (!PageLRU(page))
946	goto isolate_fail;
947
948	/*
949	* Page become compound since the non-locked check,
950	* and it's on LRU. It can only be a THP so the order
951	* is safe to read and it's 0 for tail pages.
952	*/
953	if (unlikely(PageCompound(page))) {
954	low_pfn += (1UL << compound_order(page)) - 1;
955	goto isolate_fail;
956	}
957	}
958	#ifdef CONFIG_AMLOGIC_CMA /* under protect of lock */
959	if (!page_mapping(page) &&
960	page_count(page) > page_mapcount(page))
961	goto isolate_fail;
962	#endif /* CONFIG_AMLOGIC_CMA */
963
964	lruvec = mem_cgroup_page_lruvec(page, zone->zone_pgdat);
965
966	/* Try isolate the page */
967	if (__isolate_lru_page(page, isolate_mode) != 0)
968	goto isolate_fail;
969
970	VM_BUG_ON_PAGE(PageCompound(page), page);
971
972	/* Successfully isolated */
973	del_page_from_lru_list(page, lruvec, page_lru(page));
974	inc_node_page_state(page,
975	NR_ISOLATED_ANON + page_is_file_cache(page));
976
977	isolate_success:
978	list_add(&page->lru, &cc->migratepages);
979	#ifdef CONFIG_AMLOGIC_CMA
980	if (cc->page_type == COMPACT_CMA)
981	SetPageCmaAllocating(page);
982	#endif
983	cc->nr_migratepages++;
984	nr_isolated++;
985
986	/*
987	* Record where we could have freed pages by migration and not
988	* yet flushed them to buddy allocator.
989	* - this is the lowest page that was isolated and likely be
990	* then freed by migration.
991	*/
992	if (!cc->last_migrated_pfn)
993	cc->last_migrated_pfn = low_pfn;
994
995	/* Avoid isolating too much */
996	#ifdef CONFIG_AMLOGIC_CMA
997	/* for cma, try to isolate more pages each time */
998	if (cc->page_type != COMPACT_CMA &&
999	cc->nr_migratepages == COMPACT_CLUSTER_MAX) {
1000	#else
1001	if (cc->nr_migratepages == COMPACT_CLUSTER_MAX) {
1002	#endif /* CONFIG_AMLOGIC_CMA */
1003	++low_pfn;
1004	break;
1005	}
1006
1007	continue;
1008	isolate_fail:
1009	if (!skip_on_failure)
1010	continue;
1011
1012	/*
1013	* We have isolated some pages, but then failed. Release them
1014	* instead of migrating, as we cannot form the cc->order buddy
1015	* page anyway.
1016	*/
1017	if (nr_isolated) {
1018	if (locked) {
1019	spin_unlock_irqrestore(zone_lru_lock(zone), flags);
1020	locked = false;
1021	}
1022	putback_movable_pages(&cc->migratepages);
1023	cc->nr_migratepages = 0;
1024	cc->last_migrated_pfn = 0;
1025	nr_isolated = 0;
1026	}
1027
1028	if (low_pfn < next_skip_pfn) {
1029	low_pfn = next_skip_pfn - 1;
1030	/*
1031	* The check near the loop beginning would have updated
1032	* next_skip_pfn too, but this is a bit simpler.
1033	*/
1034	next_skip_pfn += 1UL << cc->order;
1035	}
1036	}
1037
1038	/*
1039	* The PageBuddy() check could have potentially brought us outside
1040	* the range to be scanned.
1041	*/
1042	if (unlikely(low_pfn > end_pfn))
1043	low_pfn = end_pfn;
1044
1045	if (locked)
1046	spin_unlock_irqrestore(zone_lru_lock(zone), flags);
1047
1048	/*
1049	* Update the pageblock-skip information and cached scanner pfn,
1050	* if the whole pageblock was scanned without isolating any page.
1051	*/
1052	if (low_pfn == end_pfn)
1053	update_pageblock_skip(cc, valid_page, nr_isolated, true);
1054
1055	trace_mm_compaction_isolate_migratepages(start_pfn, low_pfn,
1056	nr_scanned, nr_isolated);
1057
1058	count_compact_events(COMPACTMIGRATE_SCANNED, nr_scanned);
1059	if (nr_isolated)
1060	count_compact_events(COMPACTISOLATED, nr_isolated);
1061
1062	return low_pfn;
1063	}
1064
1065	/**
1066	* isolate_migratepages_range() - isolate migrate-able pages in a PFN range
1067	* @cc: Compaction control structure.
1068	* @start_pfn: The first PFN to start isolating.
1069	* @end_pfn: The one-past-last PFN.
1070	*
1071	* Returns zero if isolation fails fatally due to e.g. pending signal.
1072	* Otherwise, function returns one-past-the-last PFN of isolated page
1073	* (which may be greater than end_pfn if end fell in a middle of a THP page).
1074	*/
1075	unsigned long
1076	isolate_migratepages_range(struct compact_control *cc, unsigned long start_pfn,
1077	unsigned long end_pfn)
1078	{
1079	unsigned long pfn, block_start_pfn, block_end_pfn;
1080
1081	/* Scan block by block. First and last block may be incomplete */
1082	pfn = start_pfn;
1083	block_start_pfn = pageblock_start_pfn(pfn);
1084	if (block_start_pfn < cc->zone->zone_start_pfn)
1085	block_start_pfn = cc->zone->zone_start_pfn;
1086	block_end_pfn = pageblock_end_pfn(pfn);
1087
1088	for (; pfn < end_pfn; pfn = block_end_pfn,
1089	block_start_pfn = block_end_pfn,
1090	block_end_pfn += pageblock_nr_pages) {
1091
1092	block_end_pfn = min(block_end_pfn, end_pfn);
1093
1094	if (!pageblock_pfn_to_page(block_start_pfn,
1095	block_end_pfn, cc->zone))
1096	continue;
1097
1098	pfn = isolate_migratepages_block(cc, pfn, block_end_pfn,
1099	ISOLATE_UNEVICTABLE);
1100
1101	if (!pfn)
1102	break;
1103
1104	if (cc->nr_migratepages == COMPACT_CLUSTER_MAX)
1105	break;
1106	}
1107
1108	return pfn;
1109	}
1110
1111	#endif /* CONFIG_COMPACTION || CONFIG_CMA */
1112	#ifdef CONFIG_COMPACTION
1113
1114	/* Returns true if the page is within a block suitable for migration to */
1115	static bool suitable_migration_target(struct compact_control *cc,
1116	struct page *page)
1117	{
1118	if (cc->ignore_block_suitable)
1119	return true;
1120
1121	/* If the page is a large free page, then disallow migration */
1122	if (PageBuddy(page)) {
1123	/*
1124	* We are checking page_order without zone->lock taken. But
1125	* the only small danger is that we skip a potentially suitable
1126	* pageblock, so it's not worth to check order for valid range.
1127	*/
1128	if (page_order_unsafe(page) >= pageblock_order)
1129	return false;
1130	}
1131
1132	/* If the block is MIGRATE_MOVABLE or MIGRATE_CMA, allow migration */
1133	if (migrate_async_suitable(get_pageblock_migratetype(page)))
1134	return true;
1135
1136	/* Otherwise skip the block */
1137	return false;
1138	}
1139
1140	/*
1141	* Test whether the free scanner has reached the same or lower pageblock than
1142	* the migration scanner, and compaction should thus terminate.
1143	*/
1144	static inline bool compact_scanners_met(struct compact_control *cc)
1145	{
1146	return (cc->free_pfn >> pageblock_order)
1147	<= (cc->migrate_pfn >> pageblock_order);
1148	}
1149
1150	/*
1151	* Based on information in the current compact_control, find blocks
1152	* suitable for isolating free pages from and then isolate them.
1153	*/
1154	static void isolate_freepages(struct compact_control *cc)
1155	{
1156	struct zone *zone = cc->zone;
1157	struct page *page;
1158	unsigned long block_start_pfn; /* start of current pageblock */
1159	unsigned long isolate_start_pfn; /* exact pfn we start at */
1160	unsigned long block_end_pfn; /* end of current pageblock */
1161	unsigned long low_pfn; /* lowest pfn scanner is able to scan */
1162	struct list_head *freelist = &cc->freepages;
1163	#ifdef CONFIG_AMLOGIC_CMA
1164	int migrate_type;
1165	#endif /* CONFIG_AMLOGIC_CMA */
1166
1167	/*
1168	* Initialise the free scanner. The starting point is where we last
1169	* successfully isolated from, zone-cached value, or the end of the
1170	* zone when isolating for the first time. For looping we also need
1171	* this pfn aligned down to the pageblock boundary, because we do
1172	* block_start_pfn -= pageblock_nr_pages in the for loop.
1173	* For ending point, take care when isolating in last pageblock of a
1174	* a zone which ends in the middle of a pageblock.
1175	* The low boundary is the end of the pageblock the migration scanner
1176	* is using.
1177	*/
1178	isolate_start_pfn = cc->free_pfn;
1179	block_start_pfn = pageblock_start_pfn(cc->free_pfn);
1180	block_end_pfn = min(block_start_pfn + pageblock_nr_pages,
1181	zone_end_pfn(zone));
1182	low_pfn = pageblock_end_pfn(cc->migrate_pfn);
1183
1184	/*
1185	* Isolate free pages until enough are available to migrate the
1186	* pages on cc->migratepages. We stop searching if the migrate
1187	* and free page scanners meet or enough free pages are isolated.
1188	*/
1189	for (; block_start_pfn >= low_pfn;
1190	block_end_pfn = block_start_pfn,
1191	block_start_pfn -= pageblock_nr_pages,
1192	isolate_start_pfn = block_start_pfn) {
1193	/*
1194	* This can iterate a massively long zone without finding any
1195	* suitable migration targets, so periodically check if we need
1196	* to schedule, or even abort async compaction.
1197	*/
1198	if (!(block_start_pfn % (SWAP_CLUSTER_MAX * pageblock_nr_pages))
1199	&& compact_should_abort(cc))
1200	break;
1201
1202	page = pageblock_pfn_to_page(block_start_pfn, block_end_pfn,
1203	zone);
1204	if (!page)
1205	continue;
1206
1207	/* Check the block is suitable for migration */
1208	if (!suitable_migration_target(cc, page))
1209	continue;
1210
1211	/* If isolation recently failed, do not retry */
1212	if (!isolation_suitable(cc, page))
1213	continue;
1214
1215	#ifdef CONFIG_AMLOGIC_CMA
1216	migrate_type = get_pageblock_migratetype(page);
1217	if (is_migrate_isolate(migrate_type))
1218	continue;
1219	if (is_migrate_cma(migrate_type) && cc->forbid_to_cma)
1220	continue;
1221	#endif /* CONFIG_AMLOGIC_CMA */
1222	/* Found a block suitable for isolating free pages from. */
1223	isolate_freepages_block(cc, &isolate_start_pfn, block_end_pfn,
1224	freelist, false);
1225
1226	/*
1227	* If we isolated enough freepages, or aborted due to lock
1228	* contention, terminate.
1229	*/
1230	if ((cc->nr_freepages >= cc->nr_migratepages)
1231	|| cc->contended) {
1232	if (isolate_start_pfn >= block_end_pfn) {
1233	/*
1234	* Restart at previous pageblock if more
1235	* freepages can be isolated next time.
1236	*/
1237	isolate_start_pfn =
1238	block_start_pfn - pageblock_nr_pages;
1239	}
1240	break;
1241	} else if (isolate_start_pfn < block_end_pfn) {
1242	/*
1243	* If isolation failed early, do not continue
1244	* needlessly.
1245	*/
1246	break;
1247	}
1248	}
1249
1250	/* __isolate_free_page() does not map the pages */
1251	map_pages(freelist);
1252
1253	/*
1254	* Record where the free scanner will restart next time. Either we
1255	* broke from the loop and set isolate_start_pfn based on the last
1256	* call to isolate_freepages_block(), or we met the migration scanner
1257	* and the loop terminated due to isolate_start_pfn < low_pfn
1258	*/
1259	cc->free_pfn = isolate_start_pfn;
1260	}
1261
1262	/*
1263	* This is a migrate-callback that "allocates" freepages by taking pages
1264	* from the isolated freelists in the block we are migrating to.
1265	*/
1266	static struct page *compaction_alloc(struct page *migratepage,
1267	unsigned long data,
1268	int **result)
1269	{
1270	struct compact_control *cc = (struct compact_control *)data;
1271	struct page *freepage;
1272	#ifdef CONFIG_AMLOGIC_PAGE_TRACE
1273	struct page_trace *old_trace, *new_trace;
1274	#endif
1275
1276	/*
1277	* Isolate free pages if necessary, and if we are not aborting due to
1278	* contention.
1279	*/
1280	if (list_empty(&cc->freepages)) {
1281	if (!cc->contended)
1282	isolate_freepages(cc);
1283
1284	if (list_empty(&cc->freepages))
1285	return NULL;
1286	}
1287
1288	freepage = list_entry(cc->freepages.next, struct page, lru);
1289	list_del(&freepage->lru);
1290	cc->nr_freepages--;
1291	#ifdef CONFIG_AMLOGIC_PAGE_TRACE
1292	if (freepage) {
1293	old_trace = find_page_base(migratepage);
1294	new_trace = find_page_base(freepage);
1295	*new_trace = *old_trace;
1296	}
1297	#endif
1298
1299	return freepage;
1300	}
1301
1302	/*
1303	* This is a migrate-callback that "frees" freepages back to the isolated
1304	* freelist. All pages on the freelist are from the same zone, so there is no
1305	* special handling needed for NUMA.
1306	*/
1307	static void compaction_free(struct page *page, unsigned long data)
1308	{
1309	struct compact_control *cc = (struct compact_control *)data;
1310
1311	list_add(&page->lru, &cc->freepages);
1312	cc->nr_freepages++;
1313	}
1314
1315	/* possible outcome of isolate_migratepages */
1316	typedef enum {
1317	ISOLATE_ABORT, /* Abort compaction now */
1318	ISOLATE_NONE, /* No pages isolated, continue scanning */
1319	ISOLATE_SUCCESS, /* Pages isolated, migrate */
1320	} isolate_migrate_t;
1321
1322	/*
1323	* Allow userspace to control policy on scanning the unevictable LRU for
1324	* compactable pages.
1325	*/
1326	int sysctl_compact_unevictable_allowed __read_mostly = 1;
1327
1328	/*
1329	* Isolate all pages that can be migrated from the first suitable block,
1330	* starting at the block pointed to by the migrate scanner pfn within
1331	* compact_control.
1332	*/
1333	static isolate_migrate_t isolate_migratepages(struct zone *zone,
1334	struct compact_control *cc)
1335	{
1336	unsigned long block_start_pfn;
1337	unsigned long block_end_pfn;
1338	unsigned long low_pfn;
1339	struct page *page;
1340	const isolate_mode_t isolate_mode =
1341	(sysctl_compact_unevictable_allowed ? ISOLATE_UNEVICTABLE : 0) |
1342	(cc->mode != MIGRATE_SYNC ? ISOLATE_ASYNC_MIGRATE : 0);
1343
1344	/*
1345	* Start at where we last stopped, or beginning of the zone as
1346	* initialized by compact_zone()
1347	*/
1348	low_pfn = cc->migrate_pfn;
1349	block_start_pfn = pageblock_start_pfn(low_pfn);
1350	if (block_start_pfn < zone->zone_start_pfn)
1351	block_start_pfn = zone->zone_start_pfn;
1352
1353	/* Only scan within a pageblock boundary */
1354	block_end_pfn = pageblock_end_pfn(low_pfn);
1355
1356	/*
1357	* Iterate over whole pageblocks until we find the first suitable.
1358	* Do not cross the free scanner.
1359	*/
1360	for (; block_end_pfn <= cc->free_pfn;
1361	low_pfn = block_end_pfn,
1362	block_start_pfn = block_end_pfn,
1363	block_end_pfn += pageblock_nr_pages) {
1364
1365	/*
1366	* This can potentially iterate a massively long zone with
1367	* many pageblocks unsuitable, so periodically check if we
1368	* need to schedule, or even abort async compaction.
1369	*/
1370	if (!(low_pfn % (SWAP_CLUSTER_MAX * pageblock_nr_pages))
1371	&& compact_should_abort(cc))
1372	break;
1373
1374	page = pageblock_pfn_to_page(block_start_pfn, block_end_pfn,
1375	zone);
1376	if (!page)
1377	continue;
1378
1379	/* If isolation recently failed, do not retry */
1380	if (!isolation_suitable(cc, page))
1381	continue;
1382
1383	/*
1384	* For async compaction, also only scan in MOVABLE blocks.
1385	* Async compaction is optimistic to see if the minimum amount
1386	* of work satisfies the allocation.
1387	*/
1388	if (cc->mode == MIGRATE_ASYNC &&
1389	!migrate_async_suitable(get_pageblock_migratetype(page)))
1390	continue;
1391
1392	/* Perform the isolation */
1393	low_pfn = isolate_migratepages_block(cc, low_pfn,
1394	block_end_pfn, isolate_mode);
1395
1396	if (!low_pfn || cc->contended)
1397	return ISOLATE_ABORT;
1398
1399	/*
1400	* Either we isolated something and proceed with migration. Or
1401	* we failed and compact_zone should decide if we should
1402	* continue or not.
1403	*/
1404	break;
1405	}
1406
1407	/* Record where migration scanner will be restarted. */
1408	cc->migrate_pfn = low_pfn;
1409
1410	return cc->nr_migratepages ? ISOLATE_SUCCESS : ISOLATE_NONE;
1411	}
1412
1413	/*
1414	* order == -1 is expected when compacting via
1415	* /proc/sys/vm/compact_memory
1416	*/
1417	static inline bool is_via_compact_memory(int order)
1418	{
1419	return order == -1;
1420	}
1421
1422	static enum compact_result __compact_finished(struct zone *zone, struct compact_control *cc,
1423	const int migratetype)
1424	{
1425	unsigned int order;
1426	unsigned long watermark;
1427
1428	if (cc->contended || fatal_signal_pending(current))
1429	return COMPACT_CONTENDED;
1430
1431	/* Compaction run completes if the migrate and free scanner meet */
1432	if (compact_scanners_met(cc)) {
1433	/* Let the next compaction start anew. */
1434	reset_cached_positions(zone);
1435
1436	/*
1437	* Mark that the PG_migrate_skip information should be cleared
1438	* by kswapd when it goes to sleep. kcompactd does not set the
1439	* flag itself as the decision to be clear should be directly
1440	* based on an allocation request.
1441	*/
1442	if (cc->direct_compaction)
1443	zone->compact_blockskip_flush = true;
1444
1445	if (cc->whole_zone)
1446	return COMPACT_COMPLETE;
1447	else
1448	return COMPACT_PARTIAL_SKIPPED;
1449	}
1450
1451	if (is_via_compact_memory(cc->order))
1452	return COMPACT_CONTINUE;
1453
1454	/* Compaction run is not finished if the watermark is not met */
1455	watermark = zone->watermark[cc->alloc_flags & ALLOC_WMARK_MASK];
1456
1457	if (!zone_watermark_ok(zone, cc->order, watermark, cc->classzone_idx,
1458	cc->alloc_flags))
1459	return COMPACT_CONTINUE;
1460
1461	/* Direct compactor: Is a suitable page free? */
1462	for (order = cc->order; order < MAX_ORDER; order++) {
1463	struct free_area *area = &zone->free_area[order];
1464	bool can_steal;
1465
1466	/* Job done if page is free of the right migratetype */
1467	if (!list_empty(&area->free_list[migratetype]))
1468	return COMPACT_SUCCESS;
1469
1470	#ifdef CONFIG_CMA
1471	/* MIGRATE_MOVABLE can fallback on MIGRATE_CMA */
1472	if (migratetype == MIGRATE_MOVABLE &&
1473	!list_empty(&area->free_list[MIGRATE_CMA]))
1474	return COMPACT_SUCCESS;
1475	#endif
1476	/*
1477	* Job done if allocation would steal freepages from
1478	* other migratetype buddy lists.
1479	*/
1480	if (find_suitable_fallback(area, order, migratetype,
1481	true, &can_steal) != -1)
1482	return COMPACT_SUCCESS;
1483	}
1484
1485	return COMPACT_NO_SUITABLE_PAGE;
1486	}
1487
1488	static enum compact_result compact_finished(struct zone *zone,
1489	struct compact_control *cc,
1490	const int migratetype)
1491	{
1492	int ret;
1493
1494	ret = __compact_finished(zone, cc, migratetype);
1495	trace_mm_compaction_finished(zone, cc->order, ret);
1496	if (ret == COMPACT_NO_SUITABLE_PAGE)
1497	ret = COMPACT_CONTINUE;
1498
1499	return ret;
1500	}
1501
1502	/*
1503	* compaction_suitable: Is this suitable to run compaction on this zone now?
1504	* Returns
1505	* COMPACT_SKIPPED - If there are too few free pages for compaction
1506	* COMPACT_SUCCESS - If the allocation would succeed without compaction
1507	* COMPACT_CONTINUE - If compaction should run now
1508	*/
1509	static enum compact_result __compaction_suitable(struct zone *zone, int order,
1510	unsigned int alloc_flags,
1511	int classzone_idx,
1512	unsigned long wmark_target)
1513	{
1514	unsigned long watermark;
1515
1516	if (is_via_compact_memory(order))
1517	return COMPACT_CONTINUE;
1518
1519	watermark = zone->watermark[alloc_flags & ALLOC_WMARK_MASK];
1520	/*
1521	* If watermarks for high-order allocation are already met, there
1522	* should be no need for compaction at all.
1523	*/
1524	if (zone_watermark_ok(zone, order, watermark, classzone_idx,
1525	alloc_flags))
1526	return COMPACT_SUCCESS;
1527
1528	/*
1529	* Watermarks for order-0 must be met for compaction to be able to
1530	* isolate free pages for migration targets. This means that the
1531	* watermark and alloc_flags have to match, or be more pessimistic than
1532	* the check in __isolate_free_page(). We don't use the direct
1533	* compactor's alloc_flags, as they are not relevant for freepage
1534	* isolation. We however do use the direct compactor's classzone_idx to
1535	* skip over zones where lowmem reserves would prevent allocation even
1536	* if compaction succeeds.
1537	* For costly orders, we require low watermark instead of min for
1538	* compaction to proceed to increase its chances.
1539	* ALLOC_CMA is used, as pages in CMA pageblocks are considered
1540	* suitable migration targets
1541	*/
1542	watermark = (order > PAGE_ALLOC_COSTLY_ORDER) ?
1543	low_wmark_pages(zone) : min_wmark_pages(zone);
1544	watermark += compact_gap(order);
1545	if (!__zone_watermark_ok(zone, 0, watermark, classzone_idx,
1546	ALLOC_CMA, wmark_target))
1547	return COMPACT_SKIPPED;
1548
1549	return COMPACT_CONTINUE;
1550	}
1551
1552	enum compact_result compaction_suitable(struct zone *zone, int order,
1553	unsigned int alloc_flags,
1554	int classzone_idx)
1555	{
1556	enum compact_result ret;
1557	int fragindex;
1558
1559	ret = __compaction_suitable(zone, order, alloc_flags, classzone_idx,
1560	zone_page_state(zone, NR_FREE_PAGES));
1561	/*
1562	* fragmentation index determines if allocation failures are due to
1563	* low memory or external fragmentation
1564	*
1565	* index of -1000 would imply allocations might succeed depending on
1566	* watermarks, but we already failed the high-order watermark check
1567	* index towards 0 implies failure is due to lack of memory
1568	* index towards 1000 implies failure is due to fragmentation
1569	*
1570	* Only compact if a failure would be due to fragmentation. Also
1571	* ignore fragindex for non-costly orders where the alternative to
1572	* a successful reclaim/compaction is OOM. Fragindex and the
1573	* vm.extfrag_threshold sysctl is meant as a heuristic to prevent
1574	* excessive compaction for costly orders, but it should not be at the
1575	* expense of system stability.
1576	*/
1577	if (ret == COMPACT_CONTINUE && (order > PAGE_ALLOC_COSTLY_ORDER)) {
1578	fragindex = fragmentation_index(zone, order);
1579	if (fragindex >= 0 && fragindex <= sysctl_extfrag_threshold)
1580	ret = COMPACT_NOT_SUITABLE_ZONE;
1581	}
1582
1583	trace_mm_compaction_suitable(zone, order, ret);
1584	if (ret == COMPACT_NOT_SUITABLE_ZONE)
1585	ret = COMPACT_SKIPPED;
1586
1587	return ret;
1588	}
1589
1590	bool compaction_zonelist_suitable(struct alloc_context *ac, int order,
1591	int alloc_flags)
1592	{
1593	struct zone *zone;
1594	struct zoneref *z;
1595
1596	/*
1597	* Make sure at least one zone would pass __compaction_suitable if we continue
1598	* retrying the reclaim.
1599	*/
1600	for_each_zone_zonelist_nodemask(zone, z, ac->zonelist, ac->high_zoneidx,
1601	ac->nodemask) {
1602	unsigned long available;
1603	enum compact_result compact_result;
1604
1605	/*
1606	* Do not consider all the reclaimable memory because we do not
1607	* want to trash just for a single high order allocation which
1608	* is even not guaranteed to appear even if __compaction_suitable
1609	* is happy about the watermark check.
1610	*/
1611	available = zone_reclaimable_pages(zone) / order;
1612	available += zone_page_state_snapshot(zone, NR_FREE_PAGES);
1613	compact_result = __compaction_suitable(zone, order, alloc_flags,
1614	ac_classzone_idx(ac), available);
1615	if (compact_result != COMPACT_SKIPPED)
1616	return true;
1617	}
1618
1619	return false;
1620	}
1621
1622	static enum compact_result compact_zone(struct zone *zone, struct compact_control *cc)
1623	{
1624	enum compact_result ret;
1625	unsigned long start_pfn = zone->zone_start_pfn;
1626	unsigned long end_pfn = zone_end_pfn(zone);
1627	const int migratetype = gfpflags_to_migratetype(cc->gfp_mask);
1628	const bool sync = cc->mode != MIGRATE_ASYNC;
1629
1630	ret = compaction_suitable(zone, cc->order, cc->alloc_flags,
1631	cc->classzone_idx);
1632	/* Compaction is likely to fail */
1633	if (ret == COMPACT_SUCCESS || ret == COMPACT_SKIPPED)
1634	return ret;
1635
1636	/* huh, compaction_suitable is returning something unexpected */
1637	VM_BUG_ON(ret != COMPACT_CONTINUE);
1638
1639	/*
1640	* Clear pageblock skip if there were failures recently and compaction
1641	* is about to be retried after being deferred.
1642	*/
1643	if (compaction_restarting(zone, cc->order))
1644	__reset_isolation_suitable(zone);
1645
1646	/*
1647	* Setup to move all movable pages to the end of the zone. Used cached
1648	* information on where the scanners should start (unless we explicitly
1649	* want to compact the whole zone), but check that it is initialised
1650	* by ensuring the values are within zone boundaries.
1651	*/
1652	if (cc->whole_zone) {
1653	cc->migrate_pfn = start_pfn;
1654	cc->free_pfn = pageblock_start_pfn(end_pfn - 1);
1655	} else {
1656	cc->migrate_pfn = zone->compact_cached_migrate_pfn[sync];
1657	cc->free_pfn = zone->compact_cached_free_pfn;
1658	if (cc->free_pfn < start_pfn || cc->free_pfn >= end_pfn) {
1659	cc->free_pfn = pageblock_start_pfn(end_pfn - 1);
1660	zone->compact_cached_free_pfn = cc->free_pfn;
1661	}
1662	if (cc->migrate_pfn < start_pfn || cc->migrate_pfn >= end_pfn) {
1663	cc->migrate_pfn = start_pfn;
1664	zone->compact_cached_migrate_pfn[0] = cc->migrate_pfn;
1665	zone->compact_cached_migrate_pfn[1] = cc->migrate_pfn;
1666	}
1667
1668	if (cc->migrate_pfn == start_pfn)
1669	cc->whole_zone = true;
1670	}
1671
1672	cc->last_migrated_pfn = 0;
1673
1674	trace_mm_compaction_begin(start_pfn, cc->migrate_pfn,
1675	cc->free_pfn, end_pfn, sync);
1676
1677	migrate_prep_local();
1678
1679	#ifdef CONFIG_AMLOGIC_CMA
1680	cc->forbid_to_cma = false;
1681	#endif
1682	while ((ret = compact_finished(zone, cc, migratetype)) ==
1683	COMPACT_CONTINUE) {
1684	int err;
1685
1686	switch (isolate_migratepages(zone, cc)) {
1687	case ISOLATE_ABORT:
1688	ret = COMPACT_CONTENDED;
1689	putback_movable_pages(&cc->migratepages);
1690	cc->nr_migratepages = 0;
1691	goto out;
1692	case ISOLATE_NONE:
1693	/*
1694	* We haven't isolated and migrated anything, but
1695	* there might still be unflushed migrations from
1696	* previous cc->order aligned block.
1697	*/
1698	goto check_drain;
1699	case ISOLATE_SUCCESS:
1700	;
1701	}
1702
1703	err = migrate_pages(&cc->migratepages, compaction_alloc,
1704	compaction_free, (unsigned long)cc, cc->mode,
1705	MR_COMPACTION);
1706
1707	trace_mm_compaction_migratepages(cc->nr_migratepages, err,
1708	&cc->migratepages);
1709
1710	/* All pages were either migrated or will be released */
1711	cc->nr_migratepages = 0;
1712	if (err) {
1713	putback_movable_pages(&cc->migratepages);
1714	/*
1715	* migrate_pages() may return -ENOMEM when scanners meet
1716	* and we want compact_finished() to detect it
1717	*/
1718	if (err == -ENOMEM && !compact_scanners_met(cc)) {
1719	ret = COMPACT_CONTENDED;
1720	goto out;
1721	}
1722	/*
1723	* We failed to migrate at least one page in the current
1724	* order-aligned block, so skip the rest of it.
1725	*/
1726	if (cc->direct_compaction &&
1727	(cc->mode == MIGRATE_ASYNC)) {
1728	cc->migrate_pfn = block_end_pfn(
1729	cc->migrate_pfn - 1, cc->order);
1730	/* Draining pcplists is useless in this case */
1731	cc->last_migrated_pfn = 0;
1732
1733	}
1734	}
1735
1736	check_drain:
1737	/*
1738	* Has the migration scanner moved away from the previous
1739	* cc->order aligned block where we migrated from? If yes,
1740	* flush the pages that were freed, so that they can merge and
1741	* compact_finished() can detect immediately if allocation
1742	* would succeed.
1743	*/
1744	if (cc->order > 0 && cc->last_migrated_pfn) {
1745	int cpu;
1746	unsigned long current_block_start =
1747	block_start_pfn(cc->migrate_pfn, cc->order);
1748
1749	if (cc->last_migrated_pfn < current_block_start) {
1750	cpu = get_cpu();
1751	lru_add_drain_cpu(cpu);
1752	drain_local_pages(zone);
1753	put_cpu();
1754	/* No more flushing until we migrate again */
1755	cc->last_migrated_pfn = 0;
1756	}
1757	}
1758
1759	}
1760
1761	out:
1762	/*
1763	* Release free pages and update where the free scanner should restart,
1764	* so we don't leave any returned pages behind in the next attempt.
1765	*/
1766	if (cc->nr_freepages > 0) {
1767	unsigned long free_pfn = release_freepages(&cc->freepages);
1768
1769	cc->nr_freepages = 0;
1770	VM_BUG_ON(free_pfn == 0);
1771	/* The cached pfn is always the first in a pageblock */
1772	free_pfn = pageblock_start_pfn(free_pfn);
1773	/*
1774	* Only go back, not forward. The cached pfn might have been
1775	* already reset to zone end in compact_finished()
1776	*/
1777	if (free_pfn > zone->compact_cached_free_pfn)
1778	zone->compact_cached_free_pfn = free_pfn;
1779	}
1780
1781	trace_mm_compaction_end(start_pfn, cc->migrate_pfn,
1782	cc->free_pfn, end_pfn, sync, ret);
1783
1784	return ret;
1785	}
1786
1787	static enum compact_result compact_zone_order(struct zone *zone, int order,
1788	gfp_t gfp_mask, enum compact_priority prio,
1789	unsigned int alloc_flags, int classzone_idx)
1790	{
1791	enum compact_result ret;
1792	struct compact_control cc = {
1793	.nr_freepages = 0,
1794	.nr_migratepages = 0,
1795	.order = order,
1796	.gfp_mask = gfp_mask,
1797	.zone = zone,
1798	.mode = (prio == COMPACT_PRIO_ASYNC) ?
1799	MIGRATE_ASYNC : MIGRATE_SYNC_LIGHT,
1800	.alloc_flags = alloc_flags,
1801	.classzone_idx = classzone_idx,
1802	.direct_compaction = true,
1803	.whole_zone = (prio == MIN_COMPACT_PRIORITY),
1804	.ignore_skip_hint = (prio == MIN_COMPACT_PRIORITY),
1805	.ignore_block_suitable = (prio == MIN_COMPACT_PRIORITY)
1806	};
1807	INIT_LIST_HEAD(&cc.freepages);
1808	INIT_LIST_HEAD(&cc.migratepages);
1809
1810	ret = compact_zone(zone, &cc);
1811
1812	VM_BUG_ON(!list_empty(&cc.freepages));
1813	VM_BUG_ON(!list_empty(&cc.migratepages));
1814
1815	return ret;
1816	}
1817
1818	int sysctl_extfrag_threshold = 500;
1819
1820	/**
1821	* try_to_compact_pages - Direct compact to satisfy a high-order allocation
1822	* @gfp_mask: The GFP mask of the current allocation
1823	* @order: The order of the current allocation
1824	* @alloc_flags: The allocation flags of the current allocation
1825	* @ac: The context of current allocation
1826	* @mode: The migration mode for async, sync light, or sync migration
1827	*
1828	* This is the main entry point for direct page compaction.
1829	*/
1830	enum compact_result try_to_compact_pages(gfp_t gfp_mask, unsigned int order,
1831	unsigned int alloc_flags, const struct alloc_context *ac,
1832	enum compact_priority prio)
1833	{
1834	int may_enter_fs = gfp_mask & __GFP_FS;
1835	int may_perform_io = gfp_mask & __GFP_IO;
1836	struct zoneref *z;
1837	struct zone *zone;
1838	enum compact_result rc = COMPACT_SKIPPED;
1839
1840	/* Check if the GFP flags allow compaction */
1841	if (!may_enter_fs || !may_perform_io)
1842	return COMPACT_SKIPPED;
1843
1844	trace_mm_compaction_try_to_compact_pages(order, gfp_mask, prio);
1845
1846	/* Compact each zone in the list */
1847	for_each_zone_zonelist_nodemask(zone, z, ac->zonelist, ac->high_zoneidx,
1848	ac->nodemask) {
1849	enum compact_result status;
1850
1851	if (prio > MIN_COMPACT_PRIORITY
1852	&& compaction_deferred(zone, order)) {
1853	rc = max_t(enum compact_result, COMPACT_DEFERRED, rc);
1854	continue;
1855	}
1856
1857	status = compact_zone_order(zone, order, gfp_mask, prio,
1858	alloc_flags, ac_classzone_idx(ac));
1859	rc = max(status, rc);
1860
1861	/* The allocation should succeed, stop compacting */
1862	if (status == COMPACT_SUCCESS) {
1863	/*
1864	* We think the allocation will succeed in this zone,
1865	* but it is not certain, hence the false. The caller
1866	* will repeat this with true if allocation indeed
1867	* succeeds in this zone.
1868	*/
1869	compaction_defer_reset(zone, order, false);
1870
1871	break;
1872	}
1873
1874	if (prio != COMPACT_PRIO_ASYNC && (status == COMPACT_COMPLETE ||
1875	status == COMPACT_PARTIAL_SKIPPED))
1876	/*
1877	* We think that allocation won't succeed in this zone
1878	* so we defer compaction there. If it ends up
1879	* succeeding after all, it will be reset.
1880	*/
1881	defer_compaction(zone, order);
1882
1883	/*
1884	* We might have stopped compacting due to need_resched() in
1885	* async compaction, or due to a fatal signal detected. In that
1886	* case do not try further zones
1887	*/
1888	if ((prio == COMPACT_PRIO_ASYNC && need_resched())
1889	|| fatal_signal_pending(current))
1890	break;
1891	}
1892
1893	return rc;
1894	}
1895
1896
1897	/* Compact all zones within a node */
1898	static void compact_node(int nid)
1899	{
1900	pg_data_t *pgdat = NODE_DATA(nid);
1901	int zoneid;
1902	struct zone *zone;
1903	struct compact_control cc = {
1904	.order = -1,
1905	.mode = MIGRATE_SYNC,
1906	.ignore_skip_hint = true,
1907	.whole_zone = true,
1908	};
1909
1910
1911	for (zoneid = 0; zoneid < MAX_NR_ZONES; zoneid++) {
1912
1913	zone = &pgdat->node_zones[zoneid];
1914	if (!populated_zone(zone))
1915	continue;
1916
1917	cc.nr_freepages = 0;
1918	cc.nr_migratepages = 0;
1919	cc.zone = zone;
1920	INIT_LIST_HEAD(&cc.freepages);
1921	INIT_LIST_HEAD(&cc.migratepages);
1922
1923	compact_zone(zone, &cc);
1924
1925	VM_BUG_ON(!list_empty(&cc.freepages));
1926	VM_BUG_ON(!list_empty(&cc.migratepages));
1927	}
1928	}
1929
1930	/* Compact all nodes in the system */
1931	static void compact_nodes(void)
1932	{
1933	int nid;
1934
1935	/* Flush pending updates to the LRU lists */
1936	lru_add_drain_all();
1937
1938	for_each_online_node(nid)
1939	compact_node(nid);
1940	}
1941
1942	/* The written value is actually unused, all memory is compacted */
1943	int sysctl_compact_memory;
1944
1945	/*
1946	* This is the entry point for compacting all nodes via
1947	* /proc/sys/vm/compact_memory
1948	*/
1949	int sysctl_compaction_handler(struct ctl_table *table, int write,
1950	void __user *buffer, size_t *length, loff_t *ppos)
1951	{
1952	if (write)
1953	compact_nodes();
1954
1955	return 0;
1956	}
1957
1958	int sysctl_extfrag_handler(struct ctl_table *table, int write,
1959	void __user *buffer, size_t *length, loff_t *ppos)
1960	{
1961	proc_dointvec_minmax(table, write, buffer, length, ppos);
1962
1963	return 0;
1964	}
1965
1966	#if defined(CONFIG_SYSFS) && defined(CONFIG_NUMA)
1967	static ssize_t sysfs_compact_node(struct device *dev,
1968	struct device_attribute *attr,
1969	const char *buf, size_t count)
1970	{
1971	int nid = dev->id;
1972
1973	if (nid >= 0 && nid < nr_node_ids && node_online(nid)) {
1974	/* Flush pending updates to the LRU lists */
1975	lru_add_drain_all();
1976
1977	compact_node(nid);
1978	}
1979
1980	return count;
1981	}
1982	static DEVICE_ATTR(compact, S_IWUSR, NULL, sysfs_compact_node);
1983
1984	int compaction_register_node(struct node *node)
1985	{
1986	return device_create_file(&node->dev, &dev_attr_compact);
1987	}
1988
1989	void compaction_unregister_node(struct node *node)
1990	{
1991	return device_remove_file(&node->dev, &dev_attr_compact);
1992	}
1993	#endif /* CONFIG_SYSFS && CONFIG_NUMA */
1994
1995	static inline bool kcompactd_work_requested(pg_data_t *pgdat)
1996	{
1997	return pgdat->kcompactd_max_order > 0 || kthread_should_stop();
1998	}
1999
2000	static bool kcompactd_node_suitable(pg_data_t *pgdat)
2001	{
2002	int zoneid;
2003	struct zone *zone;
2004	enum zone_type classzone_idx = pgdat->kcompactd_classzone_idx;
2005
2006	for (zoneid = 0; zoneid <= classzone_idx; zoneid++) {
2007	zone = &pgdat->node_zones[zoneid];
2008
2009	if (!populated_zone(zone))
2010	continue;
2011
2012	if (compaction_suitable(zone, pgdat->kcompactd_max_order, 0,
2013	classzone_idx) == COMPACT_CONTINUE)
2014	return true;
2015	}
2016
2017	return false;
2018	}
2019
2020	static void kcompactd_do_work(pg_data_t *pgdat)
2021	{
2022	/*
2023	* With no special task, compact all zones so that a page of requested
2024	* order is allocatable.
2025	*/
2026	int zoneid;
2027	struct zone *zone;
2028	struct compact_control cc = {
2029	.order = pgdat->kcompactd_max_order,
2030	.classzone_idx = pgdat->kcompactd_classzone_idx,
2031	.mode = MIGRATE_SYNC_LIGHT,
2032	.ignore_skip_hint = true,
2033
2034	};
2035	trace_mm_compaction_kcompactd_wake(pgdat->node_id, cc.order,
2036	cc.classzone_idx);
2037	count_vm_event(KCOMPACTD_WAKE);
2038
2039	for (zoneid = 0; zoneid <= cc.classzone_idx; zoneid++) {
2040	int status;
2041
2042	zone = &pgdat->node_zones[zoneid];
2043	if (!populated_zone(zone))
2044	continue;
2045
2046	if (compaction_deferred(zone, cc.order))
2047	continue;
2048
2049	if (compaction_suitable(zone, cc.order, 0, zoneid) !=
2050	COMPACT_CONTINUE)
2051	continue;
2052
2053	cc.nr_freepages = 0;
2054	cc.nr_migratepages = 0;
2055	cc.zone = zone;
2056	INIT_LIST_HEAD(&cc.freepages);
2057	INIT_LIST_HEAD(&cc.migratepages);
2058
2059	if (kthread_should_stop())
2060	return;
2061	status = compact_zone(zone, &cc);
2062
2063	if (status == COMPACT_SUCCESS) {
2064	compaction_defer_reset(zone, cc.order, false);
2065	} else if (status == COMPACT_PARTIAL_SKIPPED || status == COMPACT_COMPLETE) {
2066	/*
2067	* We use sync migration mode here, so we defer like
2068	* sync direct compaction does.
2069	*/
2070	defer_compaction(zone, cc.order);
2071	}
2072
2073	VM_BUG_ON(!list_empty(&cc.freepages));
2074	VM_BUG_ON(!list_empty(&cc.migratepages));
2075	}
2076
2077	/*
2078	* Regardless of success, we are done until woken up next. But remember
2079	* the requested order/classzone_idx in case it was higher/tighter than
2080	* our current ones
2081	*/
2082	if (pgdat->kcompactd_max_order <= cc.order)
2083	pgdat->kcompactd_max_order = 0;
2084	if (pgdat->kcompactd_classzone_idx >= cc.classzone_idx)
2085	pgdat->kcompactd_classzone_idx = pgdat->nr_zones - 1;
2086	}
2087
2088	void wakeup_kcompactd(pg_data_t *pgdat, int order, int classzone_idx)
2089	{
2090	if (!order)
2091	return;
2092
2093	if (pgdat->kcompactd_max_order < order)
2094	pgdat->kcompactd_max_order = order;
2095
2096	if (pgdat->kcompactd_classzone_idx > classzone_idx)
2097	pgdat->kcompactd_classzone_idx = classzone_idx;
2098
2099	if (!waitqueue_active(&pgdat->kcompactd_wait))
2100	return;
2101
2102	if (!kcompactd_node_suitable(pgdat))
2103	return;
2104
2105	trace_mm_compaction_wakeup_kcompactd(pgdat->node_id, order,
2106	classzone_idx);
2107	wake_up_interruptible(&pgdat->kcompactd_wait);
2108	}
2109
2110	/*
2111	* The background compaction daemon, started as a kernel thread
2112	* from the init process.
2113	*/
2114	static int kcompactd(void *p)
2115	{
2116	pg_data_t *pgdat = (pg_data_t*)p;
2117	struct task_struct *tsk = current;
2118
2119	const struct cpumask *cpumask = cpumask_of_node(pgdat->node_id);
2120
2121	if (!cpumask_empty(cpumask))
2122	set_cpus_allowed_ptr(tsk, cpumask);
2123
2124	set_freezable();
2125
2126	pgdat->kcompactd_max_order = 0;
2127	pgdat->kcompactd_classzone_idx = pgdat->nr_zones - 1;
2128
2129	while (!kthread_should_stop()) {
2130	unsigned long pflags;
2131
2132	trace_mm_compaction_kcompactd_sleep(pgdat->node_id);
2133	wait_event_freezable(pgdat->kcompactd_wait,
2134	kcompactd_work_requested(pgdat));
2135
2136	psi_memstall_enter(&pflags);
2137	kcompactd_do_work(pgdat);
2138	psi_memstall_leave(&pflags);
2139	}
2140
2141	return 0;
2142	}
2143
2144	/*
2145	* This kcompactd start function will be called by init and node-hot-add.
2146	* On node-hot-add, kcompactd will moved to proper cpus if cpus are hot-added.
2147	*/
2148	int kcompactd_run(int nid)
2149	{
2150	pg_data_t *pgdat = NODE_DATA(nid);
2151	int ret = 0;
2152
2153	if (pgdat->kcompactd)
2154	return 0;
2155
2156	pgdat->kcompactd = kthread_run(kcompactd, pgdat, "kcompactd%d", nid);
2157	if (IS_ERR(pgdat->kcompactd)) {
2158	pr_err("Failed to start kcompactd on node %d\n", nid);
2159	ret = PTR_ERR(pgdat->kcompactd);
2160	pgdat->kcompactd = NULL;
2161	}
2162	return ret;
2163	}
2164
2165	/*
2166	* Called by memory hotplug when all memory in a node is offlined. Caller must
2167	* hold mem_hotplug_begin/end().
2168	*/
2169	void kcompactd_stop(int nid)
2170	{
2171	struct task_struct *kcompactd = NODE_DATA(nid)->kcompactd;
2172
2173	if (kcompactd) {
2174	kthread_stop(kcompactd);
2175	NODE_DATA(nid)->kcompactd = NULL;
2176	}
2177	}
2178
2179	/*
2180	* It's optimal to keep kcompactd on the same CPUs as their memory, but
2181	* not required for correctness. So if the last cpu in a node goes
2182	* away, we get changed to run anywhere: as the first one comes back,
2183	* restore their cpu bindings.
2184	*/
2185	static int cpu_callback(struct notifier_block *nfb, unsigned long action,
2186	void *hcpu)
2187	{
2188	int nid;
2189
2190	if (action == CPU_ONLINE || action == CPU_ONLINE_FROZEN) {
2191	for_each_node_state(nid, N_MEMORY) {
2192	pg_data_t *pgdat = NODE_DATA(nid);
2193	const struct cpumask *mask;
2194
2195	mask = cpumask_of_node(pgdat->node_id);
2196
2197	if (cpumask_any_and(cpu_online_mask, mask) < nr_cpu_ids)
2198	/* One of our CPUs online: restore mask */
2199	set_cpus_allowed_ptr(pgdat->kcompactd, mask);
2200	}
2201	}
2202	return NOTIFY_OK;
2203	}
2204
2205	static int __init kcompactd_init(void)
2206	{
2207	int nid;
2208
2209	for_each_node_state(nid, N_MEMORY)
2210	kcompactd_run(nid);
2211	hotcpu_notifier(cpu_callback, 0);
2212	return 0;
2213	}
2214	subsys_initcall(kcompactd_init)
2215
2216	#endif /* CONFIG_COMPACTION */
2217