path: root/mm/memcontrol.c (plain)
blob: 86a6b331b96488d05c4bd1b3fa245e0a25a4c1b3

1	/* memcontrol.c - Memory Controller
2	*
3	* Copyright IBM Corporation, 2007
4	* Author Balbir Singh <balbir@linux.vnet.ibm.com>
5	*
6	* Copyright 2007 OpenVZ SWsoft Inc
7	* Author: Pavel Emelianov <xemul@openvz.org>
8	*
9	* Memory thresholds
10	* Copyright (C) 2009 Nokia Corporation
11	* Author: Kirill A. Shutemov
12	*
13	* Kernel Memory Controller
14	* Copyright (C) 2012 Parallels Inc. and Google Inc.
15	* Authors: Glauber Costa and Suleiman Souhlal
16	*
17	* Native page reclaim
18	* Charge lifetime sanitation
19	* Lockless page tracking & accounting
20	* Unified hierarchy configuration model
21	* Copyright (C) 2015 Red Hat, Inc., Johannes Weiner
22	*
23	* This program is free software; you can redistribute it and/or modify
24	* it under the terms of the GNU General Public License as published by
25	* the Free Software Foundation; either version 2 of the License, or
26	* (at your option) any later version.
27	*
28	* This program is distributed in the hope that it will be useful,
29	* but WITHOUT ANY WARRANTY; without even the implied warranty of
30	* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
31	* GNU General Public License for more details.
32	*/
33
34	#include <linux/page_counter.h>
35	#include <linux/memcontrol.h>
36	#include <linux/cgroup.h>
37	#include <linux/mm.h>
38	#include <linux/hugetlb.h>
39	#include <linux/pagemap.h>
40	#include <linux/smp.h>
41	#include <linux/page-flags.h>
42	#include <linux/backing-dev.h>
43	#include <linux/bit_spinlock.h>
44	#include <linux/rcupdate.h>
45	#include <linux/limits.h>
46	#include <linux/export.h>
47	#include <linux/mutex.h>
48	#include <linux/rbtree.h>
49	#include <linux/slab.h>
50	#include <linux/swap.h>
51	#include <linux/swapops.h>
52	#include <linux/spinlock.h>
53	#include <linux/eventfd.h>
54	#include <linux/poll.h>
55	#include <linux/sort.h>
56	#include <linux/fs.h>
57	#include <linux/seq_file.h>
58	#include <linux/vmpressure.h>
59	#include <linux/mm_inline.h>
60	#include <linux/swap_cgroup.h>
61	#include <linux/cpu.h>
62	#include <linux/oom.h>
63	#include <linux/lockdep.h>
64	#include <linux/file.h>
65	#include <linux/tracehook.h>
66	#include "internal.h"
67	#include <net/sock.h>
68	#include <net/ip.h>
69	#include "slab.h"
70
71	#include <asm/uaccess.h>
72
73	#include <trace/events/vmscan.h>
74
75	struct cgroup_subsys memory_cgrp_subsys __read_mostly;
76	EXPORT_SYMBOL(memory_cgrp_subsys);
77
78	struct mem_cgroup *root_mem_cgroup __read_mostly;
79
80	#define MEM_CGROUP_RECLAIM_RETRIES 5
81
82	/* Socket memory accounting disabled? */
83	static bool cgroup_memory_nosocket;
84
85	/* Kernel memory accounting disabled? */
86	static bool cgroup_memory_nokmem;
87
88	/* Whether the swap controller is active */
89	#ifdef CONFIG_MEMCG_SWAP
90	int do_swap_account __read_mostly;
91	#else
92	#define do_swap_account 0
93	#endif
94
95	/* Whether legacy memory+swap accounting is active */
96	static bool do_memsw_account(void)
97	{
98	return !cgroup_subsys_on_dfl(memory_cgrp_subsys) && do_swap_account;
99	}
100
101	static const char * const mem_cgroup_stat_names[] = {
102	"cache",
103	"rss",
104	"rss_huge",
105	"mapped_file",
106	"dirty",
107	"writeback",
108	"swap",
109	};
110
111	static const char * const mem_cgroup_events_names[] = {
112	"pgpgin",
113	"pgpgout",
114	"pgfault",
115	"pgmajfault",
116	};
117
118	static const char * const mem_cgroup_lru_names[] = {
119	"inactive_anon",
120	"active_anon",
121	"inactive_file",
122	"active_file",
123	"unevictable",
124	};
125
126	#define THRESHOLDS_EVENTS_TARGET 128
127	#define SOFTLIMIT_EVENTS_TARGET 1024
128	#define NUMAINFO_EVENTS_TARGET 1024
129
130	/*
131	* Cgroups above their limits are maintained in a RB-Tree, independent of
132	* their hierarchy representation
133	*/
134
135	struct mem_cgroup_tree_per_node {
136	struct rb_root rb_root;
137	spinlock_t lock;
138	};
139
140	struct mem_cgroup_tree {
141	struct mem_cgroup_tree_per_node *rb_tree_per_node[MAX_NUMNODES];
142	};
143
144	static struct mem_cgroup_tree soft_limit_tree __read_mostly;
145
146	/* for OOM */
147	struct mem_cgroup_eventfd_list {
148	struct list_head list;
149	struct eventfd_ctx *eventfd;
150	};
151
152	/*
153	* cgroup_event represents events which userspace want to receive.
154	*/
155	struct mem_cgroup_event {
156	/*
157	* memcg which the event belongs to.
158	*/
159	struct mem_cgroup *memcg;
160	/*
161	* eventfd to signal userspace about the event.
162	*/
163	struct eventfd_ctx *eventfd;
164	/*
165	* Each of these stored in a list by the cgroup.
166	*/
167	struct list_head list;
168	/*
169	* register_event() callback will be used to add new userspace
170	* waiter for changes related to this event. Use eventfd_signal()
171	* on eventfd to send notification to userspace.
172	*/
173	int (*register_event)(struct mem_cgroup *memcg,
174	struct eventfd_ctx *eventfd, const char *args);
175	/*
176	* unregister_event() callback will be called when userspace closes
177	* the eventfd or on cgroup removing. This callback must be set,
178	* if you want provide notification functionality.
179	*/
180	void (*unregister_event)(struct mem_cgroup *memcg,
181	struct eventfd_ctx *eventfd);
182	/*
183	* All fields below needed to unregister event when
184	* userspace closes eventfd.
185	*/
186	poll_table pt;
187	wait_queue_head_t *wqh;
188	wait_queue_t wait;
189	struct work_struct remove;
190	};
191
192	static void mem_cgroup_threshold(struct mem_cgroup *memcg);
193	static void mem_cgroup_oom_notify(struct mem_cgroup *memcg);
194
195	/* Stuffs for move charges at task migration. */
196	/*
197	* Types of charges to be moved.
198	*/
199	#define MOVE_ANON 0x1U
200	#define MOVE_FILE 0x2U
201	#define MOVE_MASK (MOVE_ANON | MOVE_FILE)
202
203	/* "mc" and its members are protected by cgroup_mutex */
204	static struct move_charge_struct {
205	spinlock_t lock; /* for from, to */
206	struct mm_struct *mm;
207	struct mem_cgroup *from;
208	struct mem_cgroup *to;
209	unsigned long flags;
210	unsigned long precharge;
211	unsigned long moved_charge;
212	unsigned long moved_swap;
213	struct task_struct *moving_task; /* a task moving charges */
214	wait_queue_head_t waitq; /* a waitq for other context */
215	} mc = {
216	.lock = __SPIN_LOCK_UNLOCKED(mc.lock),
217	.waitq = __WAIT_QUEUE_HEAD_INITIALIZER(mc.waitq),
218	};
219
220	/*
221	* Maximum loops in mem_cgroup_hierarchical_reclaim(), used for soft
222	* limit reclaim to prevent infinite loops, if they ever occur.
223	*/
224	#define MEM_CGROUP_MAX_RECLAIM_LOOPS 100
225	#define MEM_CGROUP_MAX_SOFT_LIMIT_RECLAIM_LOOPS 2
226
227	enum charge_type {
228	MEM_CGROUP_CHARGE_TYPE_CACHE = 0,
229	MEM_CGROUP_CHARGE_TYPE_ANON,
230	MEM_CGROUP_CHARGE_TYPE_SWAPOUT, /* for accounting swapcache */
231	MEM_CGROUP_CHARGE_TYPE_DROP, /* a page was unused swap cache */
232	NR_CHARGE_TYPE,
233	};
234
235	/* for encoding cft->private value on file */
236	enum res_type {
237	_MEM,
238	_MEMSWAP,
239	_OOM_TYPE,
240	_KMEM,
241	_TCP,
242	};
243
244	#define MEMFILE_PRIVATE(x, val) ((x) << 16 | (val))
245	#define MEMFILE_TYPE(val) ((val) >> 16 & 0xffff)
246	#define MEMFILE_ATTR(val) ((val) & 0xffff)
247	/* Used for OOM nofiier */
248	#define OOM_CONTROL (0)
249
250	/* Some nice accessors for the vmpressure. */
251	struct vmpressure *memcg_to_vmpressure(struct mem_cgroup *memcg)
252	{
253	if (!memcg)
254	memcg = root_mem_cgroup;
255	return &memcg->vmpressure;
256	}
257
258	struct cgroup_subsys_state *vmpressure_to_css(struct vmpressure *vmpr)
259	{
260	return &container_of(vmpr, struct mem_cgroup, vmpressure)->css;
261	}
262
263	static inline bool mem_cgroup_is_root(struct mem_cgroup *memcg)
264	{
265	return (memcg == root_mem_cgroup);
266	}
267
268	#ifndef CONFIG_SLOB
269	/*
270	* This will be the memcg's index in each cache's ->memcg_params.memcg_caches.
271	* The main reason for not using cgroup id for this:
272	* this works better in sparse environments, where we have a lot of memcgs,
273	* but only a few kmem-limited. Or also, if we have, for instance, 200
274	* memcgs, and none but the 200th is kmem-limited, we'd have to have a
275	* 200 entry array for that.
276	*
277	* The current size of the caches array is stored in memcg_nr_cache_ids. It
278	* will double each time we have to increase it.
279	*/
280	static DEFINE_IDA(memcg_cache_ida);
281	int memcg_nr_cache_ids;
282
283	/* Protects memcg_nr_cache_ids */
284	static DECLARE_RWSEM(memcg_cache_ids_sem);
285
286	void memcg_get_cache_ids(void)
287	{
288	down_read(&memcg_cache_ids_sem);
289	}
290
291	void memcg_put_cache_ids(void)
292	{
293	up_read(&memcg_cache_ids_sem);
294	}
295
296	/*
297	* MIN_SIZE is different than 1, because we would like to avoid going through
298	* the alloc/free process all the time. In a small machine, 4 kmem-limited
299	* cgroups is a reasonable guess. In the future, it could be a parameter or
300	* tunable, but that is strictly not necessary.
301	*
302	* MAX_SIZE should be as large as the number of cgrp_ids. Ideally, we could get
303	* this constant directly from cgroup, but it is understandable that this is
304	* better kept as an internal representation in cgroup.c. In any case, the
305	* cgrp_id space is not getting any smaller, and we don't have to necessarily
306	* increase ours as well if it increases.
307	*/
308	#define MEMCG_CACHES_MIN_SIZE 4
309	#define MEMCG_CACHES_MAX_SIZE MEM_CGROUP_ID_MAX
310
311	/*
312	* A lot of the calls to the cache allocation functions are expected to be
313	* inlined by the compiler. Since the calls to memcg_kmem_get_cache are
314	* conditional to this static branch, we'll have to allow modules that does
315	* kmem_cache_alloc and the such to see this symbol as well
316	*/
317	DEFINE_STATIC_KEY_FALSE(memcg_kmem_enabled_key);
318	EXPORT_SYMBOL(memcg_kmem_enabled_key);
319
320	#endif /* !CONFIG_SLOB */
321
322	/**
323	* mem_cgroup_css_from_page - css of the memcg associated with a page
324	* @page: page of interest
325	*
326	* If memcg is bound to the default hierarchy, css of the memcg associated
327	* with @page is returned. The returned css remains associated with @page
328	* until it is released.
329	*
330	* If memcg is bound to a traditional hierarchy, the css of root_mem_cgroup
331	* is returned.
332	*/
333	struct cgroup_subsys_state *mem_cgroup_css_from_page(struct page *page)
334	{
335	struct mem_cgroup *memcg;
336
337	memcg = page->mem_cgroup;
338
339	if (!memcg || !cgroup_subsys_on_dfl(memory_cgrp_subsys))
340	memcg = root_mem_cgroup;
341
342	return &memcg->css;
343	}
344
345	/**
346	* page_cgroup_ino - return inode number of the memcg a page is charged to
347	* @page: the page
348	*
349	* Look up the closest online ancestor of the memory cgroup @page is charged to
350	* and return its inode number or 0 if @page is not charged to any cgroup. It
351	* is safe to call this function without holding a reference to @page.
352	*
353	* Note, this function is inherently racy, because there is nothing to prevent
354	* the cgroup inode from getting torn down and potentially reallocated a moment
355	* after page_cgroup_ino() returns, so it only should be used by callers that
356	* do not care (such as procfs interfaces).
357	*/
358	ino_t page_cgroup_ino(struct page *page)
359	{
360	struct mem_cgroup *memcg;
361	unsigned long ino = 0;
362
363	rcu_read_lock();
364	memcg = READ_ONCE(page->mem_cgroup);
365	while (memcg && !(memcg->css.flags & CSS_ONLINE))
366	memcg = parent_mem_cgroup(memcg);
367	if (memcg)
368	ino = cgroup_ino(memcg->css.cgroup);
369	rcu_read_unlock();
370	return ino;
371	}
372
373	static struct mem_cgroup_per_node *
374	mem_cgroup_page_nodeinfo(struct mem_cgroup *memcg, struct page *page)
375	{
376	int nid = page_to_nid(page);
377
378	return memcg->nodeinfo[nid];
379	}
380
381	static struct mem_cgroup_tree_per_node *
382	soft_limit_tree_node(int nid)
383	{
384	return soft_limit_tree.rb_tree_per_node[nid];
385	}
386
387	static struct mem_cgroup_tree_per_node *
388	soft_limit_tree_from_page(struct page *page)
389	{
390	int nid = page_to_nid(page);
391
392	return soft_limit_tree.rb_tree_per_node[nid];
393	}
394
395	static void __mem_cgroup_insert_exceeded(struct mem_cgroup_per_node *mz,
396	struct mem_cgroup_tree_per_node *mctz,
397	unsigned long new_usage_in_excess)
398	{
399	struct rb_node **p = &mctz->rb_root.rb_node;
400	struct rb_node *parent = NULL;
401	struct mem_cgroup_per_node *mz_node;
402
403	if (mz->on_tree)
404	return;
405
406	mz->usage_in_excess = new_usage_in_excess;
407	if (!mz->usage_in_excess)
408	return;
409	while (*p) {
410	parent = *p;
411	mz_node = rb_entry(parent, struct mem_cgroup_per_node,
412	tree_node);
413	if (mz->usage_in_excess < mz_node->usage_in_excess)
414	p = &(*p)->rb_left;
415	/*
416	* We can't avoid mem cgroups that are over their soft
417	* limit by the same amount
418	*/
419	else if (mz->usage_in_excess >= mz_node->usage_in_excess)
420	p = &(*p)->rb_right;
421	}
422	rb_link_node(&mz->tree_node, parent, p);
423	rb_insert_color(&mz->tree_node, &mctz->rb_root);
424	mz->on_tree = true;
425	}
426
427	static void __mem_cgroup_remove_exceeded(struct mem_cgroup_per_node *mz,
428	struct mem_cgroup_tree_per_node *mctz)
429	{
430	if (!mz->on_tree)
431	return;
432	rb_erase(&mz->tree_node, &mctz->rb_root);
433	mz->on_tree = false;
434	}
435
436	static void mem_cgroup_remove_exceeded(struct mem_cgroup_per_node *mz,
437	struct mem_cgroup_tree_per_node *mctz)
438	{
439	unsigned long flags;
440
441	spin_lock_irqsave(&mctz->lock, flags);
442	__mem_cgroup_remove_exceeded(mz, mctz);
443	spin_unlock_irqrestore(&mctz->lock, flags);
444	}
445
446	static unsigned long soft_limit_excess(struct mem_cgroup *memcg)
447	{
448	unsigned long nr_pages = page_counter_read(&memcg->memory);
449	unsigned long soft_limit = READ_ONCE(memcg->soft_limit);
450	unsigned long excess = 0;
451
452	if (nr_pages > soft_limit)
453	excess = nr_pages - soft_limit;
454
455	return excess;
456	}
457
458	static void mem_cgroup_update_tree(struct mem_cgroup *memcg, struct page *page)
459	{
460	unsigned long excess;
461	struct mem_cgroup_per_node *mz;
462	struct mem_cgroup_tree_per_node *mctz;
463
464	mctz = soft_limit_tree_from_page(page);
465	if (!mctz)
466	return;
467	/*
468	* Necessary to update all ancestors when hierarchy is used.
469	* because their event counter is not touched.
470	*/
471	for (; memcg; memcg = parent_mem_cgroup(memcg)) {
472	mz = mem_cgroup_page_nodeinfo(memcg, page);
473	excess = soft_limit_excess(memcg);
474	/*
475	* We have to update the tree if mz is on RB-tree or
476	* mem is over its softlimit.
477	*/
478	if (excess || mz->on_tree) {
479	unsigned long flags;
480
481	spin_lock_irqsave(&mctz->lock, flags);
482	/* if on-tree, remove it */
483	if (mz->on_tree)
484	__mem_cgroup_remove_exceeded(mz, mctz);
485	/*
486	* Insert again. mz->usage_in_excess will be updated.
487	* If excess is 0, no tree ops.
488	*/
489	__mem_cgroup_insert_exceeded(mz, mctz, excess);
490	spin_unlock_irqrestore(&mctz->lock, flags);
491	}
492	}
493	}
494
495	static void mem_cgroup_remove_from_trees(struct mem_cgroup *memcg)
496	{
497	struct mem_cgroup_tree_per_node *mctz;
498	struct mem_cgroup_per_node *mz;
499	int nid;
500
501	for_each_node(nid) {
502	mz = mem_cgroup_nodeinfo(memcg, nid);
503	mctz = soft_limit_tree_node(nid);
504	if (mctz)
505	mem_cgroup_remove_exceeded(mz, mctz);
506	}
507	}
508
509	static struct mem_cgroup_per_node *
510	__mem_cgroup_largest_soft_limit_node(struct mem_cgroup_tree_per_node *mctz)
511	{
512	struct rb_node *rightmost = NULL;
513	struct mem_cgroup_per_node *mz;
514
515	retry:
516	mz = NULL;
517	rightmost = rb_last(&mctz->rb_root);
518	if (!rightmost)
519	goto done; /* Nothing to reclaim from */
520
521	mz = rb_entry(rightmost, struct mem_cgroup_per_node, tree_node);
522	/*
523	* Remove the node now but someone else can add it back,
524	* we will to add it back at the end of reclaim to its correct
525	* position in the tree.
526	*/
527	__mem_cgroup_remove_exceeded(mz, mctz);
528	if (!soft_limit_excess(mz->memcg) ||
529	!css_tryget_online(&mz->memcg->css))
530	goto retry;
531	done:
532	return mz;
533	}
534
535	static struct mem_cgroup_per_node *
536	mem_cgroup_largest_soft_limit_node(struct mem_cgroup_tree_per_node *mctz)
537	{
538	struct mem_cgroup_per_node *mz;
539
540	spin_lock_irq(&mctz->lock);
541	mz = __mem_cgroup_largest_soft_limit_node(mctz);
542	spin_unlock_irq(&mctz->lock);
543	return mz;
544	}
545
546	/*
547	* Return page count for single (non recursive) @memcg.
548	*
549	* Implementation Note: reading percpu statistics for memcg.
550	*
551	* Both of vmstat[] and percpu_counter has threshold and do periodic
552	* synchronization to implement "quick" read. There are trade-off between
553	* reading cost and precision of value. Then, we may have a chance to implement
554	* a periodic synchronization of counter in memcg's counter.
555	*
556	* But this _read() function is used for user interface now. The user accounts
557	* memory usage by memory cgroup and he _always_ requires exact value because
558	* he accounts memory. Even if we provide quick-and-fuzzy read, we always
559	* have to visit all online cpus and make sum. So, for now, unnecessary
560	* synchronization is not implemented. (just implemented for cpu hotplug)
561	*
562	* If there are kernel internal actions which can make use of some not-exact
563	* value, and reading all cpu value can be performance bottleneck in some
564	* common workload, threshold and synchronization as vmstat[] should be
565	* implemented.
566	*/
567	static unsigned long
568	mem_cgroup_read_stat(struct mem_cgroup *memcg, enum mem_cgroup_stat_index idx)
569	{
570	long val = 0;
571	int cpu;
572
573	/* Per-cpu values can be negative, use a signed accumulator */
574	for_each_possible_cpu(cpu)
575	val += per_cpu(memcg->stat->count[idx], cpu);
576	/*
577	* Summing races with updates, so val may be negative. Avoid exposing
578	* transient negative values.
579	*/
580	if (val < 0)
581	val = 0;
582	return val;
583	}
584
585	static unsigned long mem_cgroup_read_events(struct mem_cgroup *memcg,
586	enum mem_cgroup_events_index idx)
587	{
588	unsigned long val = 0;
589	int cpu;
590
591	for_each_possible_cpu(cpu)
592	val += per_cpu(memcg->stat->events[idx], cpu);
593	return val;
594	}
595
596	static void mem_cgroup_charge_statistics(struct mem_cgroup *memcg,
597	struct page *page,
598	bool compound, int nr_pages)
599	{
600	/*
601	* Here, RSS means 'mapped anon' and anon's SwapCache. Shmem/tmpfs is
602	* counted as CACHE even if it's on ANON LRU.
603	*/
604	if (PageAnon(page))
605	__this_cpu_add(memcg->stat->count[MEM_CGROUP_STAT_RSS],
606	nr_pages);
607	else
608	__this_cpu_add(memcg->stat->count[MEM_CGROUP_STAT_CACHE],
609	nr_pages);
610
611	if (compound) {
612	VM_BUG_ON_PAGE(!PageTransHuge(page), page);
613	__this_cpu_add(memcg->stat->count[MEM_CGROUP_STAT_RSS_HUGE],
614	nr_pages);
615	}
616
617	/* pagein of a big page is an event. So, ignore page size */
618	if (nr_pages > 0)
619	__this_cpu_inc(memcg->stat->events[MEM_CGROUP_EVENTS_PGPGIN]);
620	else {
621	__this_cpu_inc(memcg->stat->events[MEM_CGROUP_EVENTS_PGPGOUT]);
622	nr_pages = -nr_pages; /* for event */
623	}
624
625	__this_cpu_add(memcg->stat->nr_page_events, nr_pages);
626	}
627
628	unsigned long mem_cgroup_node_nr_lru_pages(struct mem_cgroup *memcg,
629	int nid, unsigned int lru_mask)
630	{
631	struct lruvec *lruvec = mem_cgroup_lruvec(NODE_DATA(nid), memcg);
632	unsigned long nr = 0;
633	enum lru_list lru;
634
635	VM_BUG_ON((unsigned)nid >= nr_node_ids);
636
637	for_each_lru(lru) {
638	if (!(BIT(lru) & lru_mask))
639	continue;
640	nr += mem_cgroup_get_lru_size(lruvec, lru);
641	}
642	return nr;
643	}
644
645	static unsigned long mem_cgroup_nr_lru_pages(struct mem_cgroup *memcg,
646	unsigned int lru_mask)
647	{
648	unsigned long nr = 0;
649	int nid;
650
651	for_each_node_state(nid, N_MEMORY)
652	nr += mem_cgroup_node_nr_lru_pages(memcg, nid, lru_mask);
653	return nr;
654	}
655
656	static bool mem_cgroup_event_ratelimit(struct mem_cgroup *memcg,
657	enum mem_cgroup_events_target target)
658	{
659	unsigned long val, next;
660
661	val = __this_cpu_read(memcg->stat->nr_page_events);
662	next = __this_cpu_read(memcg->stat->targets[target]);
663	/* from time_after() in jiffies.h */
664	if ((long)next - (long)val < 0) {
665	switch (target) {
666	case MEM_CGROUP_TARGET_THRESH:
667	next = val + THRESHOLDS_EVENTS_TARGET;
668	break;
669	case MEM_CGROUP_TARGET_SOFTLIMIT:
670	next = val + SOFTLIMIT_EVENTS_TARGET;
671	break;
672	case MEM_CGROUP_TARGET_NUMAINFO:
673	next = val + NUMAINFO_EVENTS_TARGET;
674	break;
675	default:
676	break;
677	}
678	__this_cpu_write(memcg->stat->targets[target], next);
679	return true;
680	}
681	return false;
682	}
683
684	/*
685	* Check events in order.
686	*
687	*/
688	static void memcg_check_events(struct mem_cgroup *memcg, struct page *page)
689	{
690	/* threshold event is triggered in finer grain than soft limit */
691	if (unlikely(mem_cgroup_event_ratelimit(memcg,
692	MEM_CGROUP_TARGET_THRESH))) {
693	bool do_softlimit;
694	bool do_numainfo __maybe_unused;
695
696	do_softlimit = mem_cgroup_event_ratelimit(memcg,
697	MEM_CGROUP_TARGET_SOFTLIMIT);
698	#if MAX_NUMNODES > 1
699	do_numainfo = mem_cgroup_event_ratelimit(memcg,
700	MEM_CGROUP_TARGET_NUMAINFO);
701	#endif
702	mem_cgroup_threshold(memcg);
703	if (unlikely(do_softlimit))
704	mem_cgroup_update_tree(memcg, page);
705	#if MAX_NUMNODES > 1
706	if (unlikely(do_numainfo))
707	atomic_inc(&memcg->numainfo_events);
708	#endif
709	}
710	}
711
712	struct mem_cgroup *mem_cgroup_from_task(struct task_struct *p)
713	{
714	/*
715	* mm_update_next_owner() may clear mm->owner to NULL
716	* if it races with swapoff, page migration, etc.
717	* So this can be called with p == NULL.
718	*/
719	if (unlikely(!p))
720	return NULL;
721
722	return mem_cgroup_from_css(task_css(p, memory_cgrp_id));
723	}
724	EXPORT_SYMBOL(mem_cgroup_from_task);
725
726	static struct mem_cgroup *get_mem_cgroup_from_mm(struct mm_struct *mm)
727	{
728	struct mem_cgroup *memcg = NULL;
729
730	rcu_read_lock();
731	do {
732	/*
733	* Page cache insertions can happen withou an
734	* actual mm context, e.g. during disk probing
735	* on boot, loopback IO, acct() writes etc.
736	*/
737	if (unlikely(!mm))
738	memcg = root_mem_cgroup;
739	else {
740	memcg = mem_cgroup_from_task(rcu_dereference(mm->owner));
741	if (unlikely(!memcg))
742	memcg = root_mem_cgroup;
743	}
744	} while (!css_tryget_online(&memcg->css));
745	rcu_read_unlock();
746	return memcg;
747	}
748
749	/**
750	* mem_cgroup_iter - iterate over memory cgroup hierarchy
751	* @root: hierarchy root
752	* @prev: previously returned memcg, NULL on first invocation
753	* @reclaim: cookie for shared reclaim walks, NULL for full walks
754	*
755	* Returns references to children of the hierarchy below @root, or
756	* @root itself, or %NULL after a full round-trip.
757	*
758	* Caller must pass the return value in @prev on subsequent
759	* invocations for reference counting, or use mem_cgroup_iter_break()
760	* to cancel a hierarchy walk before the round-trip is complete.
761	*
762	* Reclaimers can specify a zone and a priority level in @reclaim to
763	* divide up the memcgs in the hierarchy among all concurrent
764	* reclaimers operating on the same zone and priority.
765	*/
766	struct mem_cgroup *mem_cgroup_iter(struct mem_cgroup *root,
767	struct mem_cgroup *prev,
768	struct mem_cgroup_reclaim_cookie *reclaim)
769	{
770	struct mem_cgroup_reclaim_iter *uninitialized_var(iter);
771	struct cgroup_subsys_state *css = NULL;
772	struct mem_cgroup *memcg = NULL;
773	struct mem_cgroup *pos = NULL;
774
775	if (mem_cgroup_disabled())
776	return NULL;
777
778	if (!root)
779	root = root_mem_cgroup;
780
781	if (prev && !reclaim)
782	pos = prev;
783
784	if (!root->use_hierarchy && root != root_mem_cgroup) {
785	if (prev)
786	goto out;
787	return root;
788	}
789
790	rcu_read_lock();
791
792	if (reclaim) {
793	struct mem_cgroup_per_node *mz;
794
795	mz = mem_cgroup_nodeinfo(root, reclaim->pgdat->node_id);
796	iter = &mz->iter[reclaim->priority];
797
798	if (prev && reclaim->generation != iter->generation)
799	goto out_unlock;
800
801	while (1) {
802	pos = READ_ONCE(iter->position);
803	if (!pos || css_tryget(&pos->css))
804	break;
805	/*
806	* css reference reached zero, so iter->position will
807	* be cleared by ->css_released. However, we should not
808	* rely on this happening soon, because ->css_released
809	* is called from a work queue, and by busy-waiting we
810	* might block it. So we clear iter->position right
811	* away.
812	*/
813	(void)cmpxchg(&iter->position, pos, NULL);
814	}
815	}
816
817	if (pos)
818	css = &pos->css;
819
820	for (;;) {
821	css = css_next_descendant_pre(css, &root->css);
822	if (!css) {
823	/*
824	* Reclaimers share the hierarchy walk, and a
825	* new one might jump in right at the end of
826	* the hierarchy - make sure they see at least
827	* one group and restart from the beginning.
828	*/
829	if (!prev)
830	continue;
831	break;
832	}
833
834	/*
835	* Verify the css and acquire a reference. The root
836	* is provided by the caller, so we know it's alive
837	* and kicking, and don't take an extra reference.
838	*/
839	memcg = mem_cgroup_from_css(css);
840
841	if (css == &root->css)
842	break;
843
844	if (css_tryget(css))
845	break;
846
847	memcg = NULL;
848	}
849
850	if (reclaim) {
851	/*
852	* The position could have already been updated by a competing
853	* thread, so check that the value hasn't changed since we read
854	* it to avoid reclaiming from the same cgroup twice.
855	*/
856	(void)cmpxchg(&iter->position, pos, memcg);
857
858	if (pos)
859	css_put(&pos->css);
860
861	if (!memcg)
862	iter->generation++;
863	else if (!prev)
864	reclaim->generation = iter->generation;
865	}
866
867	out_unlock:
868	rcu_read_unlock();
869	out:
870	if (prev && prev != root)
871	css_put(&prev->css);
872
873	return memcg;
874	}
875
876	/**
877	* mem_cgroup_iter_break - abort a hierarchy walk prematurely
878	* @root: hierarchy root
879	* @prev: last visited hierarchy member as returned by mem_cgroup_iter()
880	*/
881	void mem_cgroup_iter_break(struct mem_cgroup *root,
882	struct mem_cgroup *prev)
883	{
884	if (!root)
885	root = root_mem_cgroup;
886	if (prev && prev != root)
887	css_put(&prev->css);
888	}
889
890	static void invalidate_reclaim_iterators(struct mem_cgroup *dead_memcg)
891	{
892	struct mem_cgroup *memcg = dead_memcg;
893	struct mem_cgroup_reclaim_iter *iter;
894	struct mem_cgroup_per_node *mz;
895	int nid;
896	int i;
897
898	for (; memcg; memcg = parent_mem_cgroup(memcg)) {
899	for_each_node(nid) {
900	mz = mem_cgroup_nodeinfo(memcg, nid);
901	for (i = 0; i <= DEF_PRIORITY; i++) {
902	iter = &mz->iter[i];
903	cmpxchg(&iter->position,
904	dead_memcg, NULL);
905	}
906	}
907	}
908	}
909
910	/*
911	* Iteration constructs for visiting all cgroups (under a tree). If
912	* loops are exited prematurely (break), mem_cgroup_iter_break() must
913	* be used for reference counting.
914	*/
915	#define for_each_mem_cgroup_tree(iter, root) \
916	for (iter = mem_cgroup_iter(root, NULL, NULL); \
917	iter != NULL; \
918	iter = mem_cgroup_iter(root, iter, NULL))
919
920	#define for_each_mem_cgroup(iter) \
921	for (iter = mem_cgroup_iter(NULL, NULL, NULL); \
922	iter != NULL; \
923	iter = mem_cgroup_iter(NULL, iter, NULL))
924
925	/**
926	* mem_cgroup_scan_tasks - iterate over tasks of a memory cgroup hierarchy
927	* @memcg: hierarchy root
928	* @fn: function to call for each task
929	* @arg: argument passed to @fn
930	*
931	* This function iterates over tasks attached to @memcg or to any of its
932	* descendants and calls @fn for each task. If @fn returns a non-zero
933	* value, the function breaks the iteration loop and returns the value.
934	* Otherwise, it will iterate over all tasks and return 0.
935	*
936	* This function must not be called for the root memory cgroup.
937	*/
938	int mem_cgroup_scan_tasks(struct mem_cgroup *memcg,
939	int (*fn)(struct task_struct *, void *), void *arg)
940	{
941	struct mem_cgroup *iter;
942	int ret = 0;
943
944	BUG_ON(memcg == root_mem_cgroup);
945
946	for_each_mem_cgroup_tree(iter, memcg) {
947	struct css_task_iter it;
948	struct task_struct *task;
949
950	css_task_iter_start(&iter->css, &it);
951	while (!ret && (task = css_task_iter_next(&it)))
952	ret = fn(task, arg);
953	css_task_iter_end(&it);
954	if (ret) {
955	mem_cgroup_iter_break(memcg, iter);
956	break;
957	}
958	}
959	return ret;
960	}
961
962	/**
963	* mem_cgroup_page_lruvec - return lruvec for isolating/putting an LRU page
964	* @page: the page
965	* @zone: zone of the page
966	*
967	* This function is only safe when following the LRU page isolation
968	* and putback protocol: the LRU lock must be held, and the page must
969	* either be PageLRU() or the caller must have isolated/allocated it.
970	*/
971	struct lruvec *mem_cgroup_page_lruvec(struct page *page, struct pglist_data *pgdat)
972	{
973	struct mem_cgroup_per_node *mz;
974	struct mem_cgroup *memcg;
975	struct lruvec *lruvec;
976
977	if (mem_cgroup_disabled()) {
978	lruvec = &pgdat->lruvec;
979	goto out;
980	}
981
982	memcg = page->mem_cgroup;
983	/*
984	* Swapcache readahead pages are added to the LRU - and
985	* possibly migrated - before they are charged.
986	*/
987	if (!memcg)
988	memcg = root_mem_cgroup;
989
990	mz = mem_cgroup_page_nodeinfo(memcg, page);
991	lruvec = &mz->lruvec;
992	out:
993	/*
994	* Since a node can be onlined after the mem_cgroup was created,
995	* we have to be prepared to initialize lruvec->zone here;
996	* and if offlined then reonlined, we need to reinitialize it.
997	*/
998	if (unlikely(lruvec->pgdat != pgdat))
999	lruvec->pgdat = pgdat;
1000	return lruvec;
1001	}
1002
1003	/**
1004	* mem_cgroup_update_lru_size - account for adding or removing an lru page
1005	* @lruvec: mem_cgroup per zone lru vector
1006	* @lru: index of lru list the page is sitting on
1007	* @zid: zone id of the accounted pages
1008	* @nr_pages: positive when adding or negative when removing
1009	*
1010	* This function must be called under lru_lock, just before a page is added
1011	* to or just after a page is removed from an lru list (that ordering being
1012	* so as to allow it to check that lru_size 0 is consistent with list_empty).
1013	*/
1014	void mem_cgroup_update_lru_size(struct lruvec *lruvec, enum lru_list lru,
1015	int zid, int nr_pages)
1016	{
1017	struct mem_cgroup_per_node *mz;
1018	unsigned long *lru_size;
1019	long size;
1020
1021	if (mem_cgroup_disabled())
1022	return;
1023
1024	mz = container_of(lruvec, struct mem_cgroup_per_node, lruvec);
1025	lru_size = &mz->lru_zone_size[zid][lru];
1026
1027	if (nr_pages < 0)
1028	*lru_size += nr_pages;
1029
1030	size = *lru_size;
1031	if (WARN_ONCE(size < 0,
1032	"%s(%p, %d, %d): lru_size %ld\n",
1033	__func__, lruvec, lru, nr_pages, size)) {
1034	VM_BUG_ON(1);
1035	*lru_size = 0;
1036	}
1037
1038	if (nr_pages > 0)
1039	*lru_size += nr_pages;
1040	}
1041
1042	bool task_in_mem_cgroup(struct task_struct *task, struct mem_cgroup *memcg)
1043	{
1044	struct mem_cgroup *task_memcg;
1045	struct task_struct *p;
1046	bool ret;
1047
1048	p = find_lock_task_mm(task);
1049	if (p) {
1050	task_memcg = get_mem_cgroup_from_mm(p->mm);
1051	task_unlock(p);
1052	} else {
1053	/*
1054	* All threads may have already detached their mm's, but the oom
1055	* killer still needs to detect if they have already been oom
1056	* killed to prevent needlessly killing additional tasks.
1057	*/
1058	rcu_read_lock();
1059	task_memcg = mem_cgroup_from_task(task);
1060	css_get(&task_memcg->css);
1061	rcu_read_unlock();
1062	}
1063	ret = mem_cgroup_is_descendant(task_memcg, memcg);
1064	css_put(&task_memcg->css);
1065	return ret;
1066	}
1067
1068	/**
1069	* mem_cgroup_margin - calculate chargeable space of a memory cgroup
1070	* @memcg: the memory cgroup
1071	*
1072	* Returns the maximum amount of memory @mem can be charged with, in
1073	* pages.
1074	*/
1075	static unsigned long mem_cgroup_margin(struct mem_cgroup *memcg)
1076	{
1077	unsigned long margin = 0;
1078	unsigned long count;
1079	unsigned long limit;
1080
1081	count = page_counter_read(&memcg->memory);
1082	limit = READ_ONCE(memcg->memory.limit);
1083	if (count < limit)
1084	margin = limit - count;
1085
1086	if (do_memsw_account()) {
1087	count = page_counter_read(&memcg->memsw);
1088	limit = READ_ONCE(memcg->memsw.limit);
1089	if (count <= limit)
1090	margin = min(margin, limit - count);
1091	else
1092	margin = 0;
1093	}
1094
1095	return margin;
1096	}
1097
1098	/*
1099	* A routine for checking "mem" is under move_account() or not.
1100	*
1101	* Checking a cgroup is mc.from or mc.to or under hierarchy of
1102	* moving cgroups. This is for waiting at high-memory pressure
1103	* caused by "move".
1104	*/
1105	static bool mem_cgroup_under_move(struct mem_cgroup *memcg)
1106	{
1107	struct mem_cgroup *from;
1108	struct mem_cgroup *to;
1109	bool ret = false;
1110	/*
1111	* Unlike task_move routines, we access mc.to, mc.from not under
1112	* mutual exclusion by cgroup_mutex. Here, we take spinlock instead.
1113	*/
1114	spin_lock(&mc.lock);
1115	from = mc.from;
1116	to = mc.to;
1117	if (!from)
1118	goto unlock;
1119
1120	ret = mem_cgroup_is_descendant(from, memcg) ||
1121	mem_cgroup_is_descendant(to, memcg);
1122	unlock:
1123	spin_unlock(&mc.lock);
1124	return ret;
1125	}
1126
1127	static bool mem_cgroup_wait_acct_move(struct mem_cgroup *memcg)
1128	{
1129	if (mc.moving_task && current != mc.moving_task) {
1130	if (mem_cgroup_under_move(memcg)) {
1131	DEFINE_WAIT(wait);
1132	prepare_to_wait(&mc.waitq, &wait, TASK_INTERRUPTIBLE);
1133	/* moving charge context might have finished. */
1134	if (mc.moving_task)
1135	schedule();
1136	finish_wait(&mc.waitq, &wait);
1137	return true;
1138	}
1139	}
1140	return false;
1141	}
1142
1143	#define K(x) ((x) << (PAGE_SHIFT-10))
1144	/**
1145	* mem_cgroup_print_oom_info: Print OOM information relevant to memory controller.
1146	* @memcg: The memory cgroup that went over limit
1147	* @p: Task that is going to be killed
1148	*
1149	* NOTE: @memcg and @p's mem_cgroup can be different when hierarchy is
1150	* enabled
1151	*/
1152	void mem_cgroup_print_oom_info(struct mem_cgroup *memcg, struct task_struct *p)
1153	{
1154	struct mem_cgroup *iter;
1155	unsigned int i;
1156
1157	rcu_read_lock();
1158
1159	if (p) {
1160	pr_info("Task in ");
1161	pr_cont_cgroup_path(task_cgroup(p, memory_cgrp_id));
1162	pr_cont(" killed as a result of limit of ");
1163	} else {
1164	pr_info("Memory limit reached of cgroup ");
1165	}
1166
1167	pr_cont_cgroup_path(memcg->css.cgroup);
1168	pr_cont("\n");
1169
1170	rcu_read_unlock();
1171
1172	pr_info("memory: usage %llukB, limit %llukB, failcnt %lu\n",
1173	K((u64)page_counter_read(&memcg->memory)),
1174	K((u64)memcg->memory.limit), memcg->memory.failcnt);
1175	pr_info("memory+swap: usage %llukB, limit %llukB, failcnt %lu\n",
1176	K((u64)page_counter_read(&memcg->memsw)),
1177	K((u64)memcg->memsw.limit), memcg->memsw.failcnt);
1178	pr_info("kmem: usage %llukB, limit %llukB, failcnt %lu\n",
1179	K((u64)page_counter_read(&memcg->kmem)),
1180	K((u64)memcg->kmem.limit), memcg->kmem.failcnt);
1181
1182	for_each_mem_cgroup_tree(iter, memcg) {
1183	pr_info("Memory cgroup stats for ");
1184	pr_cont_cgroup_path(iter->css.cgroup);
1185	pr_cont(":");
1186
1187	for (i = 0; i < MEM_CGROUP_STAT_NSTATS; i++) {
1188	if (i == MEM_CGROUP_STAT_SWAP && !do_swap_account)
1189	continue;
1190	pr_cont(" %s:%luKB", mem_cgroup_stat_names[i],
1191	K(mem_cgroup_read_stat(iter, i)));
1192	}
1193
1194	for (i = 0; i < NR_LRU_LISTS; i++)
1195	pr_cont(" %s:%luKB", mem_cgroup_lru_names[i],
1196	K(mem_cgroup_nr_lru_pages(iter, BIT(i))));
1197
1198	pr_cont("\n");
1199	}
1200	}
1201
1202	/*
1203	* This function returns the number of memcg under hierarchy tree. Returns
1204	* 1(self count) if no children.
1205	*/
1206	static int mem_cgroup_count_children(struct mem_cgroup *memcg)
1207	{
1208	int num = 0;
1209	struct mem_cgroup *iter;
1210
1211	for_each_mem_cgroup_tree(iter, memcg)
1212	num++;
1213	return num;
1214	}
1215
1216	/*
1217	* Return the memory (and swap, if configured) limit for a memcg.
1218	*/
1219	unsigned long mem_cgroup_get_limit(struct mem_cgroup *memcg)
1220	{
1221	unsigned long limit;
1222
1223	limit = memcg->memory.limit;
1224	if (mem_cgroup_swappiness(memcg)) {
1225	unsigned long memsw_limit;
1226	unsigned long swap_limit;
1227
1228	memsw_limit = memcg->memsw.limit;
1229	swap_limit = memcg->swap.limit;
1230	swap_limit = min(swap_limit, (unsigned long)total_swap_pages);
1231	limit = min(limit + swap_limit, memsw_limit);
1232	}
1233	return limit;
1234	}
1235
1236	static bool mem_cgroup_out_of_memory(struct mem_cgroup *memcg, gfp_t gfp_mask,
1237	int order)
1238	{
1239	struct oom_control oc = {
1240	.zonelist = NULL,
1241	.nodemask = NULL,
1242	.memcg = memcg,
1243	.gfp_mask = gfp_mask,
1244	.order = order,
1245	};
1246	bool ret;
1247
1248	mutex_lock(&oom_lock);
1249	ret = out_of_memory(&oc);
1250	mutex_unlock(&oom_lock);
1251	return ret;
1252	}
1253
1254	#if MAX_NUMNODES > 1
1255
1256	/**
1257	* test_mem_cgroup_node_reclaimable
1258	* @memcg: the target memcg
1259	* @nid: the node ID to be checked.
1260	* @noswap : specify true here if the user wants flle only information.
1261	*
1262	* This function returns whether the specified memcg contains any
1263	* reclaimable pages on a node. Returns true if there are any reclaimable
1264	* pages in the node.
1265	*/
1266	static bool test_mem_cgroup_node_reclaimable(struct mem_cgroup *memcg,
1267	int nid, bool noswap)
1268	{
1269	if (mem_cgroup_node_nr_lru_pages(memcg, nid, LRU_ALL_FILE))
1270	return true;
1271	if (noswap || !total_swap_pages)
1272	return false;
1273	if (mem_cgroup_node_nr_lru_pages(memcg, nid, LRU_ALL_ANON))
1274	return true;
1275	return false;
1276
1277	}
1278
1279	/*
1280	* Always updating the nodemask is not very good - even if we have an empty
1281	* list or the wrong list here, we can start from some node and traverse all
1282	* nodes based on the zonelist. So update the list loosely once per 10 secs.
1283	*
1284	*/
1285	static void mem_cgroup_may_update_nodemask(struct mem_cgroup *memcg)
1286	{
1287	int nid;
1288	/*
1289	* numainfo_events > 0 means there was at least NUMAINFO_EVENTS_TARGET
1290	* pagein/pageout changes since the last update.
1291	*/
1292	if (!atomic_read(&memcg->numainfo_events))
1293	return;
1294	if (atomic_inc_return(&memcg->numainfo_updating) > 1)
1295	return;
1296
1297	/* make a nodemask where this memcg uses memory from */
1298	memcg->scan_nodes = node_states[N_MEMORY];
1299
1300	for_each_node_mask(nid, node_states[N_MEMORY]) {
1301
1302	if (!test_mem_cgroup_node_reclaimable(memcg, nid, false))
1303	node_clear(nid, memcg->scan_nodes);
1304	}
1305
1306	atomic_set(&memcg->numainfo_events, 0);
1307	atomic_set(&memcg->numainfo_updating, 0);
1308	}
1309
1310	/*
1311	* Selecting a node where we start reclaim from. Because what we need is just
1312	* reducing usage counter, start from anywhere is O,K. Considering
1313	* memory reclaim from current node, there are pros. and cons.
1314	*
1315	* Freeing memory from current node means freeing memory from a node which
1316	* we'll use or we've used. So, it may make LRU bad. And if several threads
1317	* hit limits, it will see a contention on a node. But freeing from remote
1318	* node means more costs for memory reclaim because of memory latency.
1319	*
1320	* Now, we use round-robin. Better algorithm is welcomed.
1321	*/
1322	int mem_cgroup_select_victim_node(struct mem_cgroup *memcg)
1323	{
1324	int node;
1325
1326	mem_cgroup_may_update_nodemask(memcg);
1327	node = memcg->last_scanned_node;
1328
1329	node = next_node_in(node, memcg->scan_nodes);
1330	/*
1331	* mem_cgroup_may_update_nodemask might have seen no reclaimmable pages
1332	* last time it really checked all the LRUs due to rate limiting.
1333	* Fallback to the current node in that case for simplicity.
1334	*/
1335	if (unlikely(node == MAX_NUMNODES))
1336	node = numa_node_id();
1337
1338	memcg->last_scanned_node = node;
1339	return node;
1340	}
1341	#else
1342	int mem_cgroup_select_victim_node(struct mem_cgroup *memcg)
1343	{
1344	return 0;
1345	}
1346	#endif
1347
1348	static int mem_cgroup_soft_reclaim(struct mem_cgroup *root_memcg,
1349	pg_data_t *pgdat,
1350	gfp_t gfp_mask,
1351	unsigned long *total_scanned)
1352	{
1353	struct mem_cgroup *victim = NULL;
1354	int total = 0;
1355	int loop = 0;
1356	unsigned long excess;
1357	unsigned long nr_scanned;
1358	struct mem_cgroup_reclaim_cookie reclaim = {
1359	.pgdat = pgdat,
1360	.priority = 0,
1361	};
1362
1363	excess = soft_limit_excess(root_memcg);
1364
1365	while (1) {
1366	victim = mem_cgroup_iter(root_memcg, victim, &reclaim);
1367	if (!victim) {
1368	loop++;
1369	if (loop >= 2) {
1370	/*
1371	* If we have not been able to reclaim
1372	* anything, it might because there are
1373	* no reclaimable pages under this hierarchy
1374	*/
1375	if (!total)
1376	break;
1377	/*
1378	* We want to do more targeted reclaim.
1379	* excess >> 2 is not to excessive so as to
1380	* reclaim too much, nor too less that we keep
1381	* coming back to reclaim from this cgroup
1382	*/
1383	if (total >= (excess >> 2) ||
1384	(loop > MEM_CGROUP_MAX_RECLAIM_LOOPS))
1385	break;
1386	}
1387	continue;
1388	}
1389	total += mem_cgroup_shrink_node(victim, gfp_mask, false,
1390	pgdat, &nr_scanned);
1391	*total_scanned += nr_scanned;
1392	if (!soft_limit_excess(root_memcg))
1393	break;
1394	}
1395	mem_cgroup_iter_break(root_memcg, victim);
1396	return total;
1397	}
1398
1399	#ifdef CONFIG_LOCKDEP
1400	static struct lockdep_map memcg_oom_lock_dep_map = {
1401	.name = "memcg_oom_lock",
1402	};
1403	#endif
1404
1405	static DEFINE_SPINLOCK(memcg_oom_lock);
1406
1407	/*
1408	* Check OOM-Killer is already running under our hierarchy.
1409	* If someone is running, return false.
1410	*/
1411	static bool mem_cgroup_oom_trylock(struct mem_cgroup *memcg)
1412	{
1413	struct mem_cgroup *iter, *failed = NULL;
1414
1415	spin_lock(&memcg_oom_lock);
1416
1417	for_each_mem_cgroup_tree(iter, memcg) {
1418	if (iter->oom_lock) {
1419	/*
1420	* this subtree of our hierarchy is already locked
1421	* so we cannot give a lock.
1422	*/
1423	failed = iter;
1424	mem_cgroup_iter_break(memcg, iter);
1425	break;
1426	} else
1427	iter->oom_lock = true;
1428	}
1429
1430	if (failed) {
1431	/*
1432	* OK, we failed to lock the whole subtree so we have
1433	* to clean up what we set up to the failing subtree
1434	*/
1435	for_each_mem_cgroup_tree(iter, memcg) {
1436	if (iter == failed) {
1437	mem_cgroup_iter_break(memcg, iter);
1438	break;
1439	}
1440	iter->oom_lock = false;
1441	}
1442	} else
1443	mutex_acquire(&memcg_oom_lock_dep_map, 0, 1, _RET_IP_);
1444
1445	spin_unlock(&memcg_oom_lock);
1446
1447	return !failed;
1448	}
1449
1450	static void mem_cgroup_oom_unlock(struct mem_cgroup *memcg)
1451	{
1452	struct mem_cgroup *iter;
1453
1454	spin_lock(&memcg_oom_lock);
1455	mutex_release(&memcg_oom_lock_dep_map, 1, _RET_IP_);
1456	for_each_mem_cgroup_tree(iter, memcg)
1457	iter->oom_lock = false;
1458	spin_unlock(&memcg_oom_lock);
1459	}
1460
1461	static void mem_cgroup_mark_under_oom(struct mem_cgroup *memcg)
1462	{
1463	struct mem_cgroup *iter;
1464
1465	spin_lock(&memcg_oom_lock);
1466	for_each_mem_cgroup_tree(iter, memcg)
1467	iter->under_oom++;
1468	spin_unlock(&memcg_oom_lock);
1469	}
1470
1471	static void mem_cgroup_unmark_under_oom(struct mem_cgroup *memcg)
1472	{
1473	struct mem_cgroup *iter;
1474
1475	/*
1476	* When a new child is created while the hierarchy is under oom,
1477	* mem_cgroup_oom_lock() may not be called. Watch for underflow.
1478	*/
1479	spin_lock(&memcg_oom_lock);
1480	for_each_mem_cgroup_tree(iter, memcg)
1481	if (iter->under_oom > 0)
1482	iter->under_oom--;
1483	spin_unlock(&memcg_oom_lock);
1484	}
1485
1486	static DECLARE_WAIT_QUEUE_HEAD(memcg_oom_waitq);
1487
1488	struct oom_wait_info {
1489	struct mem_cgroup *memcg;
1490	wait_queue_t wait;
1491	};
1492
1493	static int memcg_oom_wake_function(wait_queue_t *wait,
1494	unsigned mode, int sync, void *arg)
1495	{
1496	struct mem_cgroup *wake_memcg = (struct mem_cgroup *)arg;
1497	struct mem_cgroup *oom_wait_memcg;
1498	struct oom_wait_info *oom_wait_info;
1499
1500	oom_wait_info = container_of(wait, struct oom_wait_info, wait);
1501	oom_wait_memcg = oom_wait_info->memcg;
1502
1503	if (!mem_cgroup_is_descendant(wake_memcg, oom_wait_memcg) &&
1504	!mem_cgroup_is_descendant(oom_wait_memcg, wake_memcg))
1505	return 0;
1506	return autoremove_wake_function(wait, mode, sync, arg);
1507	}
1508
1509	static void memcg_oom_recover(struct mem_cgroup *memcg)
1510	{
1511	/*
1512	* For the following lockless ->under_oom test, the only required
1513	* guarantee is that it must see the state asserted by an OOM when
1514	* this function is called as a result of userland actions
1515	* triggered by the notification of the OOM. This is trivially
1516	* achieved by invoking mem_cgroup_mark_under_oom() before
1517	* triggering notification.
1518	*/
1519	if (memcg && memcg->under_oom)
1520	__wake_up(&memcg_oom_waitq, TASK_NORMAL, 0, memcg);
1521	}
1522
1523	static void mem_cgroup_oom(struct mem_cgroup *memcg, gfp_t mask, int order)
1524	{
1525	if (!current->memcg_may_oom)
1526	return;
1527	/*
1528	* We are in the middle of the charge context here, so we
1529	* don't want to block when potentially sitting on a callstack
1530	* that holds all kinds of filesystem and mm locks.
1531	*
1532	* Also, the caller may handle a failed allocation gracefully
1533	* (like optional page cache readahead) and so an OOM killer
1534	* invocation might not even be necessary.
1535	*
1536	* That's why we don't do anything here except remember the
1537	* OOM context and then deal with it at the end of the page
1538	* fault when the stack is unwound, the locks are released,
1539	* and when we know whether the fault was overall successful.
1540	*/
1541	css_get(&memcg->css);
1542	current->memcg_in_oom = memcg;
1543	current->memcg_oom_gfp_mask = mask;
1544	current->memcg_oom_order = order;
1545	}
1546
1547	/**
1548	* mem_cgroup_oom_synchronize - complete memcg OOM handling
1549	* @handle: actually kill/wait or just clean up the OOM state
1550	*
1551	* This has to be called at the end of a page fault if the memcg OOM
1552	* handler was enabled.
1553	*
1554	* Memcg supports userspace OOM handling where failed allocations must
1555	* sleep on a waitqueue until the userspace task resolves the
1556	* situation. Sleeping directly in the charge context with all kinds
1557	* of locks held is not a good idea, instead we remember an OOM state
1558	* in the task and mem_cgroup_oom_synchronize() has to be called at
1559	* the end of the page fault to complete the OOM handling.
1560	*
1561	* Returns %true if an ongoing memcg OOM situation was detected and
1562	* completed, %false otherwise.
1563	*/
1564	bool mem_cgroup_oom_synchronize(bool handle)
1565	{
1566	struct mem_cgroup *memcg = current->memcg_in_oom;
1567	struct oom_wait_info owait;
1568	bool locked;
1569
1570	/* OOM is global, do not handle */
1571	if (!memcg)
1572	return false;
1573
1574	if (!handle)
1575	goto cleanup;
1576
1577	owait.memcg = memcg;
1578	owait.wait.flags = 0;
1579	owait.wait.func = memcg_oom_wake_function;
1580	owait.wait.private = current;
1581	INIT_LIST_HEAD(&owait.wait.task_list);
1582
1583	prepare_to_wait(&memcg_oom_waitq, &owait.wait, TASK_KILLABLE);
1584	mem_cgroup_mark_under_oom(memcg);
1585
1586	locked = mem_cgroup_oom_trylock(memcg);
1587
1588	if (locked)
1589	mem_cgroup_oom_notify(memcg);
1590
1591	if (locked && !memcg->oom_kill_disable) {
1592	mem_cgroup_unmark_under_oom(memcg);
1593	finish_wait(&memcg_oom_waitq, &owait.wait);
1594	mem_cgroup_out_of_memory(memcg, current->memcg_oom_gfp_mask,
1595	current->memcg_oom_order);
1596	} else {
1597	schedule();
1598	mem_cgroup_unmark_under_oom(memcg);
1599	finish_wait(&memcg_oom_waitq, &owait.wait);
1600	}
1601
1602	if (locked) {
1603	mem_cgroup_oom_unlock(memcg);
1604	/*
1605	* There is no guarantee that an OOM-lock contender
1606	* sees the wakeups triggered by the OOM kill
1607	* uncharges. Wake any sleepers explicitely.
1608	*/
1609	memcg_oom_recover(memcg);
1610	}
1611	cleanup:
1612	current->memcg_in_oom = NULL;
1613	css_put(&memcg->css);
1614	return true;
1615	}
1616
1617	/**
1618	* lock_page_memcg - lock a page->mem_cgroup binding
1619	* @page: the page
1620	*
1621	* This function protects unlocked LRU pages from being moved to
1622	* another cgroup and stabilizes their page->mem_cgroup binding.
1623	*/
1624	void lock_page_memcg(struct page *page)
1625	{
1626	struct mem_cgroup *memcg;
1627	unsigned long flags;
1628
1629	/*
1630	* The RCU lock is held throughout the transaction. The fast
1631	* path can get away without acquiring the memcg->move_lock
1632	* because page moving starts with an RCU grace period.
1633	*/
1634	rcu_read_lock();
1635
1636	if (mem_cgroup_disabled())
1637	return;
1638	again:
1639	memcg = page->mem_cgroup;
1640	if (unlikely(!memcg))
1641	return;
1642
1643	if (atomic_read(&memcg->moving_account) <= 0)
1644	return;
1645
1646	spin_lock_irqsave(&memcg->move_lock, flags);
1647	if (memcg != page->mem_cgroup) {
1648	spin_unlock_irqrestore(&memcg->move_lock, flags);
1649	goto again;
1650	}
1651
1652	/*
1653	* When charge migration first begins, we can have locked and
1654	* unlocked page stat updates happening concurrently. Track
1655	* the task who has the lock for unlock_page_memcg().
1656	*/
1657	memcg->move_lock_task = current;
1658	memcg->move_lock_flags = flags;
1659
1660	return;
1661	}
1662	EXPORT_SYMBOL(lock_page_memcg);
1663
1664	/**
1665	* unlock_page_memcg - unlock a page->mem_cgroup binding
1666	* @page: the page
1667	*/
1668	void unlock_page_memcg(struct page *page)
1669	{
1670	struct mem_cgroup *memcg = page->mem_cgroup;
1671
1672	if (memcg && memcg->move_lock_task == current) {
1673	unsigned long flags = memcg->move_lock_flags;
1674
1675	memcg->move_lock_task = NULL;
1676	memcg->move_lock_flags = 0;
1677
1678	spin_unlock_irqrestore(&memcg->move_lock, flags);
1679	}
1680
1681	rcu_read_unlock();
1682	}
1683	EXPORT_SYMBOL(unlock_page_memcg);
1684
1685	/*
1686	* size of first charge trial. "32" comes from vmscan.c's magic value.
1687	* TODO: maybe necessary to use big numbers in big irons.
1688	*/
1689	#define CHARGE_BATCH 32U
1690	struct memcg_stock_pcp {
1691	struct mem_cgroup *cached; /* this never be root cgroup */
1692	unsigned int nr_pages;
1693	struct work_struct work;
1694	unsigned long flags;
1695	#define FLUSHING_CACHED_CHARGE 0
1696	};
1697	static DEFINE_PER_CPU(struct memcg_stock_pcp, memcg_stock);
1698	static DEFINE_MUTEX(percpu_charge_mutex);
1699
1700	/**
1701	* consume_stock: Try to consume stocked charge on this cpu.
1702	* @memcg: memcg to consume from.
1703	* @nr_pages: how many pages to charge.
1704	*
1705	* The charges will only happen if @memcg matches the current cpu's memcg
1706	* stock, and at least @nr_pages are available in that stock. Failure to
1707	* service an allocation will refill the stock.
1708	*
1709	* returns true if successful, false otherwise.
1710	*/
1711	static bool consume_stock(struct mem_cgroup *memcg, unsigned int nr_pages)
1712	{
1713	struct memcg_stock_pcp *stock;
1714	unsigned long flags;
1715	bool ret = false;
1716
1717	if (nr_pages > CHARGE_BATCH)
1718	return ret;
1719
1720	local_irq_save(flags);
1721
1722	stock = this_cpu_ptr(&memcg_stock);
1723	if (memcg == stock->cached && stock->nr_pages >= nr_pages) {
1724	stock->nr_pages -= nr_pages;
1725	ret = true;
1726	}
1727
1728	local_irq_restore(flags);
1729
1730	return ret;
1731	}
1732
1733	/*
1734	* Returns stocks cached in percpu and reset cached information.
1735	*/
1736	static void drain_stock(struct memcg_stock_pcp *stock)
1737	{
1738	struct mem_cgroup *old = stock->cached;
1739
1740	if (stock->nr_pages) {
1741	page_counter_uncharge(&old->memory, stock->nr_pages);
1742	if (do_memsw_account())
1743	page_counter_uncharge(&old->memsw, stock->nr_pages);
1744	css_put_many(&old->css, stock->nr_pages);
1745	stock->nr_pages = 0;
1746	}
1747	stock->cached = NULL;
1748	}
1749
1750	static void drain_local_stock(struct work_struct *dummy)
1751	{
1752	struct memcg_stock_pcp *stock;
1753	unsigned long flags;
1754
1755	local_irq_save(flags);
1756
1757	stock = this_cpu_ptr(&memcg_stock);
1758	drain_stock(stock);
1759	clear_bit(FLUSHING_CACHED_CHARGE, &stock->flags);
1760
1761	local_irq_restore(flags);
1762	}
1763
1764	/*
1765	* Cache charges(val) to local per_cpu area.
1766	* This will be consumed by consume_stock() function, later.
1767	*/
1768	static void refill_stock(struct mem_cgroup *memcg, unsigned int nr_pages)
1769	{
1770	struct memcg_stock_pcp *stock;
1771	unsigned long flags;
1772
1773	local_irq_save(flags);
1774
1775	stock = this_cpu_ptr(&memcg_stock);
1776	if (stock->cached != memcg) { /* reset if necessary */
1777	drain_stock(stock);
1778	stock->cached = memcg;
1779	}
1780	stock->nr_pages += nr_pages;
1781
1782	local_irq_restore(flags);
1783	}
1784
1785	/*
1786	* Drains all per-CPU charge caches for given root_memcg resp. subtree
1787	* of the hierarchy under it.
1788	*/
1789	static void drain_all_stock(struct mem_cgroup *root_memcg)
1790	{
1791	int cpu, curcpu;
1792
1793	/* If someone's already draining, avoid adding running more workers. */
1794	if (!mutex_trylock(&percpu_charge_mutex))
1795	return;
1796	/* Notify other cpus that system-wide "drain" is running */
1797	get_online_cpus();
1798	curcpu = get_cpu();
1799	for_each_online_cpu(cpu) {
1800	struct memcg_stock_pcp *stock = &per_cpu(memcg_stock, cpu);
1801	struct mem_cgroup *memcg;
1802
1803	memcg = stock->cached;
1804	if (!memcg || !stock->nr_pages)
1805	continue;
1806	if (!mem_cgroup_is_descendant(memcg, root_memcg))
1807	continue;
1808	if (!test_and_set_bit(FLUSHING_CACHED_CHARGE, &stock->flags)) {
1809	if (cpu == curcpu)
1810	drain_local_stock(&stock->work);
1811	else
1812	schedule_work_on(cpu, &stock->work);
1813	}
1814	}
1815	put_cpu();
1816	put_online_cpus();
1817	mutex_unlock(&percpu_charge_mutex);
1818	}
1819
1820	static int memcg_cpu_hotplug_callback(struct notifier_block *nb,
1821	unsigned long action,
1822	void *hcpu)
1823	{
1824	int cpu = (unsigned long)hcpu;
1825	struct memcg_stock_pcp *stock;
1826
1827	if (action == CPU_ONLINE)
1828	return NOTIFY_OK;
1829
1830	if (action != CPU_DEAD && action != CPU_DEAD_FROZEN)
1831	return NOTIFY_OK;
1832
1833	stock = &per_cpu(memcg_stock, cpu);
1834	drain_stock(stock);
1835	return NOTIFY_OK;
1836	}
1837
1838	static void reclaim_high(struct mem_cgroup *memcg,
1839	unsigned int nr_pages,
1840	gfp_t gfp_mask)
1841	{
1842	do {
1843	if (page_counter_read(&memcg->memory) <= memcg->high)
1844	continue;
1845	mem_cgroup_events(memcg, MEMCG_HIGH, 1);
1846	try_to_free_mem_cgroup_pages(memcg, nr_pages, gfp_mask, true);
1847	} while ((memcg = parent_mem_cgroup(memcg)));
1848	}
1849
1850	static void high_work_func(struct work_struct *work)
1851	{
1852	struct mem_cgroup *memcg;
1853
1854	memcg = container_of(work, struct mem_cgroup, high_work);
1855	reclaim_high(memcg, CHARGE_BATCH, GFP_KERNEL);
1856	}
1857
1858	/*
1859	* Scheduled by try_charge() to be executed from the userland return path
1860	* and reclaims memory over the high limit.
1861	*/
1862	void mem_cgroup_handle_over_high(void)
1863	{
1864	unsigned int nr_pages = current->memcg_nr_pages_over_high;
1865	struct mem_cgroup *memcg;
1866
1867	if (likely(!nr_pages))
1868	return;
1869
1870	memcg = get_mem_cgroup_from_mm(current->mm);
1871	reclaim_high(memcg, nr_pages, GFP_KERNEL);
1872	css_put(&memcg->css);
1873	current->memcg_nr_pages_over_high = 0;
1874	}
1875
1876	static int try_charge(struct mem_cgroup *memcg, gfp_t gfp_mask,
1877	unsigned int nr_pages)
1878	{
1879	unsigned int batch = max(CHARGE_BATCH, nr_pages);
1880	int nr_retries = MEM_CGROUP_RECLAIM_RETRIES;
1881	struct mem_cgroup *mem_over_limit;
1882	struct page_counter *counter;
1883	unsigned long nr_reclaimed;
1884	bool may_swap = true;
1885	bool drained = false;
1886
1887	if (mem_cgroup_is_root(memcg))
1888	return 0;
1889	retry:
1890	if (consume_stock(memcg, nr_pages))
1891	return 0;
1892
1893	if (!do_memsw_account() ||
1894	page_counter_try_charge(&memcg->memsw, batch, &counter)) {
1895	if (page_counter_try_charge(&memcg->memory, batch, &counter))
1896	goto done_restock;
1897	if (do_memsw_account())
1898	page_counter_uncharge(&memcg->memsw, batch);
1899	mem_over_limit = mem_cgroup_from_counter(counter, memory);
1900	} else {
1901	mem_over_limit = mem_cgroup_from_counter(counter, memsw);
1902	may_swap = false;
1903	}
1904
1905	if (batch > nr_pages) {
1906	batch = nr_pages;
1907	goto retry;
1908	}
1909
1910	/*
1911	* Unlike in global OOM situations, memcg is not in a physical
1912	* memory shortage. Allow dying and OOM-killed tasks to
1913	* bypass the last charges so that they can exit quickly and
1914	* free their memory.
1915	*/
1916	if (unlikely(test_thread_flag(TIF_MEMDIE) ||
1917	fatal_signal_pending(current) ||
1918	current->flags & PF_EXITING))
1919	goto force;
1920
1921	/*
1922	* Prevent unbounded recursion when reclaim operations need to
1923	* allocate memory. This might exceed the limits temporarily,
1924	* but we prefer facilitating memory reclaim and getting back
1925	* under the limit over triggering OOM kills in these cases.
1926	*/
1927	if (unlikely(current->flags & PF_MEMALLOC))
1928	goto force;
1929
1930	if (unlikely(task_in_memcg_oom(current)))
1931	goto nomem;
1932
1933	if (!gfpflags_allow_blocking(gfp_mask))
1934	goto nomem;
1935
1936	mem_cgroup_events(mem_over_limit, MEMCG_MAX, 1);
1937
1938	nr_reclaimed = try_to_free_mem_cgroup_pages(mem_over_limit, nr_pages,
1939	gfp_mask, may_swap);
1940
1941	if (mem_cgroup_margin(mem_over_limit) >= nr_pages)
1942	goto retry;
1943
1944	if (!drained) {
1945	drain_all_stock(mem_over_limit);
1946	drained = true;
1947	goto retry;
1948	}
1949
1950	if (gfp_mask & __GFP_NORETRY)
1951	goto nomem;
1952	/*
1953	* Even though the limit is exceeded at this point, reclaim
1954	* may have been able to free some pages. Retry the charge
1955	* before killing the task.
1956	*
1957	* Only for regular pages, though: huge pages are rather
1958	* unlikely to succeed so close to the limit, and we fall back
1959	* to regular pages anyway in case of failure.
1960	*/
1961	if (nr_reclaimed && nr_pages <= (1 << PAGE_ALLOC_COSTLY_ORDER))
1962	goto retry;
1963	/*
1964	* At task move, charge accounts can be doubly counted. So, it's
1965	* better to wait until the end of task_move if something is going on.
1966	*/
1967	if (mem_cgroup_wait_acct_move(mem_over_limit))
1968	goto retry;
1969
1970	if (nr_retries--)
1971	goto retry;
1972
1973	if (gfp_mask & __GFP_NOFAIL)
1974	goto force;
1975
1976	if (fatal_signal_pending(current))
1977	goto force;
1978
1979	mem_cgroup_events(mem_over_limit, MEMCG_OOM, 1);
1980
1981	mem_cgroup_oom(mem_over_limit, gfp_mask,
1982	get_order(nr_pages * PAGE_SIZE));
1983	nomem:
1984	if (!(gfp_mask & __GFP_NOFAIL))
1985	return -ENOMEM;
1986	force:
1987	/*
1988	* The allocation either can't fail or will lead to more memory
1989	* being freed very soon. Allow memory usage go over the limit
1990	* temporarily by force charging it.
1991	*/
1992	page_counter_charge(&memcg->memory, nr_pages);
1993	if (do_memsw_account())
1994	page_counter_charge(&memcg->memsw, nr_pages);
1995	css_get_many(&memcg->css, nr_pages);
1996
1997	return 0;
1998
1999	done_restock:
2000	css_get_many(&memcg->css, batch);
2001	if (batch > nr_pages)
2002	refill_stock(memcg, batch - nr_pages);
2003
2004	/*
2005	* If the hierarchy is above the normal consumption range, schedule
2006	* reclaim on returning to userland. We can perform reclaim here
2007	* if __GFP_RECLAIM but let's always punt for simplicity and so that
2008	* GFP_KERNEL can consistently be used during reclaim. @memcg is
2009	* not recorded as it most likely matches current's and won't
2010	* change in the meantime. As high limit is checked again before
2011	* reclaim, the cost of mismatch is negligible.
2012	*/
2013	do {
2014	if (page_counter_read(&memcg->memory) > memcg->high) {
2015	/* Don't bother a random interrupted task */
2016	if (in_interrupt()) {
2017	schedule_work(&memcg->high_work);
2018	break;
2019	}
2020	current->memcg_nr_pages_over_high += batch;
2021	set_notify_resume(current);
2022	break;
2023	}
2024	} while ((memcg = parent_mem_cgroup(memcg)));
2025
2026	return 0;
2027	}
2028
2029	static void cancel_charge(struct mem_cgroup *memcg, unsigned int nr_pages)
2030	{
2031	if (mem_cgroup_is_root(memcg))
2032	return;
2033
2034	page_counter_uncharge(&memcg->memory, nr_pages);
2035	if (do_memsw_account())
2036	page_counter_uncharge(&memcg->memsw, nr_pages);
2037
2038	css_put_many(&memcg->css, nr_pages);
2039	}
2040
2041	static void lock_page_lru(struct page *page, int *isolated)
2042	{
2043	struct zone *zone = page_zone(page);
2044
2045	spin_lock_irq(zone_lru_lock(zone));
2046	if (PageLRU(page)) {
2047	struct lruvec *lruvec;
2048
2049	lruvec = mem_cgroup_page_lruvec(page, zone->zone_pgdat);
2050	ClearPageLRU(page);
2051	del_page_from_lru_list(page, lruvec, page_lru(page));
2052	*isolated = 1;
2053	} else
2054	*isolated = 0;
2055	}
2056
2057	static void unlock_page_lru(struct page *page, int isolated)
2058	{
2059	struct zone *zone = page_zone(page);
2060
2061	if (isolated) {
2062	struct lruvec *lruvec;
2063
2064	lruvec = mem_cgroup_page_lruvec(page, zone->zone_pgdat);
2065	VM_BUG_ON_PAGE(PageLRU(page), page);
2066	SetPageLRU(page);
2067	add_page_to_lru_list(page, lruvec, page_lru(page));
2068	}
2069	spin_unlock_irq(zone_lru_lock(zone));
2070	}
2071
2072	static void commit_charge(struct page *page, struct mem_cgroup *memcg,
2073	bool lrucare)
2074	{
2075	int isolated;
2076
2077	VM_BUG_ON_PAGE(page->mem_cgroup, page);
2078
2079	/*
2080	* In some cases, SwapCache and FUSE(splice_buf->radixtree), the page
2081	* may already be on some other mem_cgroup's LRU. Take care of it.
2082	*/
2083	if (lrucare)
2084	lock_page_lru(page, &isolated);
2085
2086	/*
2087	* Nobody should be changing or seriously looking at
2088	* page->mem_cgroup at this point:
2089	*
2090	* - the page is uncharged
2091	*
2092	* - the page is off-LRU
2093	*
2094	* - an anonymous fault has exclusive page access, except for
2095	* a locked page table
2096	*
2097	* - a page cache insertion, a swapin fault, or a migration
2098	* have the page locked
2099	*/
2100	page->mem_cgroup = memcg;
2101
2102	if (lrucare)
2103	unlock_page_lru(page, isolated);
2104	}
2105
2106	#ifndef CONFIG_SLOB
2107	static int memcg_alloc_cache_id(void)
2108	{
2109	int id, size;
2110	int err;
2111
2112	id = ida_simple_get(&memcg_cache_ida,
2113	0, MEMCG_CACHES_MAX_SIZE, GFP_KERNEL);
2114	if (id < 0)
2115	return id;
2116
2117	if (id < memcg_nr_cache_ids)
2118	return id;
2119
2120	/*
2121	* There's no space for the new id in memcg_caches arrays,
2122	* so we have to grow them.
2123	*/
2124	down_write(&memcg_cache_ids_sem);
2125
2126	size = 2 * (id + 1);
2127	if (size < MEMCG_CACHES_MIN_SIZE)
2128	size = MEMCG_CACHES_MIN_SIZE;
2129	else if (size > MEMCG_CACHES_MAX_SIZE)
2130	size = MEMCG_CACHES_MAX_SIZE;
2131
2132	err = memcg_update_all_caches(size);
2133	if (!err)
2134	err = memcg_update_all_list_lrus(size);
2135	if (!err)
2136	memcg_nr_cache_ids = size;
2137
2138	up_write(&memcg_cache_ids_sem);
2139
2140	if (err) {
2141	ida_simple_remove(&memcg_cache_ida, id);
2142	return err;
2143	}
2144	return id;
2145	}
2146
2147	static void memcg_free_cache_id(int id)
2148	{
2149	ida_simple_remove(&memcg_cache_ida, id);
2150	}
2151
2152	struct memcg_kmem_cache_create_work {
2153	struct mem_cgroup *memcg;
2154	struct kmem_cache *cachep;
2155	struct work_struct work;
2156	};
2157
2158	static struct workqueue_struct *memcg_kmem_cache_create_wq;
2159
2160	static void memcg_kmem_cache_create_func(struct work_struct *w)
2161	{
2162	struct memcg_kmem_cache_create_work *cw =
2163	container_of(w, struct memcg_kmem_cache_create_work, work);
2164	struct mem_cgroup *memcg = cw->memcg;
2165	struct kmem_cache *cachep = cw->cachep;
2166
2167	memcg_create_kmem_cache(memcg, cachep);
2168
2169	css_put(&memcg->css);
2170	kfree(cw);
2171	}
2172
2173	/*
2174	* Enqueue the creation of a per-memcg kmem_cache.
2175	*/
2176	static void __memcg_schedule_kmem_cache_create(struct mem_cgroup *memcg,
2177	struct kmem_cache *cachep)
2178	{
2179	struct memcg_kmem_cache_create_work *cw;
2180
2181	cw = kmalloc(sizeof(*cw), GFP_NOWAIT);
2182	if (!cw)
2183	return;
2184
2185	css_get(&memcg->css);
2186
2187	cw->memcg = memcg;
2188	cw->cachep = cachep;
2189	INIT_WORK(&cw->work, memcg_kmem_cache_create_func);
2190
2191	queue_work(memcg_kmem_cache_create_wq, &cw->work);
2192	}
2193
2194	static void memcg_schedule_kmem_cache_create(struct mem_cgroup *memcg,
2195	struct kmem_cache *cachep)
2196	{
2197	/*
2198	* We need to stop accounting when we kmalloc, because if the
2199	* corresponding kmalloc cache is not yet created, the first allocation
2200	* in __memcg_schedule_kmem_cache_create will recurse.
2201	*
2202	* However, it is better to enclose the whole function. Depending on
2203	* the debugging options enabled, INIT_WORK(), for instance, can
2204	* trigger an allocation. This too, will make us recurse. Because at
2205	* this point we can't allow ourselves back into memcg_kmem_get_cache,
2206	* the safest choice is to do it like this, wrapping the whole function.
2207	*/
2208	current->memcg_kmem_skip_account = 1;
2209	__memcg_schedule_kmem_cache_create(memcg, cachep);
2210	current->memcg_kmem_skip_account = 0;
2211	}
2212
2213	static inline bool memcg_kmem_bypass(void)
2214	{
2215	if (in_interrupt() || !current->mm || (current->flags & PF_KTHREAD))
2216	return true;
2217	return false;
2218	}
2219
2220	/**
2221	* memcg_kmem_get_cache: select the correct per-memcg cache for allocation
2222	* @cachep: the original global kmem cache
2223	*
2224	* Return the kmem_cache we're supposed to use for a slab allocation.
2225	* We try to use the current memcg's version of the cache.
2226	*
2227	* If the cache does not exist yet, if we are the first user of it, we
2228	* create it asynchronously in a workqueue and let the current allocation
2229	* go through with the original cache.
2230	*
2231	* This function takes a reference to the cache it returns to assure it
2232	* won't get destroyed while we are working with it. Once the caller is
2233	* done with it, memcg_kmem_put_cache() must be called to release the
2234	* reference.
2235	*/
2236	struct kmem_cache *memcg_kmem_get_cache(struct kmem_cache *cachep)
2237	{
2238	struct mem_cgroup *memcg;
2239	struct kmem_cache *memcg_cachep;
2240	int kmemcg_id;
2241
2242	VM_BUG_ON(!is_root_cache(cachep));
2243
2244	if (memcg_kmem_bypass())
2245	return cachep;
2246
2247	if (current->memcg_kmem_skip_account)
2248	return cachep;
2249
2250	memcg = get_mem_cgroup_from_mm(current->mm);
2251	kmemcg_id = READ_ONCE(memcg->kmemcg_id);
2252	if (kmemcg_id < 0)
2253	goto out;
2254
2255	memcg_cachep = cache_from_memcg_idx(cachep, kmemcg_id);
2256	if (likely(memcg_cachep))
2257	return memcg_cachep;
2258
2259	/*
2260	* If we are in a safe context (can wait, and not in interrupt
2261	* context), we could be be predictable and return right away.
2262	* This would guarantee that the allocation being performed
2263	* already belongs in the new cache.
2264	*
2265	* However, there are some clashes that can arrive from locking.
2266	* For instance, because we acquire the slab_mutex while doing
2267	* memcg_create_kmem_cache, this means no further allocation
2268	* could happen with the slab_mutex held. So it's better to
2269	* defer everything.
2270	*/
2271	memcg_schedule_kmem_cache_create(memcg, cachep);
2272	out:
2273	css_put(&memcg->css);
2274	return cachep;
2275	}
2276
2277	/**
2278	* memcg_kmem_put_cache: drop reference taken by memcg_kmem_get_cache
2279	* @cachep: the cache returned by memcg_kmem_get_cache
2280	*/
2281	void memcg_kmem_put_cache(struct kmem_cache *cachep)
2282	{
2283	if (!is_root_cache(cachep))
2284	css_put(&cachep->memcg_params.memcg->css);
2285	}
2286
2287	/**
2288	* memcg_kmem_charge: charge a kmem page
2289	* @page: page to charge
2290	* @gfp: reclaim mode
2291	* @order: allocation order
2292	* @memcg: memory cgroup to charge
2293	*
2294	* Returns 0 on success, an error code on failure.
2295	*/
2296	int memcg_kmem_charge_memcg(struct page *page, gfp_t gfp, int order,
2297	struct mem_cgroup *memcg)
2298	{
2299	unsigned int nr_pages = 1 << order;
2300	struct page_counter *counter;
2301	int ret;
2302
2303	ret = try_charge(memcg, gfp, nr_pages);
2304	if (ret)
2305	return ret;
2306
2307	if (!cgroup_subsys_on_dfl(memory_cgrp_subsys) &&
2308	!page_counter_try_charge(&memcg->kmem, nr_pages, &counter)) {
2309	cancel_charge(memcg, nr_pages);
2310	return -ENOMEM;
2311	}
2312
2313	page->mem_cgroup = memcg;
2314
2315	return 0;
2316	}
2317
2318	/**
2319	* memcg_kmem_charge: charge a kmem page to the current memory cgroup
2320	* @page: page to charge
2321	* @gfp: reclaim mode
2322	* @order: allocation order
2323	*
2324	* Returns 0 on success, an error code on failure.
2325	*/
2326	int memcg_kmem_charge(struct page *page, gfp_t gfp, int order)
2327	{
2328	struct mem_cgroup *memcg;
2329	int ret = 0;
2330
2331	if (memcg_kmem_bypass())
2332	return 0;
2333
2334	memcg = get_mem_cgroup_from_mm(current->mm);
2335	if (!mem_cgroup_is_root(memcg)) {
2336	ret = memcg_kmem_charge_memcg(page, gfp, order, memcg);
2337	if (!ret)
2338	__SetPageKmemcg(page);
2339	}
2340	css_put(&memcg->css);
2341	return ret;
2342	}
2343	/**
2344	* memcg_kmem_uncharge: uncharge a kmem page
2345	* @page: page to uncharge
2346	* @order: allocation order
2347	*/
2348	void memcg_kmem_uncharge(struct page *page, int order)
2349	{
2350	struct mem_cgroup *memcg = page->mem_cgroup;
2351	unsigned int nr_pages = 1 << order;
2352
2353	if (!memcg)
2354	return;
2355
2356	VM_BUG_ON_PAGE(mem_cgroup_is_root(memcg), page);
2357
2358	if (!cgroup_subsys_on_dfl(memory_cgrp_subsys))
2359	page_counter_uncharge(&memcg->kmem, nr_pages);
2360
2361	page_counter_uncharge(&memcg->memory, nr_pages);
2362	if (do_memsw_account())
2363	page_counter_uncharge(&memcg->memsw, nr_pages);
2364
2365	page->mem_cgroup = NULL;
2366
2367	/* slab pages do not have PageKmemcg flag set */
2368	if (PageKmemcg(page))
2369	__ClearPageKmemcg(page);
2370
2371	css_put_many(&memcg->css, nr_pages);
2372	}
2373	#endif /* !CONFIG_SLOB */
2374
2375	#ifdef CONFIG_TRANSPARENT_HUGEPAGE
2376
2377	/*
2378	* Because tail pages are not marked as "used", set it. We're under
2379	* zone_lru_lock and migration entries setup in all page mappings.
2380	*/
2381	void mem_cgroup_split_huge_fixup(struct page *head)
2382	{
2383	int i;
2384
2385	if (mem_cgroup_disabled())
2386	return;
2387
2388	for (i = 1; i < HPAGE_PMD_NR; i++)
2389	head[i].mem_cgroup = head->mem_cgroup;
2390
2391	__this_cpu_sub(head->mem_cgroup->stat->count[MEM_CGROUP_STAT_RSS_HUGE],
2392	HPAGE_PMD_NR);
2393	}
2394	#endif /* CONFIG_TRANSPARENT_HUGEPAGE */
2395
2396	#ifdef CONFIG_MEMCG_SWAP
2397	static void mem_cgroup_swap_statistics(struct mem_cgroup *memcg,
2398	bool charge)
2399	{
2400	int val = (charge) ? 1 : -1;
2401	this_cpu_add(memcg->stat->count[MEM_CGROUP_STAT_SWAP], val);
2402	}
2403
2404	/**
2405	* mem_cgroup_move_swap_account - move swap charge and swap_cgroup's record.
2406	* @entry: swap entry to be moved
2407	* @from: mem_cgroup which the entry is moved from
2408	* @to: mem_cgroup which the entry is moved to
2409	*
2410	* It succeeds only when the swap_cgroup's record for this entry is the same
2411	* as the mem_cgroup's id of @from.
2412	*
2413	* Returns 0 on success, -EINVAL on failure.
2414	*
2415	* The caller must have charged to @to, IOW, called page_counter_charge() about
2416	* both res and memsw, and called css_get().
2417	*/
2418	static int mem_cgroup_move_swap_account(swp_entry_t entry,
2419	struct mem_cgroup *from, struct mem_cgroup *to)
2420	{
2421	unsigned short old_id, new_id;
2422
2423	old_id = mem_cgroup_id(from);
2424	new_id = mem_cgroup_id(to);
2425
2426	if (swap_cgroup_cmpxchg(entry, old_id, new_id) == old_id) {
2427	mem_cgroup_swap_statistics(from, false);
2428	mem_cgroup_swap_statistics(to, true);
2429	return 0;
2430	}
2431	return -EINVAL;
2432	}
2433	#else
2434	static inline int mem_cgroup_move_swap_account(swp_entry_t entry,
2435	struct mem_cgroup *from, struct mem_cgroup *to)
2436	{
2437	return -EINVAL;
2438	}
2439	#endif
2440
2441	static DEFINE_MUTEX(memcg_limit_mutex);
2442
2443	static int mem_cgroup_resize_limit(struct mem_cgroup *memcg,
2444	unsigned long limit)
2445	{
2446	unsigned long curusage;
2447	unsigned long oldusage;
2448	bool enlarge = false;
2449	int retry_count;
2450	int ret;
2451
2452	/*
2453	* For keeping hierarchical_reclaim simple, how long we should retry
2454	* is depends on callers. We set our retry-count to be function
2455	* of # of children which we should visit in this loop.
2456	*/
2457	retry_count = MEM_CGROUP_RECLAIM_RETRIES *
2458	mem_cgroup_count_children(memcg);
2459
2460	oldusage = page_counter_read(&memcg->memory);
2461
2462	do {
2463	if (signal_pending(current)) {
2464	ret = -EINTR;
2465	break;
2466	}
2467
2468	mutex_lock(&memcg_limit_mutex);
2469	if (limit > memcg->memsw.limit) {
2470	mutex_unlock(&memcg_limit_mutex);
2471	ret = -EINVAL;
2472	break;
2473	}
2474	if (limit > memcg->memory.limit)
2475	enlarge = true;
2476	ret = page_counter_limit(&memcg->memory, limit);
2477	mutex_unlock(&memcg_limit_mutex);
2478
2479	if (!ret)
2480	break;
2481
2482	try_to_free_mem_cgroup_pages(memcg, 1, GFP_KERNEL, true);
2483
2484	curusage = page_counter_read(&memcg->memory);
2485	/* Usage is reduced ? */
2486	if (curusage >= oldusage)
2487	retry_count--;
2488	else
2489	oldusage = curusage;
2490	} while (retry_count);
2491
2492	if (!ret && enlarge)
2493	memcg_oom_recover(memcg);
2494
2495	return ret;
2496	}
2497
2498	static int mem_cgroup_resize_memsw_limit(struct mem_cgroup *memcg,
2499	unsigned long limit)
2500	{
2501	unsigned long curusage;
2502	unsigned long oldusage;
2503	bool enlarge = false;
2504	int retry_count;
2505	int ret;
2506
2507	/* see mem_cgroup_resize_res_limit */
2508	retry_count = MEM_CGROUP_RECLAIM_RETRIES *
2509	mem_cgroup_count_children(memcg);
2510
2511	oldusage = page_counter_read(&memcg->memsw);
2512
2513	do {
2514	if (signal_pending(current)) {
2515	ret = -EINTR;
2516	break;
2517	}
2518
2519	mutex_lock(&memcg_limit_mutex);
2520	if (limit < memcg->memory.limit) {
2521	mutex_unlock(&memcg_limit_mutex);
2522	ret = -EINVAL;
2523	break;
2524	}
2525	if (limit > memcg->memsw.limit)
2526	enlarge = true;
2527	ret = page_counter_limit(&memcg->memsw, limit);
2528	mutex_unlock(&memcg_limit_mutex);
2529
2530	if (!ret)
2531	break;
2532
2533	try_to_free_mem_cgroup_pages(memcg, 1, GFP_KERNEL, false);
2534
2535	curusage = page_counter_read(&memcg->memsw);
2536	/* Usage is reduced ? */
2537	if (curusage >= oldusage)
2538	retry_count--;
2539	else
2540	oldusage = curusage;
2541	} while (retry_count);
2542
2543	if (!ret && enlarge)
2544	memcg_oom_recover(memcg);
2545
2546	return ret;
2547	}
2548
2549	unsigned long mem_cgroup_soft_limit_reclaim(pg_data_t *pgdat, int order,
2550	gfp_t gfp_mask,
2551	unsigned long *total_scanned)
2552	{
2553	unsigned long nr_reclaimed = 0;
2554	struct mem_cgroup_per_node *mz, *next_mz = NULL;
2555	unsigned long reclaimed;
2556	int loop = 0;
2557	struct mem_cgroup_tree_per_node *mctz;
2558	unsigned long excess;
2559	unsigned long nr_scanned;
2560
2561	if (order > 0)
2562	return 0;
2563
2564	mctz = soft_limit_tree_node(pgdat->node_id);
2565
2566	/*
2567	* Do not even bother to check the largest node if the root
2568	* is empty. Do it lockless to prevent lock bouncing. Races
2569	* are acceptable as soft limit is best effort anyway.
2570	*/
2571	if (!mctz || RB_EMPTY_ROOT(&mctz->rb_root))
2572	return 0;
2573
2574	/*
2575	* This loop can run a while, specially if mem_cgroup's continuously
2576	* keep exceeding their soft limit and putting the system under
2577	* pressure
2578	*/
2579	do {
2580	if (next_mz)
2581	mz = next_mz;
2582	else
2583	mz = mem_cgroup_largest_soft_limit_node(mctz);
2584	if (!mz)
2585	break;
2586
2587	nr_scanned = 0;
2588	reclaimed = mem_cgroup_soft_reclaim(mz->memcg, pgdat,
2589	gfp_mask, &nr_scanned);
2590	nr_reclaimed += reclaimed;
2591	*total_scanned += nr_scanned;
2592	spin_lock_irq(&mctz->lock);
2593	__mem_cgroup_remove_exceeded(mz, mctz);
2594
2595	/*
2596	* If we failed to reclaim anything from this memory cgroup
2597	* it is time to move on to the next cgroup
2598	*/
2599	next_mz = NULL;
2600	if (!reclaimed)
2601	next_mz = __mem_cgroup_largest_soft_limit_node(mctz);
2602
2603	excess = soft_limit_excess(mz->memcg);
2604	/*
2605	* One school of thought says that we should not add
2606	* back the node to the tree if reclaim returns 0.
2607	* But our reclaim could return 0, simply because due
2608	* to priority we are exposing a smaller subset of
2609	* memory to reclaim from. Consider this as a longer
2610	* term TODO.
2611	*/
2612	/* If excess == 0, no tree ops */
2613	__mem_cgroup_insert_exceeded(mz, mctz, excess);
2614	spin_unlock_irq(&mctz->lock);
2615	css_put(&mz->memcg->css);
2616	loop++;
2617	/*
2618	* Could not reclaim anything and there are no more
2619	* mem cgroups to try or we seem to be looping without
2620	* reclaiming anything.
2621	*/
2622	if (!nr_reclaimed &&
2623	(next_mz == NULL ||
2624	loop > MEM_CGROUP_MAX_SOFT_LIMIT_RECLAIM_LOOPS))
2625	break;
2626	} while (!nr_reclaimed);
2627	if (next_mz)
2628	css_put(&next_mz->memcg->css);
2629	return nr_reclaimed;
2630	}
2631
2632	/*
2633	* Test whether @memcg has children, dead or alive. Note that this
2634	* function doesn't care whether @memcg has use_hierarchy enabled and
2635	* returns %true if there are child csses according to the cgroup
2636	* hierarchy. Testing use_hierarchy is the caller's responsiblity.
2637	*/
2638	static inline bool memcg_has_children(struct mem_cgroup *memcg)
2639	{
2640	bool ret;
2641
2642	rcu_read_lock();
2643	ret = css_next_child(NULL, &memcg->css);
2644	rcu_read_unlock();
2645	return ret;
2646	}
2647
2648	/*
2649	* Reclaims as many pages from the given memcg as possible.
2650	*
2651	* Caller is responsible for holding css reference for memcg.
2652	*/
2653	static int mem_cgroup_force_empty(struct mem_cgroup *memcg)
2654	{
2655	int nr_retries = MEM_CGROUP_RECLAIM_RETRIES;
2656
2657	/* we call try-to-free pages for make this cgroup empty */
2658	lru_add_drain_all();
2659	/* try to free all pages in this cgroup */
2660	while (nr_retries && page_counter_read(&memcg->memory)) {
2661	int progress;
2662
2663	if (signal_pending(current))
2664	return -EINTR;
2665
2666	progress = try_to_free_mem_cgroup_pages(memcg, 1,
2667	GFP_KERNEL, true);
2668	if (!progress) {
2669	nr_retries--;
2670	/* maybe some writeback is necessary */
2671	congestion_wait(BLK_RW_ASYNC, HZ/10);
2672	}
2673
2674	}
2675
2676	return 0;
2677	}
2678
2679	static ssize_t mem_cgroup_force_empty_write(struct kernfs_open_file *of,
2680	char *buf, size_t nbytes,
2681	loff_t off)
2682	{
2683	struct mem_cgroup *memcg = mem_cgroup_from_css(of_css(of));
2684
2685	if (mem_cgroup_is_root(memcg))
2686	return -EINVAL;
2687	return mem_cgroup_force_empty(memcg) ?: nbytes;
2688	}
2689
2690	static u64 mem_cgroup_hierarchy_read(struct cgroup_subsys_state *css,
2691	struct cftype *cft)
2692	{
2693	return mem_cgroup_from_css(css)->use_hierarchy;
2694	}
2695
2696	static int mem_cgroup_hierarchy_write(struct cgroup_subsys_state *css,
2697	struct cftype *cft, u64 val)
2698	{
2699	int retval = 0;
2700	struct mem_cgroup *memcg = mem_cgroup_from_css(css);
2701	struct mem_cgroup *parent_memcg = mem_cgroup_from_css(memcg->css.parent);
2702
2703	if (memcg->use_hierarchy == val)
2704	return 0;
2705
2706	/*
2707	* If parent's use_hierarchy is set, we can't make any modifications
2708	* in the child subtrees. If it is unset, then the change can
2709	* occur, provided the current cgroup has no children.
2710	*
2711	* For the root cgroup, parent_mem is NULL, we allow value to be
2712	* set if there are no children.
2713	*/
2714	if ((!parent_memcg || !parent_memcg->use_hierarchy) &&
2715	(val == 1 || val == 0)) {
2716	if (!memcg_has_children(memcg))
2717	memcg->use_hierarchy = val;
2718	else
2719	retval = -EBUSY;
2720	} else
2721	retval = -EINVAL;
2722
2723	return retval;
2724	}
2725
2726	static void tree_stat(struct mem_cgroup *memcg, unsigned long *stat)
2727	{
2728	struct mem_cgroup *iter;
2729	int i;
2730
2731	memset(stat, 0, sizeof(*stat) * MEMCG_NR_STAT);
2732
2733	for_each_mem_cgroup_tree(iter, memcg) {
2734	for (i = 0; i < MEMCG_NR_STAT; i++)
2735	stat[i] += mem_cgroup_read_stat(iter, i);
2736	}
2737	}
2738
2739	static void tree_events(struct mem_cgroup *memcg, unsigned long *events)
2740	{
2741	struct mem_cgroup *iter;
2742	int i;
2743
2744	memset(events, 0, sizeof(*events) * MEMCG_NR_EVENTS);
2745
2746	for_each_mem_cgroup_tree(iter, memcg) {
2747	for (i = 0; i < MEMCG_NR_EVENTS; i++)
2748	events[i] += mem_cgroup_read_events(iter, i);
2749	}
2750	}
2751
2752	static unsigned long mem_cgroup_usage(struct mem_cgroup *memcg, bool swap)
2753	{
2754	unsigned long val = 0;
2755
2756	if (mem_cgroup_is_root(memcg)) {
2757	struct mem_cgroup *iter;
2758
2759	for_each_mem_cgroup_tree(iter, memcg) {
2760	val += mem_cgroup_read_stat(iter,
2761	MEM_CGROUP_STAT_CACHE);
2762	val += mem_cgroup_read_stat(iter,
2763	MEM_CGROUP_STAT_RSS);
2764	if (swap)
2765	val += mem_cgroup_read_stat(iter,
2766	MEM_CGROUP_STAT_SWAP);
2767	}
2768	} else {
2769	if (!swap)
2770	val = page_counter_read(&memcg->memory);
2771	else
2772	val = page_counter_read(&memcg->memsw);
2773	}
2774	return val;
2775	}
2776
2777	enum {
2778	RES_USAGE,
2779	RES_LIMIT,
2780	RES_MAX_USAGE,
2781	RES_FAILCNT,
2782	RES_SOFT_LIMIT,
2783	};
2784
2785	static u64 mem_cgroup_read_u64(struct cgroup_subsys_state *css,
2786	struct cftype *cft)
2787	{
2788	struct mem_cgroup *memcg = mem_cgroup_from_css(css);
2789	struct page_counter *counter;
2790
2791	switch (MEMFILE_TYPE(cft->private)) {
2792	case _MEM:
2793	counter = &memcg->memory;
2794	break;
2795	case _MEMSWAP:
2796	counter = &memcg->memsw;
2797	break;
2798	case _KMEM:
2799	counter = &memcg->kmem;
2800	break;
2801	case _TCP:
2802	counter = &memcg->tcpmem;
2803	break;
2804	default:
2805	BUG();
2806	}
2807
2808	switch (MEMFILE_ATTR(cft->private)) {
2809	case RES_USAGE:
2810	if (counter == &memcg->memory)
2811	return (u64)mem_cgroup_usage(memcg, false) * PAGE_SIZE;
2812	if (counter == &memcg->memsw)
2813	return (u64)mem_cgroup_usage(memcg, true) * PAGE_SIZE;
2814	return (u64)page_counter_read(counter) * PAGE_SIZE;
2815	case RES_LIMIT:
2816	return (u64)counter->limit * PAGE_SIZE;
2817	case RES_MAX_USAGE:
2818	return (u64)counter->watermark * PAGE_SIZE;
2819	case RES_FAILCNT:
2820	return counter->failcnt;
2821	case RES_SOFT_LIMIT:
2822	return (u64)memcg->soft_limit * PAGE_SIZE;
2823	default:
2824	BUG();
2825	}
2826	}
2827
2828	#ifndef CONFIG_SLOB
2829	static int memcg_online_kmem(struct mem_cgroup *memcg)
2830	{
2831	int memcg_id;
2832
2833	if (cgroup_memory_nokmem)
2834	return 0;
2835
2836	BUG_ON(memcg->kmemcg_id >= 0);
2837	BUG_ON(memcg->kmem_state);
2838
2839	memcg_id = memcg_alloc_cache_id();
2840	if (memcg_id < 0)
2841	return memcg_id;
2842
2843	static_branch_inc(&memcg_kmem_enabled_key);
2844	/*
2845	* A memory cgroup is considered kmem-online as soon as it gets
2846	* kmemcg_id. Setting the id after enabling static branching will
2847	* guarantee no one starts accounting before all call sites are
2848	* patched.
2849	*/
2850	memcg->kmemcg_id = memcg_id;
2851	memcg->kmem_state = KMEM_ONLINE;
2852
2853	return 0;
2854	}
2855
2856	static void memcg_offline_kmem(struct mem_cgroup *memcg)
2857	{
2858	struct cgroup_subsys_state *css;
2859	struct mem_cgroup *parent, *child;
2860	int kmemcg_id;
2861
2862	if (memcg->kmem_state != KMEM_ONLINE)
2863	return;
2864	/*
2865	* Clear the online state before clearing memcg_caches array
2866	* entries. The slab_mutex in memcg_deactivate_kmem_caches()
2867	* guarantees that no cache will be created for this cgroup
2868	* after we are done (see memcg_create_kmem_cache()).
2869	*/
2870	memcg->kmem_state = KMEM_ALLOCATED;
2871
2872	memcg_deactivate_kmem_caches(memcg);
2873
2874	kmemcg_id = memcg->kmemcg_id;
2875	BUG_ON(kmemcg_id < 0);
2876
2877	parent = parent_mem_cgroup(memcg);
2878	if (!parent)
2879	parent = root_mem_cgroup;
2880
2881	/*
2882	* Change kmemcg_id of this cgroup and all its descendants to the
2883	* parent's id, and then move all entries from this cgroup's list_lrus
2884	* to ones of the parent. After we have finished, all list_lrus
2885	* corresponding to this cgroup are guaranteed to remain empty. The
2886	* ordering is imposed by list_lru_node->lock taken by
2887	* memcg_drain_all_list_lrus().
2888	*/
2889	rcu_read_lock(); /* can be called from css_free w/o cgroup_mutex */
2890	css_for_each_descendant_pre(css, &memcg->css) {
2891	child = mem_cgroup_from_css(css);
2892	BUG_ON(child->kmemcg_id != kmemcg_id);
2893	child->kmemcg_id = parent->kmemcg_id;
2894	if (!memcg->use_hierarchy)
2895	break;
2896	}
2897	rcu_read_unlock();
2898
2899	memcg_drain_all_list_lrus(kmemcg_id, parent->kmemcg_id);
2900
2901	memcg_free_cache_id(kmemcg_id);
2902	}
2903
2904	static void memcg_free_kmem(struct mem_cgroup *memcg)
2905	{
2906	/* css_alloc() failed, offlining didn't happen */
2907	if (unlikely(memcg->kmem_state == KMEM_ONLINE))
2908	memcg_offline_kmem(memcg);
2909
2910	if (memcg->kmem_state == KMEM_ALLOCATED) {
2911	memcg_destroy_kmem_caches(memcg);
2912	static_branch_dec(&memcg_kmem_enabled_key);
2913	WARN_ON(page_counter_read(&memcg->kmem));
2914	}
2915	}
2916	#else
2917	static int memcg_online_kmem(struct mem_cgroup *memcg)
2918	{
2919	return 0;
2920	}
2921	static void memcg_offline_kmem(struct mem_cgroup *memcg)
2922	{
2923	}
2924	static void memcg_free_kmem(struct mem_cgroup *memcg)
2925	{
2926	}
2927	#endif /* !CONFIG_SLOB */
2928
2929	static int memcg_update_kmem_limit(struct mem_cgroup *memcg,
2930	unsigned long limit)
2931	{
2932	int ret;
2933
2934	mutex_lock(&memcg_limit_mutex);
2935	ret = page_counter_limit(&memcg->kmem, limit);
2936	mutex_unlock(&memcg_limit_mutex);
2937	return ret;
2938	}
2939
2940	static int memcg_update_tcp_limit(struct mem_cgroup *memcg, unsigned long limit)
2941	{
2942	int ret;
2943
2944	mutex_lock(&memcg_limit_mutex);
2945
2946	ret = page_counter_limit(&memcg->tcpmem, limit);
2947	if (ret)
2948	goto out;
2949
2950	if (!memcg->tcpmem_active) {
2951	/*
2952	* The active flag needs to be written after the static_key
2953	* update. This is what guarantees that the socket activation
2954	* function is the last one to run. See mem_cgroup_sk_alloc()
2955	* for details, and note that we don't mark any socket as
2956	* belonging to this memcg until that flag is up.
2957	*
2958	* We need to do this, because static_keys will span multiple
2959	* sites, but we can't control their order. If we mark a socket
2960	* as accounted, but the accounting functions are not patched in
2961	* yet, we'll lose accounting.
2962	*
2963	* We never race with the readers in mem_cgroup_sk_alloc(),
2964	* because when this value change, the code to process it is not
2965	* patched in yet.
2966	*/
2967	static_branch_inc(&memcg_sockets_enabled_key);
2968	memcg->tcpmem_active = true;
2969	}
2970	out:
2971	mutex_unlock(&memcg_limit_mutex);
2972	return ret;
2973	}
2974
2975	/*
2976	* The user of this function is...
2977	* RES_LIMIT.
2978	*/
2979	static ssize_t mem_cgroup_write(struct kernfs_open_file *of,
2980	char *buf, size_t nbytes, loff_t off)
2981	{
2982	struct mem_cgroup *memcg = mem_cgroup_from_css(of_css(of));
2983	unsigned long nr_pages;
2984	int ret;
2985
2986	buf = strstrip(buf);
2987	ret = page_counter_memparse(buf, "-1", &nr_pages);
2988	if (ret)
2989	return ret;
2990
2991	switch (MEMFILE_ATTR(of_cft(of)->private)) {
2992	case RES_LIMIT:
2993	if (mem_cgroup_is_root(memcg)) { /* Can't set limit on root */
2994	ret = -EINVAL;
2995	break;
2996	}
2997	switch (MEMFILE_TYPE(of_cft(of)->private)) {
2998	case _MEM:
2999	ret = mem_cgroup_resize_limit(memcg, nr_pages);
3000	break;
3001	case _MEMSWAP:
3002	ret = mem_cgroup_resize_memsw_limit(memcg, nr_pages);
3003	break;
3004	case _KMEM:
3005	ret = memcg_update_kmem_limit(memcg, nr_pages);
3006	break;
3007	case _TCP:
3008	ret = memcg_update_tcp_limit(memcg, nr_pages);
3009	break;
3010	}
3011	break;
3012	case RES_SOFT_LIMIT:
3013	memcg->soft_limit = nr_pages;
3014	ret = 0;
3015	break;
3016	}
3017	return ret ?: nbytes;
3018	}
3019
3020	static ssize_t mem_cgroup_reset(struct kernfs_open_file *of, char *buf,
3021	size_t nbytes, loff_t off)
3022	{
3023	struct mem_cgroup *memcg = mem_cgroup_from_css(of_css(of));
3024	struct page_counter *counter;
3025
3026	switch (MEMFILE_TYPE(of_cft(of)->private)) {
3027	case _MEM:
3028	counter = &memcg->memory;
3029	break;
3030	case _MEMSWAP:
3031	counter = &memcg->memsw;
3032	break;
3033	case _KMEM:
3034	counter = &memcg->kmem;
3035	break;
3036	case _TCP:
3037	counter = &memcg->tcpmem;
3038	break;
3039	default:
3040	BUG();
3041	}
3042
3043	switch (MEMFILE_ATTR(of_cft(of)->private)) {
3044	case RES_MAX_USAGE:
3045	page_counter_reset_watermark(counter);
3046	break;
3047	case RES_FAILCNT:
3048	counter->failcnt = 0;
3049	break;
3050	default:
3051	BUG();
3052	}
3053
3054	return nbytes;
3055	}
3056
3057	static u64 mem_cgroup_move_charge_read(struct cgroup_subsys_state *css,
3058	struct cftype *cft)
3059	{
3060	return mem_cgroup_from_css(css)->move_charge_at_immigrate;
3061	}
3062
3063	#ifdef CONFIG_MMU
3064	static int mem_cgroup_move_charge_write(struct cgroup_subsys_state *css,
3065	struct cftype *cft, u64 val)
3066	{
3067	struct mem_cgroup *memcg = mem_cgroup_from_css(css);
3068
3069	if (val & ~MOVE_MASK)
3070	return -EINVAL;
3071
3072	/*
3073	* No kind of locking is needed in here, because ->can_attach() will
3074	* check this value once in the beginning of the process, and then carry
3075	* on with stale data. This means that changes to this value will only
3076	* affect task migrations starting after the change.
3077	*/
3078	memcg->move_charge_at_immigrate = val;
3079	return 0;
3080	}
3081	#else
3082	static int mem_cgroup_move_charge_write(struct cgroup_subsys_state *css,
3083	struct cftype *cft, u64 val)
3084	{
3085	return -ENOSYS;
3086	}
3087	#endif
3088
3089	#ifdef CONFIG_NUMA
3090	static int memcg_numa_stat_show(struct seq_file *m, void *v)
3091	{
3092	struct numa_stat {
3093	const char *name;
3094	unsigned int lru_mask;
3095	};
3096
3097	static const struct numa_stat stats[] = {
3098	{ "total", LRU_ALL },
3099	{ "file", LRU_ALL_FILE },
3100	{ "anon", LRU_ALL_ANON },
3101	{ "unevictable", BIT(LRU_UNEVICTABLE) },
3102	};
3103	const struct numa_stat *stat;
3104	int nid;
3105	unsigned long nr;
3106	struct mem_cgroup *memcg = mem_cgroup_from_css(seq_css(m));
3107
3108	for (stat = stats; stat < stats + ARRAY_SIZE(stats); stat++) {
3109	nr = mem_cgroup_nr_lru_pages(memcg, stat->lru_mask);
3110	seq_printf(m, "%s=%lu", stat->name, nr);
3111	for_each_node_state(nid, N_MEMORY) {
3112	nr = mem_cgroup_node_nr_lru_pages(memcg, nid,
3113	stat->lru_mask);
3114	seq_printf(m, " N%d=%lu", nid, nr);
3115	}
3116	seq_putc(m, '\n');
3117	}
3118
3119	for (stat = stats; stat < stats + ARRAY_SIZE(stats); stat++) {
3120	struct mem_cgroup *iter;
3121
3122	nr = 0;
3123	for_each_mem_cgroup_tree(iter, memcg)
3124	nr += mem_cgroup_nr_lru_pages(iter, stat->lru_mask);
3125	seq_printf(m, "hierarchical_%s=%lu", stat->name, nr);
3126	for_each_node_state(nid, N_MEMORY) {
3127	nr = 0;
3128	for_each_mem_cgroup_tree(iter, memcg)
3129	nr += mem_cgroup_node_nr_lru_pages(
3130	iter, nid, stat->lru_mask);
3131	seq_printf(m, " N%d=%lu", nid, nr);
3132	}
3133	seq_putc(m, '\n');
3134	}
3135
3136	return 0;
3137	}
3138	#endif /* CONFIG_NUMA */
3139
3140	static int memcg_stat_show(struct seq_file *m, void *v)
3141	{
3142	struct mem_cgroup *memcg = mem_cgroup_from_css(seq_css(m));
3143	unsigned long memory, memsw;
3144	struct mem_cgroup *mi;
3145	unsigned int i;
3146
3147	BUILD_BUG_ON(ARRAY_SIZE(mem_cgroup_stat_names) !=
3148	MEM_CGROUP_STAT_NSTATS);
3149	BUILD_BUG_ON(ARRAY_SIZE(mem_cgroup_events_names) !=
3150	MEM_CGROUP_EVENTS_NSTATS);
3151	BUILD_BUG_ON(ARRAY_SIZE(mem_cgroup_lru_names) != NR_LRU_LISTS);
3152
3153	for (i = 0; i < MEM_CGROUP_STAT_NSTATS; i++) {
3154	if (i == MEM_CGROUP_STAT_SWAP && !do_memsw_account())
3155	continue;
3156	seq_printf(m, "%s %lu\n", mem_cgroup_stat_names[i],
3157	mem_cgroup_read_stat(memcg, i) * PAGE_SIZE);
3158	}
3159
3160	for (i = 0; i < MEM_CGROUP_EVENTS_NSTATS; i++)
3161	seq_printf(m, "%s %lu\n", mem_cgroup_events_names[i],
3162	mem_cgroup_read_events(memcg, i));
3163
3164	for (i = 0; i < NR_LRU_LISTS; i++)
3165	seq_printf(m, "%s %lu\n", mem_cgroup_lru_names[i],
3166	mem_cgroup_nr_lru_pages(memcg, BIT(i)) * PAGE_SIZE);
3167
3168	/* Hierarchical information */
3169	memory = memsw = PAGE_COUNTER_MAX;
3170	for (mi = memcg; mi; mi = parent_mem_cgroup(mi)) {
3171	memory = min(memory, mi->memory.limit);
3172	memsw = min(memsw, mi->memsw.limit);
3173	}
3174	seq_printf(m, "hierarchical_memory_limit %llu\n",
3175	(u64)memory * PAGE_SIZE);
3176	if (do_memsw_account())
3177	seq_printf(m, "hierarchical_memsw_limit %llu\n",
3178	(u64)memsw * PAGE_SIZE);
3179
3180	for (i = 0; i < MEM_CGROUP_STAT_NSTATS; i++) {
3181	unsigned long long val = 0;
3182
3183	if (i == MEM_CGROUP_STAT_SWAP && !do_memsw_account())
3184	continue;
3185	for_each_mem_cgroup_tree(mi, memcg)
3186	val += mem_cgroup_read_stat(mi, i) * PAGE_SIZE;
3187	seq_printf(m, "total_%s %llu\n", mem_cgroup_stat_names[i], val);
3188	}
3189
3190	for (i = 0; i < MEM_CGROUP_EVENTS_NSTATS; i++) {
3191	unsigned long long val = 0;
3192
3193	for_each_mem_cgroup_tree(mi, memcg)
3194	val += mem_cgroup_read_events(mi, i);
3195	seq_printf(m, "total_%s %llu\n",
3196	mem_cgroup_events_names[i], val);
3197	}
3198
3199	for (i = 0; i < NR_LRU_LISTS; i++) {
3200	unsigned long long val = 0;
3201
3202	for_each_mem_cgroup_tree(mi, memcg)
3203	val += mem_cgroup_nr_lru_pages(mi, BIT(i)) * PAGE_SIZE;
3204	seq_printf(m, "total_%s %llu\n", mem_cgroup_lru_names[i], val);
3205	}
3206
3207	#ifdef CONFIG_DEBUG_VM
3208	{
3209	pg_data_t *pgdat;
3210	struct mem_cgroup_per_node *mz;
3211	struct zone_reclaim_stat *rstat;
3212	unsigned long recent_rotated[2] = {0, 0};
3213	unsigned long recent_scanned[2] = {0, 0};
3214
3215	for_each_online_pgdat(pgdat) {
3216	mz = mem_cgroup_nodeinfo(memcg, pgdat->node_id);
3217	rstat = &mz->lruvec.reclaim_stat;
3218
3219	recent_rotated[0] += rstat->recent_rotated[0];
3220	recent_rotated[1] += rstat->recent_rotated[1];
3221	recent_scanned[0] += rstat->recent_scanned[0];
3222	recent_scanned[1] += rstat->recent_scanned[1];
3223	}
3224	seq_printf(m, "recent_rotated_anon %lu\n", recent_rotated[0]);
3225	seq_printf(m, "recent_rotated_file %lu\n", recent_rotated[1]);
3226	seq_printf(m, "recent_scanned_anon %lu\n", recent_scanned[0]);
3227	seq_printf(m, "recent_scanned_file %lu\n", recent_scanned[1]);
3228	}
3229	#endif
3230
3231	return 0;
3232	}
3233
3234	static u64 mem_cgroup_swappiness_read(struct cgroup_subsys_state *css,
3235	struct cftype *cft)
3236	{
3237	struct mem_cgroup *memcg = mem_cgroup_from_css(css);
3238
3239	return mem_cgroup_swappiness(memcg);
3240	}
3241
3242	static int mem_cgroup_swappiness_write(struct cgroup_subsys_state *css,
3243	struct cftype *cft, u64 val)
3244	{
3245	struct mem_cgroup *memcg = mem_cgroup_from_css(css);
3246
3247	if (val > 100)
3248	return -EINVAL;
3249
3250	if (css->parent)
3251	memcg->swappiness = val;
3252	else
3253	vm_swappiness = val;
3254
3255	return 0;
3256	}
3257
3258	static void __mem_cgroup_threshold(struct mem_cgroup *memcg, bool swap)
3259	{
3260	struct mem_cgroup_threshold_ary *t;
3261	unsigned long usage;
3262	int i;
3263
3264	rcu_read_lock();
3265	if (!swap)
3266	t = rcu_dereference(memcg->thresholds.primary);
3267	else
3268	t = rcu_dereference(memcg->memsw_thresholds.primary);
3269
3270	if (!t)
3271	goto unlock;
3272
3273	usage = mem_cgroup_usage(memcg, swap);
3274
3275	/*
3276	* current_threshold points to threshold just below or equal to usage.
3277	* If it's not true, a threshold was crossed after last
3278	* call of __mem_cgroup_threshold().
3279	*/
3280	i = t->current_threshold;
3281
3282	/*
3283	* Iterate backward over array of thresholds starting from
3284	* current_threshold and check if a threshold is crossed.
3285	* If none of thresholds below usage is crossed, we read
3286	* only one element of the array here.
3287	*/
3288	for (; i >= 0 && unlikely(t->entries[i].threshold > usage); i--)
3289	eventfd_signal(t->entries[i].eventfd, 1);
3290
3291	/* i = current_threshold + 1 */
3292	i++;
3293
3294	/*
3295	* Iterate forward over array of thresholds starting from
3296	* current_threshold+1 and check if a threshold is crossed.
3297	* If none of thresholds above usage is crossed, we read
3298	* only one element of the array here.
3299	*/
3300	for (; i < t->size && unlikely(t->entries[i].threshold <= usage); i++)
3301	eventfd_signal(t->entries[i].eventfd, 1);
3302
3303	/* Update current_threshold */
3304	t->current_threshold = i - 1;
3305	unlock:
3306	rcu_read_unlock();
3307	}
3308
3309	static void mem_cgroup_threshold(struct mem_cgroup *memcg)
3310	{
3311	while (memcg) {
3312	__mem_cgroup_threshold(memcg, false);
3313	if (do_memsw_account())
3314	__mem_cgroup_threshold(memcg, true);
3315
3316	memcg = parent_mem_cgroup(memcg);
3317	}
3318	}
3319
3320	static int compare_thresholds(const void *a, const void *b)
3321	{
3322	const struct mem_cgroup_threshold *_a = a;
3323	const struct mem_cgroup_threshold *_b = b;
3324
3325	if (_a->threshold > _b->threshold)
3326	return 1;
3327
3328	if (_a->threshold < _b->threshold)
3329	return -1;
3330
3331	return 0;
3332	}
3333
3334	static int mem_cgroup_oom_notify_cb(struct mem_cgroup *memcg)
3335	{
3336	struct mem_cgroup_eventfd_list *ev;
3337
3338	spin_lock(&memcg_oom_lock);
3339
3340	list_for_each_entry(ev, &memcg->oom_notify, list)
3341	eventfd_signal(ev->eventfd, 1);
3342
3343	spin_unlock(&memcg_oom_lock);
3344	return 0;
3345	}
3346
3347	static void mem_cgroup_oom_notify(struct mem_cgroup *memcg)
3348	{
3349	struct mem_cgroup *iter;
3350
3351	for_each_mem_cgroup_tree(iter, memcg)
3352	mem_cgroup_oom_notify_cb(iter);
3353	}
3354
3355	static int __mem_cgroup_usage_register_event(struct mem_cgroup *memcg,
3356	struct eventfd_ctx *eventfd, const char *args, enum res_type type)
3357	{
3358	struct mem_cgroup_thresholds *thresholds;
3359	struct mem_cgroup_threshold_ary *new;
3360	unsigned long threshold;
3361	unsigned long usage;
3362	int i, size, ret;
3363
3364	ret = page_counter_memparse(args, "-1", &threshold);
3365	if (ret)
3366	return ret;
3367
3368	mutex_lock(&memcg->thresholds_lock);
3369
3370	if (type == _MEM) {
3371	thresholds = &memcg->thresholds;
3372	usage = mem_cgroup_usage(memcg, false);
3373	} else if (type == _MEMSWAP) {
3374	thresholds = &memcg->memsw_thresholds;
3375	usage = mem_cgroup_usage(memcg, true);
3376	} else
3377	BUG();
3378
3379	/* Check if a threshold crossed before adding a new one */
3380	if (thresholds->primary)
3381	__mem_cgroup_threshold(memcg, type == _MEMSWAP);
3382
3383	size = thresholds->primary ? thresholds->primary->size + 1 : 1;
3384
3385	/* Allocate memory for new array of thresholds */
3386	new = kmalloc(sizeof(*new) + size * sizeof(struct mem_cgroup_threshold),
3387	GFP_KERNEL);
3388	if (!new) {
3389	ret = -ENOMEM;
3390	goto unlock;
3391	}
3392	new->size = size;
3393
3394	/* Copy thresholds (if any) to new array */
3395	if (thresholds->primary) {
3396	memcpy(new->entries, thresholds->primary->entries, (size - 1) *
3397	sizeof(struct mem_cgroup_threshold));
3398	}
3399
3400	/* Add new threshold */
3401	new->entries[size - 1].eventfd = eventfd;
3402	new->entries[size - 1].threshold = threshold;
3403
3404	/* Sort thresholds. Registering of new threshold isn't time-critical */
3405	sort(new->entries, size, sizeof(struct mem_cgroup_threshold),
3406	compare_thresholds, NULL);
3407
3408	/* Find current threshold */
3409	new->current_threshold = -1;
3410	for (i = 0; i < size; i++) {
3411	if (new->entries[i].threshold <= usage) {
3412	/*
3413	* new->current_threshold will not be used until
3414	* rcu_assign_pointer(), so it's safe to increment
3415	* it here.
3416	*/
3417	++new->current_threshold;
3418	} else
3419	break;
3420	}
3421
3422	/* Free old spare buffer and save old primary buffer as spare */
3423	kfree(thresholds->spare);
3424	thresholds->spare = thresholds->primary;
3425
3426	rcu_assign_pointer(thresholds->primary, new);
3427
3428	/* To be sure that nobody uses thresholds */
3429	synchronize_rcu();
3430
3431	unlock:
3432	mutex_unlock(&memcg->thresholds_lock);
3433
3434	return ret;
3435	}
3436
3437	static int mem_cgroup_usage_register_event(struct mem_cgroup *memcg,
3438	struct eventfd_ctx *eventfd, const char *args)
3439	{
3440	return __mem_cgroup_usage_register_event(memcg, eventfd, args, _MEM);
3441	}
3442
3443	static int memsw_cgroup_usage_register_event(struct mem_cgroup *memcg,
3444	struct eventfd_ctx *eventfd, const char *args)
3445	{
3446	return __mem_cgroup_usage_register_event(memcg, eventfd, args, _MEMSWAP);
3447	}
3448
3449	static void __mem_cgroup_usage_unregister_event(struct mem_cgroup *memcg,
3450	struct eventfd_ctx *eventfd, enum res_type type)
3451	{
3452	struct mem_cgroup_thresholds *thresholds;
3453	struct mem_cgroup_threshold_ary *new;
3454	unsigned long usage;
3455	int i, j, size;
3456
3457	mutex_lock(&memcg->thresholds_lock);
3458
3459	if (type == _MEM) {
3460	thresholds = &memcg->thresholds;
3461	usage = mem_cgroup_usage(memcg, false);
3462	} else if (type == _MEMSWAP) {
3463	thresholds = &memcg->memsw_thresholds;
3464	usage = mem_cgroup_usage(memcg, true);
3465	} else
3466	BUG();
3467
3468	if (!thresholds->primary)
3469	goto unlock;
3470
3471	/* Check if a threshold crossed before removing */
3472	__mem_cgroup_threshold(memcg, type == _MEMSWAP);
3473
3474	/* Calculate new number of threshold */
3475	size = 0;
3476	for (i = 0; i < thresholds->primary->size; i++) {
3477	if (thresholds->primary->entries[i].eventfd != eventfd)
3478	size++;
3479	}
3480
3481	new = thresholds->spare;
3482
3483	/* Set thresholds array to NULL if we don't have thresholds */
3484	if (!size) {
3485	kfree(new);
3486	new = NULL;
3487	goto swap_buffers;
3488	}
3489
3490	new->size = size;
3491
3492	/* Copy thresholds and find current threshold */
3493	new->current_threshold = -1;
3494	for (i = 0, j = 0; i < thresholds->primary->size; i++) {
3495	if (thresholds->primary->entries[i].eventfd == eventfd)
3496	continue;
3497
3498	new->entries[j] = thresholds->primary->entries[i];
3499	if (new->entries[j].threshold <= usage) {
3500	/*
3501	* new->current_threshold will not be used
3502	* until rcu_assign_pointer(), so it's safe to increment
3503	* it here.
3504	*/
3505	++new->current_threshold;
3506	}
3507	j++;
3508	}
3509
3510	swap_buffers:
3511	/* Swap primary and spare array */
3512	thresholds->spare = thresholds->primary;
3513
3514	rcu_assign_pointer(thresholds->primary, new);
3515
3516	/* To be sure that nobody uses thresholds */
3517	synchronize_rcu();
3518
3519	/* If all events are unregistered, free the spare array */
3520	if (!new) {
3521	kfree(thresholds->spare);
3522	thresholds->spare = NULL;
3523	}
3524	unlock:
3525	mutex_unlock(&memcg->thresholds_lock);
3526	}
3527
3528	static void mem_cgroup_usage_unregister_event(struct mem_cgroup *memcg,
3529	struct eventfd_ctx *eventfd)
3530	{
3531	return __mem_cgroup_usage_unregister_event(memcg, eventfd, _MEM);
3532	}
3533
3534	static void memsw_cgroup_usage_unregister_event(struct mem_cgroup *memcg,
3535	struct eventfd_ctx *eventfd)
3536	{
3537	return __mem_cgroup_usage_unregister_event(memcg, eventfd, _MEMSWAP);
3538	}
3539
3540	static int mem_cgroup_oom_register_event(struct mem_cgroup *memcg,
3541	struct eventfd_ctx *eventfd, const char *args)
3542	{
3543	struct mem_cgroup_eventfd_list *event;
3544
3545	event = kmalloc(sizeof(*event), GFP_KERNEL);
3546	if (!event)
3547	return -ENOMEM;
3548
3549	spin_lock(&memcg_oom_lock);
3550
3551	event->eventfd = eventfd;
3552	list_add(&event->list, &memcg->oom_notify);
3553
3554	/* already in OOM ? */
3555	if (memcg->under_oom)
3556	eventfd_signal(eventfd, 1);
3557	spin_unlock(&memcg_oom_lock);
3558
3559	return 0;
3560	}
3561
3562	static void mem_cgroup_oom_unregister_event(struct mem_cgroup *memcg,
3563	struct eventfd_ctx *eventfd)
3564	{
3565	struct mem_cgroup_eventfd_list *ev, *tmp;
3566
3567	spin_lock(&memcg_oom_lock);
3568
3569	list_for_each_entry_safe(ev, tmp, &memcg->oom_notify, list) {
3570	if (ev->eventfd == eventfd) {
3571	list_del(&ev->list);
3572	kfree(ev);
3573	}
3574	}
3575
3576	spin_unlock(&memcg_oom_lock);
3577	}
3578
3579	static int mem_cgroup_oom_control_read(struct seq_file *sf, void *v)
3580	{
3581	struct mem_cgroup *memcg = mem_cgroup_from_css(seq_css(sf));
3582
3583	seq_printf(sf, "oom_kill_disable %d\n", memcg->oom_kill_disable);
3584	seq_printf(sf, "under_oom %d\n", (bool)memcg->under_oom);
3585	return 0;
3586	}
3587
3588	static int mem_cgroup_oom_control_write(struct cgroup_subsys_state *css,
3589	struct cftype *cft, u64 val)
3590	{
3591	struct mem_cgroup *memcg = mem_cgroup_from_css(css);
3592
3593	/* cannot set to root cgroup and only 0 and 1 are allowed */
3594	if (!css->parent || !((val == 0) || (val == 1)))
3595	return -EINVAL;
3596
3597	memcg->oom_kill_disable = val;
3598	if (!val)
3599	memcg_oom_recover(memcg);
3600
3601	return 0;
3602	}
3603
3604	#ifdef CONFIG_CGROUP_WRITEBACK
3605
3606	struct list_head *mem_cgroup_cgwb_list(struct mem_cgroup *memcg)
3607	{
3608	return &memcg->cgwb_list;
3609	}
3610
3611	static int memcg_wb_domain_init(struct mem_cgroup *memcg, gfp_t gfp)
3612	{
3613	return wb_domain_init(&memcg->cgwb_domain, gfp);
3614	}
3615
3616	static void memcg_wb_domain_exit(struct mem_cgroup *memcg)
3617	{
3618	wb_domain_exit(&memcg->cgwb_domain);
3619	}
3620
3621	static void memcg_wb_domain_size_changed(struct mem_cgroup *memcg)
3622	{
3623	wb_domain_size_changed(&memcg->cgwb_domain);
3624	}
3625
3626	struct wb_domain *mem_cgroup_wb_domain(struct bdi_writeback *wb)
3627	{
3628	struct mem_cgroup *memcg = mem_cgroup_from_css(wb->memcg_css);
3629
3630	if (!memcg->css.parent)
3631	return NULL;
3632
3633	return &memcg->cgwb_domain;
3634	}
3635
3636	/**
3637	* mem_cgroup_wb_stats - retrieve writeback related stats from its memcg
3638	* @wb: bdi_writeback in question
3639	* @pfilepages: out parameter for number of file pages
3640	* @pheadroom: out parameter for number of allocatable pages according to memcg
3641	* @pdirty: out parameter for number of dirty pages
3642	* @pwriteback: out parameter for number of pages under writeback
3643	*
3644	* Determine the numbers of file, headroom, dirty, and writeback pages in
3645	* @wb's memcg. File, dirty and writeback are self-explanatory. Headroom
3646	* is a bit more involved.
3647	*
3648	* A memcg's headroom is "min(max, high) - used". In the hierarchy, the
3649	* headroom is calculated as the lowest headroom of itself and the
3650	* ancestors. Note that this doesn't consider the actual amount of
3651	* available memory in the system. The caller should further cap
3652	* *@pheadroom accordingly.
3653	*/
3654	void mem_cgroup_wb_stats(struct bdi_writeback *wb, unsigned long *pfilepages,
3655	unsigned long *pheadroom, unsigned long *pdirty,
3656	unsigned long *pwriteback)
3657	{
3658	struct mem_cgroup *memcg = mem_cgroup_from_css(wb->memcg_css);
3659	struct mem_cgroup *parent;
3660
3661	*pdirty = mem_cgroup_read_stat(memcg, MEM_CGROUP_STAT_DIRTY);
3662
3663	/* this should eventually include NR_UNSTABLE_NFS */
3664	*pwriteback = mem_cgroup_read_stat(memcg, MEM_CGROUP_STAT_WRITEBACK);
3665	*pfilepages = mem_cgroup_nr_lru_pages(memcg, (1 << LRU_INACTIVE_FILE) |
3666	(1 << LRU_ACTIVE_FILE));
3667	*pheadroom = PAGE_COUNTER_MAX;
3668
3669	while ((parent = parent_mem_cgroup(memcg))) {
3670	unsigned long ceiling = min(memcg->memory.limit, memcg->high);
3671	unsigned long used = page_counter_read(&memcg->memory);
3672
3673	*pheadroom = min(*pheadroom, ceiling - min(ceiling, used));
3674	memcg = parent;
3675	}
3676	}
3677
3678	#else /* CONFIG_CGROUP_WRITEBACK */
3679
3680	static int memcg_wb_domain_init(struct mem_cgroup *memcg, gfp_t gfp)
3681	{
3682	return 0;
3683	}
3684
3685	static void memcg_wb_domain_exit(struct mem_cgroup *memcg)
3686	{
3687	}
3688
3689	static void memcg_wb_domain_size_changed(struct mem_cgroup *memcg)
3690	{
3691	}
3692
3693	#endif /* CONFIG_CGROUP_WRITEBACK */
3694
3695	/*
3696	* DO NOT USE IN NEW FILES.
3697	*
3698	* "cgroup.event_control" implementation.
3699	*
3700	* This is way over-engineered. It tries to support fully configurable
3701	* events for each user. Such level of flexibility is completely
3702	* unnecessary especially in the light of the planned unified hierarchy.
3703	*
3704	* Please deprecate this and replace with something simpler if at all
3705	* possible.
3706	*/
3707
3708	/*
3709	* Unregister event and free resources.
3710	*
3711	* Gets called from workqueue.
3712	*/
3713	static void memcg_event_remove(struct work_struct *work)
3714	{
3715	struct mem_cgroup_event *event =
3716	container_of(work, struct mem_cgroup_event, remove);
3717	struct mem_cgroup *memcg = event->memcg;
3718
3719	remove_wait_queue(event->wqh, &event->wait);
3720
3721	event->unregister_event(memcg, event->eventfd);
3722
3723	/* Notify userspace the event is going away. */
3724	eventfd_signal(event->eventfd, 1);
3725
3726	eventfd_ctx_put(event->eventfd);
3727	kfree(event);
3728	css_put(&memcg->css);
3729	}
3730
3731	/*
3732	* Gets called on POLLHUP on eventfd when user closes it.
3733	*
3734	* Called with wqh->lock held and interrupts disabled.
3735	*/
3736	static int memcg_event_wake(wait_queue_t *wait, unsigned mode,
3737	int sync, void *key)
3738	{
3739	struct mem_cgroup_event *event =
3740	container_of(wait, struct mem_cgroup_event, wait);
3741	struct mem_cgroup *memcg = event->memcg;
3742	unsigned long flags = (unsigned long)key;
3743
3744	if (flags & POLLHUP) {
3745	/*
3746	* If the event has been detached at cgroup removal, we
3747	* can simply return knowing the other side will cleanup
3748	* for us.
3749	*
3750	* We can't race against event freeing since the other
3751	* side will require wqh->lock via remove_wait_queue(),
3752	* which we hold.
3753	*/
3754	spin_lock(&memcg->event_list_lock);
3755	if (!list_empty(&event->list)) {
3756	list_del_init(&event->list);
3757	/*
3758	* We are in atomic context, but cgroup_event_remove()
3759	* may sleep, so we have to call it in workqueue.
3760	*/
3761	schedule_work(&event->remove);
3762	}
3763	spin_unlock(&memcg->event_list_lock);
3764	}
3765
3766	return 0;
3767	}
3768
3769	static void memcg_event_ptable_queue_proc(struct file *file,
3770	wait_queue_head_t *wqh, poll_table *pt)
3771	{
3772	struct mem_cgroup_event *event =
3773	container_of(pt, struct mem_cgroup_event, pt);
3774
3775	event->wqh = wqh;
3776	add_wait_queue(wqh, &event->wait);
3777	}
3778
3779	/*
3780	* DO NOT USE IN NEW FILES.
3781	*
3782	* Parse input and register new cgroup event handler.
3783	*
3784	* Input must be in format '<event_fd> <control_fd> <args>'.
3785	* Interpretation of args is defined by control file implementation.
3786	*/
3787	static ssize_t memcg_write_event_control(struct kernfs_open_file *of,
3788	char *buf, size_t nbytes, loff_t off)
3789	{
3790	struct cgroup_subsys_state *css = of_css(of);
3791	struct mem_cgroup *memcg = mem_cgroup_from_css(css);
3792	struct mem_cgroup_event *event;
3793	struct cgroup_subsys_state *cfile_css;
3794	unsigned int efd, cfd;
3795	struct fd efile;
3796	struct fd cfile;
3797	const char *name;
3798	char *endp;
3799	int ret;
3800
3801	buf = strstrip(buf);
3802
3803	efd = simple_strtoul(buf, &endp, 10);
3804	if (*endp != ' ')
3805	return -EINVAL;
3806	buf = endp + 1;
3807
3808	cfd = simple_strtoul(buf, &endp, 10);
3809	if ((*endp != ' ') && (*endp != '\0'))
3810	return -EINVAL;
3811	buf = endp + 1;
3812
3813	event = kzalloc(sizeof(*event), GFP_KERNEL);
3814	if (!event)
3815	return -ENOMEM;
3816
3817	event->memcg = memcg;
3818	INIT_LIST_HEAD(&event->list);
3819	init_poll_funcptr(&event->pt, memcg_event_ptable_queue_proc);
3820	init_waitqueue_func_entry(&event->wait, memcg_event_wake);
3821	INIT_WORK(&event->remove, memcg_event_remove);
3822
3823	efile = fdget(efd);
3824	if (!efile.file) {
3825	ret = -EBADF;
3826	goto out_kfree;
3827	}
3828
3829	event->eventfd = eventfd_ctx_fileget(efile.file);
3830	if (IS_ERR(event->eventfd)) {
3831	ret = PTR_ERR(event->eventfd);
3832	goto out_put_efile;
3833	}
3834
3835	cfile = fdget(cfd);
3836	if (!cfile.file) {
3837	ret = -EBADF;
3838	goto out_put_eventfd;
3839	}
3840
3841	/* the process need read permission on control file */
3842	/* AV: shouldn't we check that it's been opened for read instead? */
3843	ret = inode_permission(file_inode(cfile.file), MAY_READ);
3844	if (ret < 0)
3845	goto out_put_cfile;
3846
3847	/*
3848	* Determine the event callbacks and set them in @event. This used
3849	* to be done via struct cftype but cgroup core no longer knows
3850	* about these events. The following is crude but the whole thing
3851	* is for compatibility anyway.
3852	*
3853	* DO NOT ADD NEW FILES.
3854	*/
3855	name = cfile.file->f_path.dentry->d_name.name;
3856
3857	if (!strcmp(name, "memory.usage_in_bytes")) {
3858	event->register_event = mem_cgroup_usage_register_event;
3859	event->unregister_event = mem_cgroup_usage_unregister_event;
3860	} else if (!strcmp(name, "memory.oom_control")) {
3861	event->register_event = mem_cgroup_oom_register_event;
3862	event->unregister_event = mem_cgroup_oom_unregister_event;
3863	} else if (!strcmp(name, "memory.pressure_level")) {
3864	event->register_event = vmpressure_register_event;
3865	event->unregister_event = vmpressure_unregister_event;
3866	} else if (!strcmp(name, "memory.memsw.usage_in_bytes")) {
3867	event->register_event = memsw_cgroup_usage_register_event;
3868	event->unregister_event = memsw_cgroup_usage_unregister_event;
3869	} else {
3870	ret = -EINVAL;
3871	goto out_put_cfile;
3872	}
3873
3874	/*
3875	* Verify @cfile should belong to @css. Also, remaining events are
3876	* automatically removed on cgroup destruction but the removal is
3877	* asynchronous, so take an extra ref on @css.
3878	*/
3879	cfile_css = css_tryget_online_from_dir(cfile.file->f_path.dentry->d_parent,
3880	&memory_cgrp_subsys);
3881	ret = -EINVAL;
3882	if (IS_ERR(cfile_css))
3883	goto out_put_cfile;
3884	if (cfile_css != css) {
3885	css_put(cfile_css);
3886	goto out_put_cfile;
3887	}
3888
3889	ret = event->register_event(memcg, event->eventfd, buf);
3890	if (ret)
3891	goto out_put_css;
3892
3893	efile.file->f_op->poll(efile.file, &event->pt);
3894
3895	spin_lock(&memcg->event_list_lock);
3896	list_add(&event->list, &memcg->event_list);
3897	spin_unlock(&memcg->event_list_lock);
3898
3899	fdput(cfile);
3900	fdput(efile);
3901
3902	return nbytes;
3903
3904	out_put_css:
3905	css_put(css);
3906	out_put_cfile:
3907	fdput(cfile);
3908	out_put_eventfd:
3909	eventfd_ctx_put(event->eventfd);
3910	out_put_efile:
3911	fdput(efile);
3912	out_kfree:
3913	kfree(event);
3914
3915	return ret;
3916	}
3917
3918	static struct cftype mem_cgroup_legacy_files[] = {
3919	{
3920	.name = "usage_in_bytes",
3921	.private = MEMFILE_PRIVATE(_MEM, RES_USAGE),
3922	.read_u64 = mem_cgroup_read_u64,
3923	},
3924	{
3925	.name = "max_usage_in_bytes",
3926	.private = MEMFILE_PRIVATE(_MEM, RES_MAX_USAGE),
3927	.write = mem_cgroup_reset,
3928	.read_u64 = mem_cgroup_read_u64,
3929	},
3930	{
3931	.name = "limit_in_bytes",
3932	.private = MEMFILE_PRIVATE(_MEM, RES_LIMIT),
3933	.write = mem_cgroup_write,
3934	.read_u64 = mem_cgroup_read_u64,
3935	},
3936	{
3937	.name = "soft_limit_in_bytes",
3938	.private = MEMFILE_PRIVATE(_MEM, RES_SOFT_LIMIT),
3939	.write = mem_cgroup_write,
3940	.read_u64 = mem_cgroup_read_u64,
3941	},
3942	{
3943	.name = "failcnt",
3944	.private = MEMFILE_PRIVATE(_MEM, RES_FAILCNT),
3945	.write = mem_cgroup_reset,
3946	.read_u64 = mem_cgroup_read_u64,
3947	},
3948	{
3949	.name = "stat",
3950	.seq_show = memcg_stat_show,
3951	},
3952	{
3953	.name = "force_empty",
3954	.write = mem_cgroup_force_empty_write,
3955	},
3956	{
3957	.name = "use_hierarchy",
3958	.write_u64 = mem_cgroup_hierarchy_write,
3959	.read_u64 = mem_cgroup_hierarchy_read,
3960	},
3961	{
3962	.name = "cgroup.event_control", /* XXX: for compat */
3963	.write = memcg_write_event_control,
3964	.flags = CFTYPE_NO_PREFIX | CFTYPE_WORLD_WRITABLE,
3965	},
3966	{
3967	.name = "swappiness",
3968	.read_u64 = mem_cgroup_swappiness_read,
3969	.write_u64 = mem_cgroup_swappiness_write,
3970	},
3971	{
3972	.name = "move_charge_at_immigrate",
3973	.read_u64 = mem_cgroup_move_charge_read,
3974	.write_u64 = mem_cgroup_move_charge_write,
3975	},
3976	{
3977	.name = "oom_control",
3978	.seq_show = mem_cgroup_oom_control_read,
3979	.write_u64 = mem_cgroup_oom_control_write,
3980	.private = MEMFILE_PRIVATE(_OOM_TYPE, OOM_CONTROL),
3981	},
3982	{
3983	.name = "pressure_level",
3984	},
3985	#ifdef CONFIG_NUMA
3986	{
3987	.name = "numa_stat",
3988	.seq_show = memcg_numa_stat_show,
3989	},
3990	#endif
3991	{
3992	.name = "kmem.limit_in_bytes",
3993	.private = MEMFILE_PRIVATE(_KMEM, RES_LIMIT),
3994	.write = mem_cgroup_write,
3995	.read_u64 = mem_cgroup_read_u64,
3996	},
3997	{
3998	.name = "kmem.usage_in_bytes",
3999	.private = MEMFILE_PRIVATE(_KMEM, RES_USAGE),
4000	.read_u64 = mem_cgroup_read_u64,
4001	},
4002	{
4003	.name = "kmem.failcnt",
4004	.private = MEMFILE_PRIVATE(_KMEM, RES_FAILCNT),
4005	.write = mem_cgroup_reset,
4006	.read_u64 = mem_cgroup_read_u64,
4007	},
4008	{
4009	.name = "kmem.max_usage_in_bytes",
4010	.private = MEMFILE_PRIVATE(_KMEM, RES_MAX_USAGE),
4011	.write = mem_cgroup_reset,
4012	.read_u64 = mem_cgroup_read_u64,
4013	},
4014	#ifdef CONFIG_SLABINFO
4015	{
4016	.name = "kmem.slabinfo",
4017	.seq_start = slab_start,
4018	.seq_next = slab_next,
4019	.seq_stop = slab_stop,
4020	.seq_show = memcg_slab_show,
4021	},
4022	#endif
4023	{
4024	.name = "kmem.tcp.limit_in_bytes",
4025	.private = MEMFILE_PRIVATE(_TCP, RES_LIMIT),
4026	.write = mem_cgroup_write,
4027	.read_u64 = mem_cgroup_read_u64,
4028	},
4029	{
4030	.name = "kmem.tcp.usage_in_bytes",
4031	.private = MEMFILE_PRIVATE(_TCP, RES_USAGE),
4032	.read_u64 = mem_cgroup_read_u64,
4033	},
4034	{
4035	.name = "kmem.tcp.failcnt",
4036	.private = MEMFILE_PRIVATE(_TCP, RES_FAILCNT),
4037	.write = mem_cgroup_reset,
4038	.read_u64 = mem_cgroup_read_u64,
4039	},
4040	{
4041	.name = "kmem.tcp.max_usage_in_bytes",
4042	.private = MEMFILE_PRIVATE(_TCP, RES_MAX_USAGE),
4043	.write = mem_cgroup_reset,
4044	.read_u64 = mem_cgroup_read_u64,
4045	},
4046	{ }, /* terminate */
4047	};
4048
4049	/*
4050	* Private memory cgroup IDR
4051	*
4052	* Swap-out records and page cache shadow entries need to store memcg
4053	* references in constrained space, so we maintain an ID space that is
4054	* limited to 16 bit (MEM_CGROUP_ID_MAX), limiting the total number of
4055	* memory-controlled cgroups to 64k.
4056	*
4057	* However, there usually are many references to the oflline CSS after
4058	* the cgroup has been destroyed, such as page cache or reclaimable
4059	* slab objects, that don't need to hang on to the ID. We want to keep
4060	* those dead CSS from occupying IDs, or we might quickly exhaust the
4061	* relatively small ID space and prevent the creation of new cgroups
4062	* even when there are much fewer than 64k cgroups - possibly none.
4063	*
4064	* Maintain a private 16-bit ID space for memcg, and allow the ID to
4065	* be freed and recycled when it's no longer needed, which is usually
4066	* when the CSS is offlined.
4067	*
4068	* The only exception to that are records of swapped out tmpfs/shmem
4069	* pages that need to be attributed to live ancestors on swapin. But
4070	* those references are manageable from userspace.
4071	*/
4072
4073	static DEFINE_IDR(mem_cgroup_idr);
4074
4075	static void mem_cgroup_id_remove(struct mem_cgroup *memcg)
4076	{
4077	if (memcg->id.id > 0) {
4078	idr_remove(&mem_cgroup_idr, memcg->id.id);
4079	memcg->id.id = 0;
4080	}
4081	}
4082
4083	static void mem_cgroup_id_get_many(struct mem_cgroup *memcg, unsigned int n)
4084	{
4085	VM_BUG_ON(atomic_read(&memcg->id.ref) <= 0);
4086	atomic_add(n, &memcg->id.ref);
4087	}
4088
4089	static void mem_cgroup_id_put_many(struct mem_cgroup *memcg, unsigned int n)
4090	{
4091	VM_BUG_ON(atomic_read(&memcg->id.ref) < n);
4092	if (atomic_sub_and_test(n, &memcg->id.ref)) {
4093	mem_cgroup_id_remove(memcg);
4094
4095	/* Memcg ID pins CSS */
4096	css_put(&memcg->css);
4097	}
4098	}
4099
4100	static inline void mem_cgroup_id_get(struct mem_cgroup *memcg)
4101	{
4102	mem_cgroup_id_get_many(memcg, 1);
4103	}
4104
4105	static inline void mem_cgroup_id_put(struct mem_cgroup *memcg)
4106	{
4107	mem_cgroup_id_put_many(memcg, 1);
4108	}
4109
4110	/**
4111	* mem_cgroup_from_id - look up a memcg from a memcg id
4112	* @id: the memcg id to look up
4113	*
4114	* Caller must hold rcu_read_lock().
4115	*/
4116	struct mem_cgroup *mem_cgroup_from_id(unsigned short id)
4117	{
4118	WARN_ON_ONCE(!rcu_read_lock_held());
4119	return idr_find(&mem_cgroup_idr, id);
4120	}
4121
4122	static int alloc_mem_cgroup_per_node_info(struct mem_cgroup *memcg, int node)
4123	{
4124	struct mem_cgroup_per_node *pn;
4125	int tmp = node;
4126	/*
4127	* This routine is called against possible nodes.
4128	* But it's BUG to call kmalloc() against offline node.
4129	*
4130	* TODO: this routine can waste much memory for nodes which will
4131	* never be onlined. It's better to use memory hotplug callback
4132	* function.
4133	*/
4134	if (!node_state(node, N_NORMAL_MEMORY))
4135	tmp = -1;
4136	pn = kzalloc_node(sizeof(*pn), GFP_KERNEL, tmp);
4137	if (!pn)
4138	return 1;
4139
4140	lruvec_init(&pn->lruvec);
4141	pn->usage_in_excess = 0;
4142	pn->on_tree = false;
4143	pn->memcg = memcg;
4144
4145	memcg->nodeinfo[node] = pn;
4146	return 0;
4147	}
4148
4149	static void free_mem_cgroup_per_node_info(struct mem_cgroup *memcg, int node)
4150	{
4151	kfree(memcg->nodeinfo[node]);
4152	}
4153
4154	static void __mem_cgroup_free(struct mem_cgroup *memcg)
4155	{
4156	int node;
4157
4158	for_each_node(node)
4159	free_mem_cgroup_per_node_info(memcg, node);
4160	free_percpu(memcg->stat);
4161	kfree(memcg);
4162	}
4163
4164	static void mem_cgroup_free(struct mem_cgroup *memcg)
4165	{
4166	memcg_wb_domain_exit(memcg);
4167	__mem_cgroup_free(memcg);
4168	}
4169
4170	static struct mem_cgroup *mem_cgroup_alloc(void)
4171	{
4172	struct mem_cgroup *memcg;
4173	size_t size;
4174	int node;
4175
4176	size = sizeof(struct mem_cgroup);
4177	size += nr_node_ids * sizeof(struct mem_cgroup_per_node *);
4178
4179	memcg = kzalloc(size, GFP_KERNEL);
4180	if (!memcg)
4181	return NULL;
4182
4183	memcg->id.id = idr_alloc(&mem_cgroup_idr, NULL,
4184	1, MEM_CGROUP_ID_MAX,
4185	GFP_KERNEL);
4186	if (memcg->id.id < 0)
4187	goto fail;
4188
4189	memcg->stat = alloc_percpu(struct mem_cgroup_stat_cpu);
4190	if (!memcg->stat)
4191	goto fail;
4192
4193	for_each_node(node)
4194	if (alloc_mem_cgroup_per_node_info(memcg, node))
4195	goto fail;
4196
4197	if (memcg_wb_domain_init(memcg, GFP_KERNEL))
4198	goto fail;
4199
4200	INIT_WORK(&memcg->high_work, high_work_func);
4201	memcg->last_scanned_node = MAX_NUMNODES;
4202	INIT_LIST_HEAD(&memcg->oom_notify);
4203	mutex_init(&memcg->thresholds_lock);
4204	spin_lock_init(&memcg->move_lock);
4205	vmpressure_init(&memcg->vmpressure);
4206	INIT_LIST_HEAD(&memcg->event_list);
4207	spin_lock_init(&memcg->event_list_lock);
4208	memcg->socket_pressure = jiffies;
4209	#ifndef CONFIG_SLOB
4210	memcg->kmemcg_id = -1;
4211	#endif
4212	#ifdef CONFIG_CGROUP_WRITEBACK
4213	INIT_LIST_HEAD(&memcg->cgwb_list);
4214	#endif
4215	idr_replace(&mem_cgroup_idr, memcg, memcg->id.id);
4216	return memcg;
4217	fail:
4218	mem_cgroup_id_remove(memcg);
4219	__mem_cgroup_free(memcg);
4220	return NULL;
4221	}
4222
4223	static struct cgroup_subsys_state * __ref
4224	mem_cgroup_css_alloc(struct cgroup_subsys_state *parent_css)
4225	{
4226	struct mem_cgroup *parent = mem_cgroup_from_css(parent_css);
4227	struct mem_cgroup *memcg;
4228	long error = -ENOMEM;
4229
4230	memcg = mem_cgroup_alloc();
4231	if (!memcg)
4232	return ERR_PTR(error);
4233
4234	memcg->high = PAGE_COUNTER_MAX;
4235	memcg->soft_limit = PAGE_COUNTER_MAX;
4236	if (parent) {
4237	memcg->swappiness = mem_cgroup_swappiness(parent);
4238	memcg->oom_kill_disable = parent->oom_kill_disable;
4239	}
4240	if (parent && parent->use_hierarchy) {
4241	memcg->use_hierarchy = true;
4242	page_counter_init(&memcg->memory, &parent->memory);
4243	page_counter_init(&memcg->swap, &parent->swap);
4244	page_counter_init(&memcg->memsw, &parent->memsw);
4245	page_counter_init(&memcg->kmem, &parent->kmem);
4246	page_counter_init(&memcg->tcpmem, &parent->tcpmem);
4247	} else {
4248	page_counter_init(&memcg->memory, NULL);
4249	page_counter_init(&memcg->swap, NULL);
4250	page_counter_init(&memcg->memsw, NULL);
4251	page_counter_init(&memcg->kmem, NULL);
4252	page_counter_init(&memcg->tcpmem, NULL);
4253	/*
4254	* Deeper hierachy with use_hierarchy == false doesn't make
4255	* much sense so let cgroup subsystem know about this
4256	* unfortunate state in our controller.
4257	*/
4258	if (parent != root_mem_cgroup)
4259	memory_cgrp_subsys.broken_hierarchy = true;
4260	}
4261
4262	/* The following stuff does not apply to the root */
4263	if (!parent) {
4264	root_mem_cgroup = memcg;
4265	return &memcg->css;
4266	}
4267
4268	error = memcg_online_kmem(memcg);
4269	if (error)
4270	goto fail;
4271
4272	if (cgroup_subsys_on_dfl(memory_cgrp_subsys) && !cgroup_memory_nosocket)
4273	static_branch_inc(&memcg_sockets_enabled_key);
4274
4275	return &memcg->css;
4276	fail:
4277	mem_cgroup_id_remove(memcg);
4278	mem_cgroup_free(memcg);
4279	return ERR_PTR(-ENOMEM);
4280	}
4281
4282	static int mem_cgroup_css_online(struct cgroup_subsys_state *css)
4283	{
4284	struct mem_cgroup *memcg = mem_cgroup_from_css(css);
4285
4286	/* Online state pins memcg ID, memcg ID pins CSS */
4287	atomic_set(&memcg->id.ref, 1);
4288	css_get(css);
4289	return 0;
4290	}
4291
4292	static void mem_cgroup_css_offline(struct cgroup_subsys_state *css)
4293	{
4294	struct mem_cgroup *memcg = mem_cgroup_from_css(css);
4295	struct mem_cgroup_event *event, *tmp;
4296
4297	/*
4298	* Unregister events and notify userspace.
4299	* Notify userspace about cgroup removing only after rmdir of cgroup
4300	* directory to avoid race between userspace and kernelspace.
4301	*/
4302	spin_lock(&memcg->event_list_lock);
4303	list_for_each_entry_safe(event, tmp, &memcg->event_list, list) {
4304	list_del_init(&event->list);
4305	schedule_work(&event->remove);
4306	}
4307	spin_unlock(&memcg->event_list_lock);
4308
4309	memcg_offline_kmem(memcg);
4310	wb_memcg_offline(memcg);
4311
4312	mem_cgroup_id_put(memcg);
4313	}
4314
4315	static void mem_cgroup_css_released(struct cgroup_subsys_state *css)
4316	{
4317	struct mem_cgroup *memcg = mem_cgroup_from_css(css);
4318
4319	invalidate_reclaim_iterators(memcg);
4320	}
4321
4322	static void mem_cgroup_css_free(struct cgroup_subsys_state *css)
4323	{
4324	struct mem_cgroup *memcg = mem_cgroup_from_css(css);
4325
4326	if (cgroup_subsys_on_dfl(memory_cgrp_subsys) && !cgroup_memory_nosocket)
4327	static_branch_dec(&memcg_sockets_enabled_key);
4328
4329	if (!cgroup_subsys_on_dfl(memory_cgrp_subsys) && memcg->tcpmem_active)
4330	static_branch_dec(&memcg_sockets_enabled_key);
4331
4332	vmpressure_cleanup(&memcg->vmpressure);
4333	cancel_work_sync(&memcg->high_work);
4334	mem_cgroup_remove_from_trees(memcg);
4335	memcg_free_kmem(memcg);
4336	mem_cgroup_free(memcg);
4337	}
4338
4339	/**
4340	* mem_cgroup_css_reset - reset the states of a mem_cgroup
4341	* @css: the target css
4342	*
4343	* Reset the states of the mem_cgroup associated with @css. This is
4344	* invoked when the userland requests disabling on the default hierarchy
4345	* but the memcg is pinned through dependency. The memcg should stop
4346	* applying policies and should revert to the vanilla state as it may be
4347	* made visible again.
4348	*
4349	* The current implementation only resets the essential configurations.
4350	* This needs to be expanded to cover all the visible parts.
4351	*/
4352	static void mem_cgroup_css_reset(struct cgroup_subsys_state *css)
4353	{
4354	struct mem_cgroup *memcg = mem_cgroup_from_css(css);
4355
4356	page_counter_limit(&memcg->memory, PAGE_COUNTER_MAX);
4357	page_counter_limit(&memcg->swap, PAGE_COUNTER_MAX);
4358	page_counter_limit(&memcg->memsw, PAGE_COUNTER_MAX);
4359	page_counter_limit(&memcg->kmem, PAGE_COUNTER_MAX);
4360	page_counter_limit(&memcg->tcpmem, PAGE_COUNTER_MAX);
4361	memcg->low = 0;
4362	memcg->high = PAGE_COUNTER_MAX;
4363	memcg->soft_limit = PAGE_COUNTER_MAX;
4364	memcg_wb_domain_size_changed(memcg);
4365	}
4366
4367	#ifdef CONFIG_MMU
4368	/* Handlers for move charge at task migration. */
4369	static int mem_cgroup_do_precharge(unsigned long count)
4370	{
4371	int ret;
4372
4373	/* Try a single bulk charge without reclaim first, kswapd may wake */
4374	ret = try_charge(mc.to, GFP_KERNEL & ~__GFP_DIRECT_RECLAIM, count);
4375	if (!ret) {
4376	mc.precharge += count;
4377	return ret;
4378	}
4379
4380	/* Try charges one by one with reclaim, but do not retry */
4381	while (count--) {
4382	ret = try_charge(mc.to, GFP_KERNEL | __GFP_NORETRY, 1);
4383	if (ret)
4384	return ret;
4385	mc.precharge++;
4386	cond_resched();
4387	}
4388	return 0;
4389	}
4390
4391	union mc_target {
4392	struct page *page;
4393	swp_entry_t ent;
4394	};
4395
4396	enum mc_target_type {
4397	MC_TARGET_NONE = 0,
4398	MC_TARGET_PAGE,
4399	MC_TARGET_SWAP,
4400	};
4401
4402	static struct page *mc_handle_present_pte(struct vm_area_struct *vma,
4403	unsigned long addr, pte_t ptent)
4404	{
4405	struct page *page = vm_normal_page(vma, addr, ptent);
4406
4407	if (!page || !page_mapped(page))
4408	return NULL;
4409	if (PageAnon(page)) {
4410	if (!(mc.flags & MOVE_ANON))
4411	return NULL;
4412	} else {
4413	if (!(mc.flags & MOVE_FILE))
4414	return NULL;
4415	}
4416	if (!get_page_unless_zero(page))
4417	return NULL;
4418
4419	return page;
4420	}
4421
4422	#ifdef CONFIG_SWAP
4423	static struct page *mc_handle_swap_pte(struct vm_area_struct *vma,
4424	pte_t ptent, swp_entry_t *entry)
4425	{
4426	struct page *page = NULL;
4427	swp_entry_t ent = pte_to_swp_entry(ptent);
4428
4429	if (!(mc.flags & MOVE_ANON) || non_swap_entry(ent))
4430	return NULL;
4431	/*
4432	* Because lookup_swap_cache() updates some statistics counter,
4433	* we call find_get_page() with swapper_space directly.
4434	*/
4435	page = find_get_page(swap_address_space(ent), swp_offset(ent));
4436	if (do_memsw_account())
4437	entry->val = ent.val;
4438
4439	return page;
4440	}
4441	#else
4442	static struct page *mc_handle_swap_pte(struct vm_area_struct *vma,
4443	pte_t ptent, swp_entry_t *entry)
4444	{
4445	return NULL;
4446	}
4447	#endif
4448
4449	static struct page *mc_handle_file_pte(struct vm_area_struct *vma,
4450	unsigned long addr, pte_t ptent, swp_entry_t *entry)
4451	{
4452	struct page *page = NULL;
4453	struct address_space *mapping;
4454	pgoff_t pgoff;
4455
4456	if (!vma->vm_file) /* anonymous vma */
4457	return NULL;
4458	if (!(mc.flags & MOVE_FILE))
4459	return NULL;
4460
4461	mapping = vma->vm_file->f_mapping;
4462	pgoff = linear_page_index(vma, addr);
4463
4464	/* page is moved even if it's not RSS of this task(page-faulted). */
4465	#ifdef CONFIG_SWAP
4466	/* shmem/tmpfs may report page out on swap: account for that too. */
4467	if (shmem_mapping(mapping)) {
4468	page = find_get_entry(mapping, pgoff);
4469	if (radix_tree_exceptional_entry(page)) {
4470	swp_entry_t swp = radix_to_swp_entry(page);
4471	if (do_memsw_account())
4472	*entry = swp;
4473	page = find_get_page(swap_address_space(swp),
4474	swp_offset(swp));
4475	}
4476	} else
4477	page = find_get_page(mapping, pgoff);
4478	#else
4479	page = find_get_page(mapping, pgoff);
4480	#endif
4481	return page;
4482	}
4483
4484	/**
4485	* mem_cgroup_move_account - move account of the page
4486	* @page: the page
4487	* @compound: charge the page as compound or small page
4488	* @from: mem_cgroup which the page is moved from.
4489	* @to: mem_cgroup which the page is moved to. @from != @to.
4490	*
4491	* The caller must make sure the page is not on LRU (isolate_page() is useful.)
4492	*
4493	* This function doesn't do "charge" to new cgroup and doesn't do "uncharge"
4494	* from old cgroup.
4495	*/
4496	static int mem_cgroup_move_account(struct page *page,
4497	bool compound,
4498	struct mem_cgroup *from,
4499	struct mem_cgroup *to)
4500	{
4501	unsigned long flags;
4502	unsigned int nr_pages = compound ? hpage_nr_pages(page) : 1;
4503	int ret;
4504	bool anon;
4505
4506	VM_BUG_ON(from == to);
4507	VM_BUG_ON_PAGE(PageLRU(page), page);
4508	VM_BUG_ON(compound && !PageTransHuge(page));
4509
4510	/*
4511	* Prevent mem_cgroup_migrate() from looking at
4512	* page->mem_cgroup of its source page while we change it.
4513	*/
4514	ret = -EBUSY;
4515	if (!trylock_page(page))
4516	goto out;
4517
4518	ret = -EINVAL;
4519	if (page->mem_cgroup != from)
4520	goto out_unlock;
4521
4522	anon = PageAnon(page);
4523
4524	spin_lock_irqsave(&from->move_lock, flags);
4525
4526	if (!anon && page_mapped(page)) {
4527	__this_cpu_sub(from->stat->count[MEM_CGROUP_STAT_FILE_MAPPED],
4528	nr_pages);
4529	__this_cpu_add(to->stat->count[MEM_CGROUP_STAT_FILE_MAPPED],
4530	nr_pages);
4531	}
4532
4533	/*
4534	* move_lock grabbed above and caller set from->moving_account, so
4535	* mem_cgroup_update_page_stat() will serialize updates to PageDirty.
4536	* So mapping should be stable for dirty pages.
4537	*/
4538	if (!anon && PageDirty(page)) {
4539	struct address_space *mapping = page_mapping(page);
4540
4541	if (mapping_cap_account_dirty(mapping)) {
4542	__this_cpu_sub(from->stat->count[MEM_CGROUP_STAT_DIRTY],
4543	nr_pages);
4544	__this_cpu_add(to->stat->count[MEM_CGROUP_STAT_DIRTY],
4545	nr_pages);
4546	}
4547	}
4548
4549	if (PageWriteback(page)) {
4550	__this_cpu_sub(from->stat->count[MEM_CGROUP_STAT_WRITEBACK],
4551	nr_pages);
4552	__this_cpu_add(to->stat->count[MEM_CGROUP_STAT_WRITEBACK],
4553	nr_pages);
4554	}
4555
4556	/*
4557	* It is safe to change page->mem_cgroup here because the page
4558	* is referenced, charged, and isolated - we can't race with
4559	* uncharging, charging, migration, or LRU putback.
4560	*/
4561
4562	/* caller should have done css_get */
4563	page->mem_cgroup = to;
4564	spin_unlock_irqrestore(&from->move_lock, flags);
4565
4566	ret = 0;
4567
4568	local_irq_disable();
4569	mem_cgroup_charge_statistics(to, page, compound, nr_pages);
4570	memcg_check_events(to, page);
4571	mem_cgroup_charge_statistics(from, page, compound, -nr_pages);
4572	memcg_check_events(from, page);
4573	local_irq_enable();
4574	out_unlock:
4575	unlock_page(page);
4576	out:
4577	return ret;
4578	}
4579
4580	/**
4581	* get_mctgt_type - get target type of moving charge
4582	* @vma: the vma the pte to be checked belongs
4583	* @addr: the address corresponding to the pte to be checked
4584	* @ptent: the pte to be checked
4585	* @target: the pointer the target page or swap ent will be stored(can be NULL)
4586	*
4587	* Returns
4588	* 0(MC_TARGET_NONE): if the pte is not a target for move charge.
4589	* 1(MC_TARGET_PAGE): if the page corresponding to this pte is a target for
4590	* move charge. if @target is not NULL, the page is stored in target->page
4591	* with extra refcnt got(Callers should handle it).
4592	* 2(MC_TARGET_SWAP): if the swap entry corresponding to this pte is a
4593	* target for charge migration. if @target is not NULL, the entry is stored
4594	* in target->ent.
4595	*
4596	* Called with pte lock held.
4597	*/
4598
4599	static enum mc_target_type get_mctgt_type(struct vm_area_struct *vma,
4600	unsigned long addr, pte_t ptent, union mc_target *target)
4601	{
4602	struct page *page = NULL;
4603	enum mc_target_type ret = MC_TARGET_NONE;
4604	swp_entry_t ent = { .val = 0 };
4605
4606	if (pte_present(ptent))
4607	page = mc_handle_present_pte(vma, addr, ptent);
4608	else if (is_swap_pte(ptent))
4609	page = mc_handle_swap_pte(vma, ptent, &ent);
4610	else if (pte_none(ptent))
4611	page = mc_handle_file_pte(vma, addr, ptent, &ent);
4612
4613	if (!page && !ent.val)
4614	return ret;
4615	if (page) {
4616	/*
4617	* Do only loose check w/o serialization.
4618	* mem_cgroup_move_account() checks the page is valid or
4619	* not under LRU exclusion.
4620	*/
4621	if (page->mem_cgroup == mc.from) {
4622	ret = MC_TARGET_PAGE;
4623	if (target)
4624	target->page = page;
4625	}
4626	if (!ret || !target)
4627	put_page(page);
4628	}
4629	/* There is a swap entry and a page doesn't exist or isn't charged */
4630	if (ent.val && !ret &&
4631	mem_cgroup_id(mc.from) == lookup_swap_cgroup_id(ent)) {
4632	ret = MC_TARGET_SWAP;
4633	if (target)
4634	target->ent = ent;
4635	}
4636	return ret;
4637	}
4638
4639	#ifdef CONFIG_TRANSPARENT_HUGEPAGE
4640	/*
4641	* We don't consider swapping or file mapped pages because THP does not
4642	* support them for now.
4643	* Caller should make sure that pmd_trans_huge(pmd) is true.
4644	*/
4645	static enum mc_target_type get_mctgt_type_thp(struct vm_area_struct *vma,
4646	unsigned long addr, pmd_t pmd, union mc_target *target)
4647	{
4648	struct page *page = NULL;
4649	enum mc_target_type ret = MC_TARGET_NONE;
4650
4651	page = pmd_page(pmd);
4652	VM_BUG_ON_PAGE(!page || !PageHead(page), page);
4653	if (!(mc.flags & MOVE_ANON))
4654	return ret;
4655	if (page->mem_cgroup == mc.from) {
4656	ret = MC_TARGET_PAGE;
4657	if (target) {
4658	get_page(page);
4659	target->page = page;
4660	}
4661	}
4662	return ret;
4663	}
4664	#else
4665	static inline enum mc_target_type get_mctgt_type_thp(struct vm_area_struct *vma,
4666	unsigned long addr, pmd_t pmd, union mc_target *target)
4667	{
4668	return MC_TARGET_NONE;
4669	}
4670	#endif
4671
4672	static int mem_cgroup_count_precharge_pte_range(pmd_t *pmd,
4673	unsigned long addr, unsigned long end,
4674	struct mm_walk *walk)
4675	{
4676	struct vm_area_struct *vma = walk->vma;
4677	pte_t *pte;
4678	spinlock_t *ptl;
4679
4680	ptl = pmd_trans_huge_lock(pmd, vma);
4681	if (ptl) {
4682	if (get_mctgt_type_thp(vma, addr, *pmd, NULL) == MC_TARGET_PAGE)
4683	mc.precharge += HPAGE_PMD_NR;
4684	spin_unlock(ptl);
4685	return 0;
4686	}
4687
4688	if (pmd_trans_unstable(pmd))
4689	return 0;
4690	pte = pte_offset_map_lock(vma->vm_mm, pmd, addr, &ptl);
4691	for (; addr != end; pte++, addr += PAGE_SIZE)
4692	if (get_mctgt_type(vma, addr, *pte, NULL))
4693	mc.precharge++; /* increment precharge temporarily */
4694	pte_unmap_unlock(pte - 1, ptl);
4695	cond_resched();
4696
4697	return 0;
4698	}
4699
4700	static unsigned long mem_cgroup_count_precharge(struct mm_struct *mm)
4701	{
4702	unsigned long precharge;
4703
4704	struct mm_walk mem_cgroup_count_precharge_walk = {
4705	.pmd_entry = mem_cgroup_count_precharge_pte_range,
4706	.mm = mm,
4707	};
4708	down_read(&mm->mmap_sem);
4709	walk_page_range(0, mm->highest_vm_end,
4710	&mem_cgroup_count_precharge_walk);
4711	up_read(&mm->mmap_sem);
4712
4713	precharge = mc.precharge;
4714	mc.precharge = 0;
4715
4716	return precharge;
4717	}
4718
4719	static int mem_cgroup_precharge_mc(struct mm_struct *mm)
4720	{
4721	unsigned long precharge = mem_cgroup_count_precharge(mm);
4722
4723	VM_BUG_ON(mc.moving_task);
4724	mc.moving_task = current;
4725	return mem_cgroup_do_precharge(precharge);
4726	}
4727
4728	/* cancels all extra charges on mc.from and mc.to, and wakes up all waiters. */
4729	static void __mem_cgroup_clear_mc(void)
4730	{
4731	struct mem_cgroup *from = mc.from;
4732	struct mem_cgroup *to = mc.to;
4733
4734	/* we must uncharge all the leftover precharges from mc.to */
4735	if (mc.precharge) {
4736	cancel_charge(mc.to, mc.precharge);
4737	mc.precharge = 0;
4738	}
4739	/*
4740	* we didn't uncharge from mc.from at mem_cgroup_move_account(), so
4741	* we must uncharge here.
4742	*/
4743	if (mc.moved_charge) {
4744	cancel_charge(mc.from, mc.moved_charge);
4745	mc.moved_charge = 0;
4746	}
4747	/* we must fixup refcnts and charges */
4748	if (mc.moved_swap) {
4749	/* uncharge swap account from the old cgroup */
4750	if (!mem_cgroup_is_root(mc.from))
4751	page_counter_uncharge(&mc.from->memsw, mc.moved_swap);
4752
4753	mem_cgroup_id_put_many(mc.from, mc.moved_swap);
4754
4755	/*
4756	* we charged both to->memory and to->memsw, so we
4757	* should uncharge to->memory.
4758	*/
4759	if (!mem_cgroup_is_root(mc.to))
4760	page_counter_uncharge(&mc.to->memory, mc.moved_swap);
4761
4762	mem_cgroup_id_get_many(mc.to, mc.moved_swap);
4763	css_put_many(&mc.to->css, mc.moved_swap);
4764
4765	mc.moved_swap = 0;
4766	}
4767	memcg_oom_recover(from);
4768	memcg_oom_recover(to);
4769	wake_up_all(&mc.waitq);
4770	}
4771
4772	static void mem_cgroup_clear_mc(void)
4773	{
4774	struct mm_struct *mm = mc.mm;
4775
4776	/*
4777	* we must clear moving_task before waking up waiters at the end of
4778	* task migration.
4779	*/
4780	mc.moving_task = NULL;
4781	__mem_cgroup_clear_mc();
4782	spin_lock(&mc.lock);
4783	mc.from = NULL;
4784	mc.to = NULL;
4785	mc.mm = NULL;
4786	spin_unlock(&mc.lock);
4787
4788	mmput(mm);
4789	}
4790
4791	static int mem_cgroup_can_attach(struct cgroup_taskset *tset)
4792	{
4793	struct cgroup_subsys_state *css;
4794	struct mem_cgroup *memcg = NULL; /* unneeded init to make gcc happy */
4795	struct mem_cgroup *from;
4796	struct task_struct *leader, *p;
4797	struct mm_struct *mm;
4798	unsigned long move_flags;
4799	int ret = 0;
4800
4801	/* charge immigration isn't supported on the default hierarchy */
4802	if (cgroup_subsys_on_dfl(memory_cgrp_subsys))
4803	return 0;
4804
4805	/*
4806	* Multi-process migrations only happen on the default hierarchy
4807	* where charge immigration is not used. Perform charge
4808	* immigration if @tset contains a leader and whine if there are
4809	* multiple.
4810	*/
4811	p = NULL;
4812	cgroup_taskset_for_each_leader(leader, css, tset) {
4813	WARN_ON_ONCE(p);
4814	p = leader;
4815	memcg = mem_cgroup_from_css(css);
4816	}
4817	if (!p)
4818	return 0;
4819
4820	/*
4821	* We are now commited to this value whatever it is. Changes in this
4822	* tunable will only affect upcoming migrations, not the current one.
4823	* So we need to save it, and keep it going.
4824	*/
4825	move_flags = READ_ONCE(memcg->move_charge_at_immigrate);
4826	if (!move_flags)
4827	return 0;
4828
4829	from = mem_cgroup_from_task(p);
4830
4831	VM_BUG_ON(from == memcg);
4832
4833	mm = get_task_mm(p);
4834	if (!mm)
4835	return 0;
4836	/* We move charges only when we move a owner of the mm */
4837	if (mm->owner == p) {
4838	VM_BUG_ON(mc.from);
4839	VM_BUG_ON(mc.to);
4840	VM_BUG_ON(mc.precharge);
4841	VM_BUG_ON(mc.moved_charge);
4842	VM_BUG_ON(mc.moved_swap);
4843
4844	spin_lock(&mc.lock);
4845	mc.mm = mm;
4846	mc.from = from;
4847	mc.to = memcg;
4848	mc.flags = move_flags;
4849	spin_unlock(&mc.lock);
4850	/* We set mc.moving_task later */
4851
4852	ret = mem_cgroup_precharge_mc(mm);
4853	if (ret)
4854	mem_cgroup_clear_mc();
4855	} else {
4856	mmput(mm);
4857	}
4858	return ret;
4859	}
4860
4861	static void mem_cgroup_cancel_attach(struct cgroup_taskset *tset)
4862	{
4863	if (mc.to)
4864	mem_cgroup_clear_mc();
4865	}
4866
4867	static int mem_cgroup_move_charge_pte_range(pmd_t *pmd,
4868	unsigned long addr, unsigned long end,
4869	struct mm_walk *walk)
4870	{
4871	int ret = 0;
4872	struct vm_area_struct *vma = walk->vma;
4873	pte_t *pte;
4874	spinlock_t *ptl;
4875	enum mc_target_type target_type;
4876	union mc_target target;
4877	struct page *page;
4878
4879	ptl = pmd_trans_huge_lock(pmd, vma);
4880	if (ptl) {
4881	if (mc.precharge < HPAGE_PMD_NR) {
4882	spin_unlock(ptl);
4883	return 0;
4884	}
4885	target_type = get_mctgt_type_thp(vma, addr, *pmd, &target);
4886	if (target_type == MC_TARGET_PAGE) {
4887	page = target.page;
4888	if (!isolate_lru_page(page)) {
4889	if (!mem_cgroup_move_account(page, true,
4890	mc.from, mc.to)) {
4891	mc.precharge -= HPAGE_PMD_NR;
4892	mc.moved_charge += HPAGE_PMD_NR;
4893	}
4894	putback_lru_page(page);
4895	}
4896	put_page(page);
4897	}
4898	spin_unlock(ptl);
4899	return 0;
4900	}
4901
4902	if (pmd_trans_unstable(pmd))
4903	return 0;
4904	retry:
4905	pte = pte_offset_map_lock(vma->vm_mm, pmd, addr, &ptl);
4906	for (; addr != end; addr += PAGE_SIZE) {
4907	pte_t ptent = *(pte++);
4908	swp_entry_t ent;
4909
4910	if (!mc.precharge)
4911	break;
4912
4913	switch (get_mctgt_type(vma, addr, ptent, &target)) {
4914	case MC_TARGET_PAGE:
4915	page = target.page;
4916	/*
4917	* We can have a part of the split pmd here. Moving it
4918	* can be done but it would be too convoluted so simply
4919	* ignore such a partial THP and keep it in original
4920	* memcg. There should be somebody mapping the head.
4921	*/
4922	if (PageTransCompound(page))
4923	goto put;
4924	if (isolate_lru_page(page))
4925	goto put;
4926	if (!mem_cgroup_move_account(page, false,
4927	mc.from, mc.to)) {
4928	mc.precharge--;
4929	/* we uncharge from mc.from later. */
4930	mc.moved_charge++;
4931	}
4932	putback_lru_page(page);
4933	put: /* get_mctgt_type() gets the page */
4934	put_page(page);
4935	break;
4936	case MC_TARGET_SWAP:
4937	ent = target.ent;
4938	if (!mem_cgroup_move_swap_account(ent, mc.from, mc.to)) {
4939	mc.precharge--;
4940	/* we fixup refcnts and charges later. */
4941	mc.moved_swap++;
4942	}
4943	break;
4944	default:
4945	break;
4946	}
4947	}
4948	pte_unmap_unlock(pte - 1, ptl);
4949	cond_resched();
4950
4951	if (addr != end) {
4952	/*
4953	* We have consumed all precharges we got in can_attach().
4954	* We try charge one by one, but don't do any additional
4955	* charges to mc.to if we have failed in charge once in attach()
4956	* phase.
4957	*/
4958	ret = mem_cgroup_do_precharge(1);
4959	if (!ret)
4960	goto retry;
4961	}
4962
4963	return ret;
4964	}
4965
4966	static void mem_cgroup_move_charge(void)
4967	{
4968	struct mm_walk mem_cgroup_move_charge_walk = {
4969	.pmd_entry = mem_cgroup_move_charge_pte_range,
4970	.mm = mc.mm,
4971	};
4972
4973	lru_add_drain_all();
4974	/*
4975	* Signal lock_page_memcg() to take the memcg's move_lock
4976	* while we're moving its pages to another memcg. Then wait
4977	* for already started RCU-only updates to finish.
4978	*/
4979	atomic_inc(&mc.from->moving_account);
4980	synchronize_rcu();
4981	retry:
4982	if (unlikely(!down_read_trylock(&mc.mm->mmap_sem))) {
4983	/*
4984	* Someone who are holding the mmap_sem might be waiting in
4985	* waitq. So we cancel all extra charges, wake up all waiters,
4986	* and retry. Because we cancel precharges, we might not be able
4987	* to move enough charges, but moving charge is a best-effort
4988	* feature anyway, so it wouldn't be a big problem.
4989	*/
4990	__mem_cgroup_clear_mc();
4991	cond_resched();
4992	goto retry;
4993	}
4994	/*
4995	* When we have consumed all precharges and failed in doing
4996	* additional charge, the page walk just aborts.
4997	*/
4998	walk_page_range(0, mc.mm->highest_vm_end, &mem_cgroup_move_charge_walk);
4999
5000	up_read(&mc.mm->mmap_sem);
5001	atomic_dec(&mc.from->moving_account);
5002	}
5003
5004	static void mem_cgroup_move_task(void)
5005	{
5006	if (mc.to) {
5007	mem_cgroup_move_charge();
5008	mem_cgroup_clear_mc();
5009	}
5010	}
5011	#else /* !CONFIG_MMU */
5012	static int mem_cgroup_can_attach(struct cgroup_taskset *tset)
5013	{
5014	return 0;
5015	}
5016	static void mem_cgroup_cancel_attach(struct cgroup_taskset *tset)
5017	{
5018	}
5019	static void mem_cgroup_move_task(void)
5020	{
5021	}
5022	#endif
5023
5024	/*
5025	* Cgroup retains root cgroups across [un]mount cycles making it necessary
5026	* to verify whether we're attached to the default hierarchy on each mount
5027	* attempt.
5028	*/
5029	static void mem_cgroup_bind(struct cgroup_subsys_state *root_css)
5030	{
5031	/*
5032	* use_hierarchy is forced on the default hierarchy. cgroup core
5033	* guarantees that @root doesn't have any children, so turning it
5034	* on for the root memcg is enough.
5035	*/
5036	if (cgroup_subsys_on_dfl(memory_cgrp_subsys))
5037	root_mem_cgroup->use_hierarchy = true;
5038	else
5039	root_mem_cgroup->use_hierarchy = false;
5040	}
5041
5042	static u64 memory_current_read(struct cgroup_subsys_state *css,
5043	struct cftype *cft)
5044	{
5045	struct mem_cgroup *memcg = mem_cgroup_from_css(css);
5046
5047	return (u64)page_counter_read(&memcg->memory) * PAGE_SIZE;
5048	}
5049
5050	static int memory_low_show(struct seq_file *m, void *v)
5051	{
5052	struct mem_cgroup *memcg = mem_cgroup_from_css(seq_css(m));
5053	unsigned long low = READ_ONCE(memcg->low);
5054
5055	if (low == PAGE_COUNTER_MAX)
5056	seq_puts(m, "max\n");
5057	else
5058	seq_printf(m, "%llu\n", (u64)low * PAGE_SIZE);
5059
5060	return 0;
5061	}
5062
5063	static ssize_t memory_low_write(struct kernfs_open_file *of,
5064	char *buf, size_t nbytes, loff_t off)
5065	{
5066	struct mem_cgroup *memcg = mem_cgroup_from_css(of_css(of));
5067	unsigned long low;
5068	int err;
5069
5070	buf = strstrip(buf);
5071	err = page_counter_memparse(buf, "max", &low);
5072	if (err)
5073	return err;
5074
5075	memcg->low = low;
5076
5077	return nbytes;
5078	}
5079
5080	static int memory_high_show(struct seq_file *m, void *v)
5081	{
5082	struct mem_cgroup *memcg = mem_cgroup_from_css(seq_css(m));
5083	unsigned long high = READ_ONCE(memcg->high);
5084
5085	if (high == PAGE_COUNTER_MAX)
5086	seq_puts(m, "max\n");
5087	else
5088	seq_printf(m, "%llu\n", (u64)high * PAGE_SIZE);
5089
5090	return 0;
5091	}
5092
5093	static ssize_t memory_high_write(struct kernfs_open_file *of,
5094	char *buf, size_t nbytes, loff_t off)
5095	{
5096	struct mem_cgroup *memcg = mem_cgroup_from_css(of_css(of));
5097	unsigned long nr_pages;
5098	unsigned long high;
5099	int err;
5100
5101	buf = strstrip(buf);
5102	err = page_counter_memparse(buf, "max", &high);
5103	if (err)
5104	return err;
5105
5106	memcg->high = high;
5107
5108	nr_pages = page_counter_read(&memcg->memory);
5109	if (nr_pages > high)
5110	try_to_free_mem_cgroup_pages(memcg, nr_pages - high,
5111	GFP_KERNEL, true);
5112
5113	memcg_wb_domain_size_changed(memcg);
5114	return nbytes;
5115	}
5116
5117	static int memory_max_show(struct seq_file *m, void *v)
5118	{
5119	struct mem_cgroup *memcg = mem_cgroup_from_css(seq_css(m));
5120	unsigned long max = READ_ONCE(memcg->memory.limit);
5121
5122	if (max == PAGE_COUNTER_MAX)
5123	seq_puts(m, "max\n");
5124	else
5125	seq_printf(m, "%llu\n", (u64)max * PAGE_SIZE);
5126
5127	return 0;
5128	}
5129
5130	static ssize_t memory_max_write(struct kernfs_open_file *of,
5131	char *buf, size_t nbytes, loff_t off)
5132	{
5133	struct mem_cgroup *memcg = mem_cgroup_from_css(of_css(of));
5134	unsigned int nr_reclaims = MEM_CGROUP_RECLAIM_RETRIES;
5135	bool drained = false;
5136	unsigned long max;
5137	int err;
5138
5139	buf = strstrip(buf);
5140	err = page_counter_memparse(buf, "max", &max);
5141	if (err)
5142	return err;
5143
5144	xchg(&memcg->memory.limit, max);
5145
5146	for (;;) {
5147	unsigned long nr_pages = page_counter_read(&memcg->memory);
5148
5149	if (nr_pages <= max)
5150	break;
5151
5152	if (signal_pending(current)) {
5153	err = -EINTR;
5154	break;
5155	}
5156
5157	if (!drained) {
5158	drain_all_stock(memcg);
5159	drained = true;
5160	continue;
5161	}
5162
5163	if (nr_reclaims) {
5164	if (!try_to_free_mem_cgroup_pages(memcg, nr_pages - max,
5165	GFP_KERNEL, true))
5166	nr_reclaims--;
5167	continue;
5168	}
5169
5170	mem_cgroup_events(memcg, MEMCG_OOM, 1);
5171	if (!mem_cgroup_out_of_memory(memcg, GFP_KERNEL, 0))
5172	break;
5173	}
5174
5175	memcg_wb_domain_size_changed(memcg);
5176	return nbytes;
5177	}
5178
5179	static int memory_events_show(struct seq_file *m, void *v)
5180	{
5181	struct mem_cgroup *memcg = mem_cgroup_from_css(seq_css(m));
5182
5183	seq_printf(m, "low %lu\n", mem_cgroup_read_events(memcg, MEMCG_LOW));
5184	seq_printf(m, "high %lu\n", mem_cgroup_read_events(memcg, MEMCG_HIGH));
5185	seq_printf(m, "max %lu\n", mem_cgroup_read_events(memcg, MEMCG_MAX));
5186	seq_printf(m, "oom %lu\n", mem_cgroup_read_events(memcg, MEMCG_OOM));
5187
5188	return 0;
5189	}
5190
5191	static int memory_stat_show(struct seq_file *m, void *v)
5192	{
5193	struct mem_cgroup *memcg = mem_cgroup_from_css(seq_css(m));
5194	unsigned long stat[MEMCG_NR_STAT];
5195	unsigned long events[MEMCG_NR_EVENTS];
5196	int i;
5197
5198	/*
5199	* Provide statistics on the state of the memory subsystem as
5200	* well as cumulative event counters that show past behavior.
5201	*
5202	* This list is ordered following a combination of these gradients:
5203	* 1) generic big picture -> specifics and details
5204	* 2) reflecting userspace activity -> reflecting kernel heuristics
5205	*
5206	* Current memory state:
5207	*/
5208
5209	tree_stat(memcg, stat);
5210	tree_events(memcg, events);
5211
5212	seq_printf(m, "anon %llu\n",
5213	(u64)stat[MEM_CGROUP_STAT_RSS] * PAGE_SIZE);
5214	seq_printf(m, "file %llu\n",
5215	(u64)stat[MEM_CGROUP_STAT_CACHE] * PAGE_SIZE);
5216	seq_printf(m, "kernel_stack %llu\n",
5217	(u64)stat[MEMCG_KERNEL_STACK_KB] * 1024);
5218	seq_printf(m, "slab %llu\n",
5219	(u64)(stat[MEMCG_SLAB_RECLAIMABLE] +
5220	stat[MEMCG_SLAB_UNRECLAIMABLE]) * PAGE_SIZE);
5221	seq_printf(m, "sock %llu\n",
5222	(u64)stat[MEMCG_SOCK] * PAGE_SIZE);
5223
5224	seq_printf(m, "file_mapped %llu\n",
5225	(u64)stat[MEM_CGROUP_STAT_FILE_MAPPED] * PAGE_SIZE);
5226	seq_printf(m, "file_dirty %llu\n",
5227	(u64)stat[MEM_CGROUP_STAT_DIRTY] * PAGE_SIZE);
5228	seq_printf(m, "file_writeback %llu\n",
5229	(u64)stat[MEM_CGROUP_STAT_WRITEBACK] * PAGE_SIZE);
5230
5231	for (i = 0; i < NR_LRU_LISTS; i++) {
5232	struct mem_cgroup *mi;
5233	unsigned long val = 0;
5234
5235	for_each_mem_cgroup_tree(mi, memcg)
5236	val += mem_cgroup_nr_lru_pages(mi, BIT(i));
5237	seq_printf(m, "%s %llu\n",
5238	mem_cgroup_lru_names[i], (u64)val * PAGE_SIZE);
5239	}
5240
5241	seq_printf(m, "slab_reclaimable %llu\n",
5242	(u64)stat[MEMCG_SLAB_RECLAIMABLE] * PAGE_SIZE);
5243	seq_printf(m, "slab_unreclaimable %llu\n",
5244	(u64)stat[MEMCG_SLAB_UNRECLAIMABLE] * PAGE_SIZE);
5245
5246	/* Accumulated memory events */
5247
5248	seq_printf(m, "pgfault %lu\n",
5249	events[MEM_CGROUP_EVENTS_PGFAULT]);
5250	seq_printf(m, "pgmajfault %lu\n",
5251	events[MEM_CGROUP_EVENTS_PGMAJFAULT]);
5252
5253	return 0;
5254	}
5255
5256	static struct cftype memory_files[] = {
5257	{
5258	.name = "current",
5259	.flags = CFTYPE_NOT_ON_ROOT,
5260	.read_u64 = memory_current_read,
5261	},
5262	{
5263	.name = "low",
5264	.flags = CFTYPE_NOT_ON_ROOT,
5265	.seq_show = memory_low_show,
5266	.write = memory_low_write,
5267	},
5268	{
5269	.name = "high",
5270	.flags = CFTYPE_NOT_ON_ROOT,
5271	.seq_show = memory_high_show,
5272	.write = memory_high_write,
5273	},
5274	{
5275	.name = "max",
5276	.flags = CFTYPE_NOT_ON_ROOT,
5277	.seq_show = memory_max_show,
5278	.write = memory_max_write,
5279	},
5280	{
5281	.name = "events",
5282	.flags = CFTYPE_NOT_ON_ROOT,
5283	.file_offset = offsetof(struct mem_cgroup, events_file),
5284	.seq_show = memory_events_show,
5285	},
5286	{
5287	.name = "stat",
5288	.flags = CFTYPE_NOT_ON_ROOT,
5289	.seq_show = memory_stat_show,
5290	},
5291	{ } /* terminate */
5292	};
5293
5294	struct cgroup_subsys memory_cgrp_subsys = {
5295	.css_alloc = mem_cgroup_css_alloc,
5296	.css_online = mem_cgroup_css_online,
5297	.css_offline = mem_cgroup_css_offline,
5298	.css_released = mem_cgroup_css_released,
5299	.css_free = mem_cgroup_css_free,
5300	.css_reset = mem_cgroup_css_reset,
5301	.can_attach = mem_cgroup_can_attach,
5302	.cancel_attach = mem_cgroup_cancel_attach,
5303	.post_attach = mem_cgroup_move_task,
5304	.bind = mem_cgroup_bind,
5305	.dfl_cftypes = memory_files,
5306	.legacy_cftypes = mem_cgroup_legacy_files,
5307	.early_init = 0,
5308	};
5309
5310	/**
5311	* mem_cgroup_low - check if memory consumption is below the normal range
5312	* @root: the highest ancestor to consider
5313	* @memcg: the memory cgroup to check
5314	*
5315	* Returns %true if memory consumption of @memcg, and that of all
5316	* configurable ancestors up to @root, is below the normal range.
5317	*/
5318	bool mem_cgroup_low(struct mem_cgroup *root, struct mem_cgroup *memcg)
5319	{
5320	if (mem_cgroup_disabled())
5321	return false;
5322
5323	/*
5324	* The toplevel group doesn't have a configurable range, so
5325	* it's never low when looked at directly, and it is not
5326	* considered an ancestor when assessing the hierarchy.
5327	*/
5328
5329	if (memcg == root_mem_cgroup)
5330	return false;
5331
5332	if (page_counter_read(&memcg->memory) >= memcg->low)
5333	return false;
5334
5335	while (memcg != root) {
5336	memcg = parent_mem_cgroup(memcg);
5337
5338	if (memcg == root_mem_cgroup)
5339	break;
5340
5341	if (page_counter_read(&memcg->memory) >= memcg->low)
5342	return false;
5343	}
5344	return true;
5345	}
5346
5347	/**
5348	* mem_cgroup_try_charge - try charging a page
5349	* @page: page to charge
5350	* @mm: mm context of the victim
5351	* @gfp_mask: reclaim mode
5352	* @memcgp: charged memcg return
5353	* @compound: charge the page as compound or small page
5354	*
5355	* Try to charge @page to the memcg that @mm belongs to, reclaiming
5356	* pages according to @gfp_mask if necessary.
5357	*
5358	* Returns 0 on success, with *@memcgp pointing to the charged memcg.
5359	* Otherwise, an error code is returned.
5360	*
5361	* After page->mapping has been set up, the caller must finalize the
5362	* charge with mem_cgroup_commit_charge(). Or abort the transaction
5363	* with mem_cgroup_cancel_charge() in case page instantiation fails.
5364	*/
5365	int mem_cgroup_try_charge(struct page *page, struct mm_struct *mm,
5366	gfp_t gfp_mask, struct mem_cgroup **memcgp,
5367	bool compound)
5368	{
5369	struct mem_cgroup *memcg = NULL;
5370	unsigned int nr_pages = compound ? hpage_nr_pages(page) : 1;
5371	int ret = 0;
5372
5373	if (mem_cgroup_disabled())
5374	goto out;
5375
5376	if (PageSwapCache(page)) {
5377	/*
5378	* Every swap fault against a single page tries to charge the
5379	* page, bail as early as possible. shmem_unuse() encounters
5380	* already charged pages, too. The USED bit is protected by
5381	* the page lock, which serializes swap cache removal, which
5382	* in turn serializes uncharging.
5383	*/
5384	VM_BUG_ON_PAGE(!PageLocked(page), page);
5385	if (page->mem_cgroup)
5386	goto out;
5387
5388	if (do_swap_account) {
5389	swp_entry_t ent = { .val = page_private(page), };
5390	unsigned short id = lookup_swap_cgroup_id(ent);
5391
5392	rcu_read_lock();
5393	memcg = mem_cgroup_from_id(id);
5394	if (memcg && !css_tryget_online(&memcg->css))
5395	memcg = NULL;
5396	rcu_read_unlock();
5397	}
5398	}
5399
5400	if (!memcg)
5401	memcg = get_mem_cgroup_from_mm(mm);
5402
5403	ret = try_charge(memcg, gfp_mask, nr_pages);
5404
5405	css_put(&memcg->css);
5406	out:
5407	*memcgp = memcg;
5408	return ret;
5409	}
5410
5411	/**
5412	* mem_cgroup_commit_charge - commit a page charge
5413	* @page: page to charge
5414	* @memcg: memcg to charge the page to
5415	* @lrucare: page might be on LRU already
5416	* @compound: charge the page as compound or small page
5417	*
5418	* Finalize a charge transaction started by mem_cgroup_try_charge(),
5419	* after page->mapping has been set up. This must happen atomically
5420	* as part of the page instantiation, i.e. under the page table lock
5421	* for anonymous pages, under the page lock for page and swap cache.
5422	*
5423	* In addition, the page must not be on the LRU during the commit, to
5424	* prevent racing with task migration. If it might be, use @lrucare.
5425	*
5426	* Use mem_cgroup_cancel_charge() to cancel the transaction instead.
5427	*/
5428	void mem_cgroup_commit_charge(struct page *page, struct mem_cgroup *memcg,
5429	bool lrucare, bool compound)
5430	{
5431	unsigned int nr_pages = compound ? hpage_nr_pages(page) : 1;
5432
5433	VM_BUG_ON_PAGE(!page->mapping, page);
5434	VM_BUG_ON_PAGE(PageLRU(page) && !lrucare, page);
5435
5436	if (mem_cgroup_disabled())
5437	return;
5438	/*
5439	* Swap faults will attempt to charge the same page multiple
5440	* times. But reuse_swap_page() might have removed the page
5441	* from swapcache already, so we can't check PageSwapCache().
5442	*/
5443	if (!memcg)
5444	return;
5445
5446	commit_charge(page, memcg, lrucare);
5447
5448	local_irq_disable();
5449	mem_cgroup_charge_statistics(memcg, page, compound, nr_pages);
5450	memcg_check_events(memcg, page);
5451	local_irq_enable();
5452
5453	if (do_memsw_account() && PageSwapCache(page)) {
5454	swp_entry_t entry = { .val = page_private(page) };
5455	/*
5456	* The swap entry might not get freed for a long time,
5457	* let's not wait for it. The page already received a
5458	* memory+swap charge, drop the swap entry duplicate.
5459	*/
5460	mem_cgroup_uncharge_swap(entry);
5461	}
5462	}
5463
5464	/**
5465	* mem_cgroup_cancel_charge - cancel a page charge
5466	* @page: page to charge
5467	* @memcg: memcg to charge the page to
5468	* @compound: charge the page as compound or small page
5469	*
5470	* Cancel a charge transaction started by mem_cgroup_try_charge().
5471	*/
5472	void mem_cgroup_cancel_charge(struct page *page, struct mem_cgroup *memcg,
5473	bool compound)
5474	{
5475	unsigned int nr_pages = compound ? hpage_nr_pages(page) : 1;
5476
5477	if (mem_cgroup_disabled())
5478	return;
5479	/*
5480	* Swap faults will attempt to charge the same page multiple
5481	* times. But reuse_swap_page() might have removed the page
5482	* from swapcache already, so we can't check PageSwapCache().
5483	*/
5484	if (!memcg)
5485	return;
5486
5487	cancel_charge(memcg, nr_pages);
5488	}
5489
5490	static void uncharge_batch(struct mem_cgroup *memcg, unsigned long pgpgout,
5491	unsigned long nr_anon, unsigned long nr_file,
5492	unsigned long nr_huge, unsigned long nr_kmem,
5493	struct page *dummy_page)
5494	{
5495	unsigned long nr_pages = nr_anon + nr_file + nr_kmem;
5496	unsigned long flags;
5497
5498	if (!mem_cgroup_is_root(memcg)) {
5499	page_counter_uncharge(&memcg->memory, nr_pages);
5500	if (do_memsw_account())
5501	page_counter_uncharge(&memcg->memsw, nr_pages);
5502	if (!cgroup_subsys_on_dfl(memory_cgrp_subsys) && nr_kmem)
5503	page_counter_uncharge(&memcg->kmem, nr_kmem);
5504	memcg_oom_recover(memcg);
5505	}
5506
5507	local_irq_save(flags);
5508	__this_cpu_sub(memcg->stat->count[MEM_CGROUP_STAT_RSS], nr_anon);
5509	__this_cpu_sub(memcg->stat->count[MEM_CGROUP_STAT_CACHE], nr_file);
5510	__this_cpu_sub(memcg->stat->count[MEM_CGROUP_STAT_RSS_HUGE], nr_huge);
5511	__this_cpu_add(memcg->stat->events[MEM_CGROUP_EVENTS_PGPGOUT], pgpgout);
5512	__this_cpu_add(memcg->stat->nr_page_events, nr_pages);
5513	memcg_check_events(memcg, dummy_page);
5514	local_irq_restore(flags);
5515
5516	if (!mem_cgroup_is_root(memcg))
5517	css_put_many(&memcg->css, nr_pages);
5518	}
5519
5520	static void uncharge_list(struct list_head *page_list)
5521	{
5522	struct mem_cgroup *memcg = NULL;
5523	unsigned long nr_anon = 0;
5524	unsigned long nr_file = 0;
5525	unsigned long nr_huge = 0;
5526	unsigned long nr_kmem = 0;
5527	unsigned long pgpgout = 0;
5528	struct list_head *next;
5529	struct page *page;
5530
5531	/*
5532	* Note that the list can be a single page->lru; hence the
5533	* do-while loop instead of a simple list_for_each_entry().
5534	*/
5535	next = page_list->next;
5536	do {
5537	page = list_entry(next, struct page, lru);
5538	next = page->lru.next;
5539
5540	VM_BUG_ON_PAGE(PageLRU(page), page);
5541	VM_BUG_ON_PAGE(!PageHWPoison(page) && page_count(page), page);
5542
5543	if (!page->mem_cgroup)
5544	continue;
5545
5546	/*
5547	* Nobody should be changing or seriously looking at
5548	* page->mem_cgroup at this point, we have fully
5549	* exclusive access to the page.
5550	*/
5551
5552	if (memcg != page->mem_cgroup) {
5553	if (memcg) {
5554	uncharge_batch(memcg, pgpgout, nr_anon, nr_file,
5555	nr_huge, nr_kmem, page);
5556	pgpgout = nr_anon = nr_file =
5557	nr_huge = nr_kmem = 0;
5558	}
5559	memcg = page->mem_cgroup;
5560	}
5561
5562	if (!PageKmemcg(page)) {
5563	unsigned int nr_pages = 1;
5564
5565	if (PageTransHuge(page)) {
5566	nr_pages <<= compound_order(page);
5567	nr_huge += nr_pages;
5568	}
5569	if (PageAnon(page))
5570	nr_anon += nr_pages;
5571	else
5572	nr_file += nr_pages;
5573	pgpgout++;
5574	} else {
5575	nr_kmem += 1 << compound_order(page);
5576	__ClearPageKmemcg(page);
5577	}
5578
5579	page->mem_cgroup = NULL;
5580	} while (next != page_list);
5581
5582	if (memcg)
5583	uncharge_batch(memcg, pgpgout, nr_anon, nr_file,
5584	nr_huge, nr_kmem, page);
5585	}
5586
5587	/**
5588	* mem_cgroup_uncharge - uncharge a page
5589	* @page: page to uncharge
5590	*
5591	* Uncharge a page previously charged with mem_cgroup_try_charge() and
5592	* mem_cgroup_commit_charge().
5593	*/
5594	void mem_cgroup_uncharge(struct page *page)
5595	{
5596	if (mem_cgroup_disabled())
5597	return;
5598
5599	/* Don't touch page->lru of any random page, pre-check: */
5600	if (!page->mem_cgroup)
5601	return;
5602
5603	INIT_LIST_HEAD(&page->lru);
5604	uncharge_list(&page->lru);
5605	}
5606
5607	/**
5608	* mem_cgroup_uncharge_list - uncharge a list of page
5609	* @page_list: list of pages to uncharge
5610	*
5611	* Uncharge a list of pages previously charged with
5612	* mem_cgroup_try_charge() and mem_cgroup_commit_charge().
5613	*/
5614	void mem_cgroup_uncharge_list(struct list_head *page_list)
5615	{
5616	if (mem_cgroup_disabled())
5617	return;
5618
5619	if (!list_empty(page_list))
5620	uncharge_list(page_list);
5621	}
5622
5623	/**
5624	* mem_cgroup_migrate - charge a page's replacement
5625	* @oldpage: currently circulating page
5626	* @newpage: replacement page
5627	*
5628	* Charge @newpage as a replacement page for @oldpage. @oldpage will
5629	* be uncharged upon free.
5630	*
5631	* Both pages must be locked, @newpage->mapping must be set up.
5632	*/
5633	void mem_cgroup_migrate(struct page *oldpage, struct page *newpage)
5634	{
5635	struct mem_cgroup *memcg;
5636	unsigned int nr_pages;
5637	bool compound;
5638	unsigned long flags;
5639
5640	VM_BUG_ON_PAGE(!PageLocked(oldpage), oldpage);
5641	VM_BUG_ON_PAGE(!PageLocked(newpage), newpage);
5642	VM_BUG_ON_PAGE(PageAnon(oldpage) != PageAnon(newpage), newpage);
5643	VM_BUG_ON_PAGE(PageTransHuge(oldpage) != PageTransHuge(newpage),
5644	newpage);
5645
5646	if (mem_cgroup_disabled())
5647	return;
5648
5649	/* Page cache replacement: new page already charged? */
5650	if (newpage->mem_cgroup)
5651	return;
5652
5653	/* Swapcache readahead pages can get replaced before being charged */
5654	memcg = oldpage->mem_cgroup;
5655	if (!memcg)
5656	return;
5657
5658	/* Force-charge the new page. The old one will be freed soon */
5659	compound = PageTransHuge(newpage);
5660	nr_pages = compound ? hpage_nr_pages(newpage) : 1;
5661
5662	page_counter_charge(&memcg->memory, nr_pages);
5663	if (do_memsw_account())
5664	page_counter_charge(&memcg->memsw, nr_pages);
5665	css_get_many(&memcg->css, nr_pages);
5666
5667	commit_charge(newpage, memcg, false);
5668
5669	local_irq_save(flags);
5670	mem_cgroup_charge_statistics(memcg, newpage, compound, nr_pages);
5671	memcg_check_events(memcg, newpage);
5672	local_irq_restore(flags);
5673	}
5674
5675	DEFINE_STATIC_KEY_FALSE(memcg_sockets_enabled_key);
5676	EXPORT_SYMBOL(memcg_sockets_enabled_key);
5677
5678	void mem_cgroup_sk_alloc(struct sock *sk)
5679	{
5680	struct mem_cgroup *memcg;
5681
5682	if (!mem_cgroup_sockets_enabled)
5683	return;
5684
5685	/*
5686	* Socket cloning can throw us here with sk_memcg already
5687	* filled. It won't however, necessarily happen from
5688	* process context. So the test for root memcg given
5689	* the current task's memcg won't help us in this case.
5690	*
5691	* Respecting the original socket's memcg is a better
5692	* decision in this case.
5693	*/
5694	if (sk->sk_memcg) {
5695	BUG_ON(mem_cgroup_is_root(sk->sk_memcg));
5696	css_get(&sk->sk_memcg->css);
5697	return;
5698	}
5699
5700	rcu_read_lock();
5701	memcg = mem_cgroup_from_task(current);
5702	if (memcg == root_mem_cgroup)
5703	goto out;
5704	if (!cgroup_subsys_on_dfl(memory_cgrp_subsys) && !memcg->tcpmem_active)
5705	goto out;
5706	if (css_tryget_online(&memcg->css))
5707	sk->sk_memcg = memcg;
5708	out:
5709	rcu_read_unlock();
5710	}
5711
5712	void mem_cgroup_sk_free(struct sock *sk)
5713	{
5714	if (sk->sk_memcg)
5715	css_put(&sk->sk_memcg->css);
5716	}
5717
5718	/**
5719	* mem_cgroup_charge_skmem - charge socket memory
5720	* @memcg: memcg to charge
5721	* @nr_pages: number of pages to charge
5722	*
5723	* Charges @nr_pages to @memcg. Returns %true if the charge fit within
5724	* @memcg's configured limit, %false if the charge had to be forced.
5725	*/
5726	bool mem_cgroup_charge_skmem(struct mem_cgroup *memcg, unsigned int nr_pages)
5727	{
5728	gfp_t gfp_mask = GFP_KERNEL;
5729
5730	if (!cgroup_subsys_on_dfl(memory_cgrp_subsys)) {
5731	struct page_counter *fail;
5732
5733	if (page_counter_try_charge(&memcg->tcpmem, nr_pages, &fail)) {
5734	memcg->tcpmem_pressure = 0;
5735	return true;
5736	}
5737	page_counter_charge(&memcg->tcpmem, nr_pages);
5738	memcg->tcpmem_pressure = 1;
5739	return false;
5740	}
5741
5742	/* Don't block in the packet receive path */
5743	if (in_softirq())
5744	gfp_mask = GFP_NOWAIT;
5745
5746	this_cpu_add(memcg->stat->count[MEMCG_SOCK], nr_pages);
5747
5748	if (try_charge(memcg, gfp_mask, nr_pages) == 0)
5749	return true;
5750
5751	try_charge(memcg, gfp_mask|__GFP_NOFAIL, nr_pages);
5752	return false;
5753	}
5754
5755	/**
5756	* mem_cgroup_uncharge_skmem - uncharge socket memory
5757	* @memcg - memcg to uncharge
5758	* @nr_pages - number of pages to uncharge
5759	*/
5760	void mem_cgroup_uncharge_skmem(struct mem_cgroup *memcg, unsigned int nr_pages)
5761	{
5762	if (!cgroup_subsys_on_dfl(memory_cgrp_subsys)) {
5763	page_counter_uncharge(&memcg->tcpmem, nr_pages);
5764	return;
5765	}
5766
5767	this_cpu_sub(memcg->stat->count[MEMCG_SOCK], nr_pages);
5768
5769	page_counter_uncharge(&memcg->memory, nr_pages);
5770	css_put_many(&memcg->css, nr_pages);
5771	}
5772
5773	static int __init cgroup_memory(char *s)
5774	{
5775	char *token;
5776
5777	while ((token = strsep(&s, ",")) != NULL) {
5778	if (!*token)
5779	continue;
5780	if (!strcmp(token, "nosocket"))
5781	cgroup_memory_nosocket = true;
5782	if (!strcmp(token, "nokmem"))
5783	cgroup_memory_nokmem = true;
5784	}
5785	return 0;
5786	}
5787	__setup("cgroup.memory=", cgroup_memory);
5788
5789	/*
5790	* subsys_initcall() for memory controller.
5791	*
5792	* Some parts like hotcpu_notifier() have to be initialized from this context
5793	* because of lock dependencies (cgroup_lock -> cpu hotplug) but basically
5794	* everything that doesn't depend on a specific mem_cgroup structure should
5795	* be initialized from here.
5796	*/
5797	static int __init mem_cgroup_init(void)
5798	{
5799	int cpu, node;
5800
5801	#ifndef CONFIG_SLOB
5802	/*
5803	* Kmem cache creation is mostly done with the slab_mutex held,
5804	* so use a special workqueue to avoid stalling all worker
5805	* threads in case lots of cgroups are created simultaneously.
5806	*/
5807	memcg_kmem_cache_create_wq =
5808	alloc_ordered_workqueue("memcg_kmem_cache_create", 0);
5809	BUG_ON(!memcg_kmem_cache_create_wq);
5810	#endif
5811
5812	hotcpu_notifier(memcg_cpu_hotplug_callback, 0);
5813
5814	for_each_possible_cpu(cpu)
5815	INIT_WORK(&per_cpu_ptr(&memcg_stock, cpu)->work,
5816	drain_local_stock);
5817
5818	for_each_node(node) {
5819	struct mem_cgroup_tree_per_node *rtpn;
5820
5821	rtpn = kzalloc_node(sizeof(*rtpn), GFP_KERNEL,
5822	node_online(node) ? node : NUMA_NO_NODE);
5823
5824	rtpn->rb_root = RB_ROOT;
5825	spin_lock_init(&rtpn->lock);
5826	soft_limit_tree.rb_tree_per_node[node] = rtpn;
5827	}
5828
5829	return 0;
5830	}
5831	subsys_initcall(mem_cgroup_init);
5832
5833	#ifdef CONFIG_MEMCG_SWAP
5834	static struct mem_cgroup *mem_cgroup_id_get_online(struct mem_cgroup *memcg)
5835	{
5836	while (!atomic_inc_not_zero(&memcg->id.ref)) {
5837	/*
5838	* The root cgroup cannot be destroyed, so it's refcount must
5839	* always be >= 1.
5840	*/
5841	if (WARN_ON_ONCE(memcg == root_mem_cgroup)) {
5842	VM_BUG_ON(1);
5843	break;
5844	}
5845	memcg = parent_mem_cgroup(memcg);
5846	if (!memcg)
5847	memcg = root_mem_cgroup;
5848	}
5849	return memcg;
5850	}
5851
5852	/**
5853	* mem_cgroup_swapout - transfer a memsw charge to swap
5854	* @page: page whose memsw charge to transfer
5855	* @entry: swap entry to move the charge to
5856	*
5857	* Transfer the memsw charge of @page to @entry.
5858	*/
5859	void mem_cgroup_swapout(struct page *page, swp_entry_t entry)
5860	{
5861	struct mem_cgroup *memcg, *swap_memcg;
5862	unsigned short oldid;
5863
5864	VM_BUG_ON_PAGE(PageLRU(page), page);
5865	VM_BUG_ON_PAGE(page_count(page), page);
5866
5867	if (!do_memsw_account())
5868	return;
5869
5870	memcg = page->mem_cgroup;
5871
5872	/* Readahead page, never charged */
5873	if (!memcg)
5874	return;
5875
5876	/*
5877	* In case the memcg owning these pages has been offlined and doesn't
5878	* have an ID allocated to it anymore, charge the closest online
5879	* ancestor for the swap instead and transfer the memory+swap charge.
5880	*/
5881	swap_memcg = mem_cgroup_id_get_online(memcg);
5882	oldid = swap_cgroup_record(entry, mem_cgroup_id(swap_memcg));
5883	VM_BUG_ON_PAGE(oldid, page);
5884	mem_cgroup_swap_statistics(swap_memcg, true);
5885
5886	page->mem_cgroup = NULL;
5887
5888	if (!mem_cgroup_is_root(memcg))
5889	page_counter_uncharge(&memcg->memory, 1);
5890
5891	if (memcg != swap_memcg) {
5892	if (!mem_cgroup_is_root(swap_memcg))
5893	page_counter_charge(&swap_memcg->memsw, 1);
5894	page_counter_uncharge(&memcg->memsw, 1);
5895	}
5896
5897	/*
5898	* Interrupts should be disabled here because the caller holds the
5899	* mapping->tree_lock lock which is taken with interrupts-off. It is
5900	* important here to have the interrupts disabled because it is the
5901	* only synchronisation we have for udpating the per-CPU variables.
5902	*/
5903	VM_BUG_ON(!irqs_disabled());
5904	mem_cgroup_charge_statistics(memcg, page, false, -1);
5905	memcg_check_events(memcg, page);
5906
5907	if (!mem_cgroup_is_root(memcg))
5908	css_put(&memcg->css);
5909	}
5910
5911	/*
5912	* mem_cgroup_try_charge_swap - try charging a swap entry
5913	* @page: page being added to swap
5914	* @entry: swap entry to charge
5915	*
5916	* Try to charge @entry to the memcg that @page belongs to.
5917	*
5918	* Returns 0 on success, -ENOMEM on failure.
5919	*/
5920	int mem_cgroup_try_charge_swap(struct page *page, swp_entry_t entry)
5921	{
5922	struct mem_cgroup *memcg;
5923	struct page_counter *counter;
5924	unsigned short oldid;
5925
5926	if (!cgroup_subsys_on_dfl(memory_cgrp_subsys) || !do_swap_account)
5927	return 0;
5928
5929	memcg = page->mem_cgroup;
5930
5931	/* Readahead page, never charged */
5932	if (!memcg)
5933	return 0;
5934
5935	memcg = mem_cgroup_id_get_online(memcg);
5936
5937	if (!mem_cgroup_is_root(memcg) &&
5938	!page_counter_try_charge(&memcg->swap, 1, &counter)) {
5939	mem_cgroup_id_put(memcg);
5940	return -ENOMEM;
5941	}
5942
5943	oldid = swap_cgroup_record(entry, mem_cgroup_id(memcg));
5944	VM_BUG_ON_PAGE(oldid, page);
5945	mem_cgroup_swap_statistics(memcg, true);
5946
5947	return 0;
5948	}
5949
5950	/**
5951	* mem_cgroup_uncharge_swap - uncharge a swap entry
5952	* @entry: swap entry to uncharge
5953	*
5954	* Drop the swap charge associated with @entry.
5955	*/
5956	void mem_cgroup_uncharge_swap(swp_entry_t entry)
5957	{
5958	struct mem_cgroup *memcg;
5959	unsigned short id;
5960
5961	if (!do_swap_account)
5962	return;
5963
5964	id = swap_cgroup_record(entry, 0);
5965	rcu_read_lock();
5966	memcg = mem_cgroup_from_id(id);
5967	if (memcg) {
5968	if (!mem_cgroup_is_root(memcg)) {
5969	if (cgroup_subsys_on_dfl(memory_cgrp_subsys))
5970	page_counter_uncharge(&memcg->swap, 1);
5971	else
5972	page_counter_uncharge(&memcg->memsw, 1);
5973	}
5974	mem_cgroup_swap_statistics(memcg, false);
5975	mem_cgroup_id_put(memcg);
5976	}
5977	rcu_read_unlock();
5978	}
5979
5980	long mem_cgroup_get_nr_swap_pages(struct mem_cgroup *memcg)
5981	{
5982	long nr_swap_pages = get_nr_swap_pages();
5983
5984	if (!do_swap_account || !cgroup_subsys_on_dfl(memory_cgrp_subsys))
5985	return nr_swap_pages;
5986	for (; memcg != root_mem_cgroup; memcg = parent_mem_cgroup(memcg))
5987	nr_swap_pages = min_t(long, nr_swap_pages,
5988	READ_ONCE(memcg->swap.limit) -
5989	page_counter_read(&memcg->swap));
5990	return nr_swap_pages;
5991	}
5992
5993	bool mem_cgroup_swap_full(struct page *page)
5994	{
5995	struct mem_cgroup *memcg;
5996
5997	VM_BUG_ON_PAGE(!PageLocked(page), page);
5998
5999	if (vm_swap_full())
6000	return true;
6001	if (!do_swap_account || !cgroup_subsys_on_dfl(memory_cgrp_subsys))
6002	return false;
6003
6004	memcg = page->mem_cgroup;
6005	if (!memcg)
6006	return false;
6007
6008	for (; memcg != root_mem_cgroup; memcg = parent_mem_cgroup(memcg))
6009	if (page_counter_read(&memcg->swap) * 2 >= memcg->swap.limit)
6010	return true;
6011
6012	return false;
6013	}
6014
6015	/* for remember boot option*/
6016	#ifdef CONFIG_MEMCG_SWAP_ENABLED
6017	static int really_do_swap_account __initdata = 1;
6018	#else
6019	static int really_do_swap_account __initdata;
6020	#endif
6021
6022	static int __init enable_swap_account(char *s)
6023	{
6024	if (!strcmp(s, "1"))
6025	really_do_swap_account = 1;
6026	else if (!strcmp(s, "0"))
6027	really_do_swap_account = 0;
6028	return 1;
6029	}
6030	__setup("swapaccount=", enable_swap_account);
6031
6032	static u64 swap_current_read(struct cgroup_subsys_state *css,
6033	struct cftype *cft)
6034	{
6035	struct mem_cgroup *memcg = mem_cgroup_from_css(css);
6036
6037	return (u64)page_counter_read(&memcg->swap) * PAGE_SIZE;
6038	}
6039
6040	static int swap_max_show(struct seq_file *m, void *v)
6041	{
6042	struct mem_cgroup *memcg = mem_cgroup_from_css(seq_css(m));
6043	unsigned long max = READ_ONCE(memcg->swap.limit);
6044
6045	if (max == PAGE_COUNTER_MAX)
6046	seq_puts(m, "max\n");
6047	else
6048	seq_printf(m, "%llu\n", (u64)max * PAGE_SIZE);
6049
6050	return 0;
6051	}
6052
6053	static ssize_t swap_max_write(struct kernfs_open_file *of,
6054	char *buf, size_t nbytes, loff_t off)
6055	{
6056	struct mem_cgroup *memcg = mem_cgroup_from_css(of_css(of));
6057	unsigned long max;
6058	int err;
6059
6060	buf = strstrip(buf);
6061	err = page_counter_memparse(buf, "max", &max);
6062	if (err)
6063	return err;
6064
6065	mutex_lock(&memcg_limit_mutex);
6066	err = page_counter_limit(&memcg->swap, max);
6067	mutex_unlock(&memcg_limit_mutex);
6068	if (err)
6069	return err;
6070
6071	return nbytes;
6072	}
6073
6074	static struct cftype swap_files[] = {
6075	{
6076	.name = "swap.current",
6077	.flags = CFTYPE_NOT_ON_ROOT,
6078	.read_u64 = swap_current_read,
6079	},
6080	{
6081	.name = "swap.max",
6082	.flags = CFTYPE_NOT_ON_ROOT,
6083	.seq_show = swap_max_show,
6084	.write = swap_max_write,
6085	},
6086	{ } /* terminate */
6087	};
6088
6089	static struct cftype memsw_cgroup_files[] = {
6090	{
6091	.name = "memsw.usage_in_bytes",
6092	.private = MEMFILE_PRIVATE(_MEMSWAP, RES_USAGE),
6093	.read_u64 = mem_cgroup_read_u64,
6094	},
6095	{
6096	.name = "memsw.max_usage_in_bytes",
6097	.private = MEMFILE_PRIVATE(_MEMSWAP, RES_MAX_USAGE),
6098	.write = mem_cgroup_reset,
6099	.read_u64 = mem_cgroup_read_u64,
6100	},
6101	{
6102	.name = "memsw.limit_in_bytes",
6103	.private = MEMFILE_PRIVATE(_MEMSWAP, RES_LIMIT),
6104	.write = mem_cgroup_write,
6105	.read_u64 = mem_cgroup_read_u64,
6106	},
6107	{
6108	.name = "memsw.failcnt",
6109	.private = MEMFILE_PRIVATE(_MEMSWAP, RES_FAILCNT),
6110	.write = mem_cgroup_reset,
6111	.read_u64 = mem_cgroup_read_u64,
6112	},
6113	{ }, /* terminate */
6114	};
6115
6116	static int __init mem_cgroup_swap_init(void)
6117	{
6118	if (!mem_cgroup_disabled() && really_do_swap_account) {
6119	do_swap_account = 1;
6120	WARN_ON(cgroup_add_dfl_cftypes(&memory_cgrp_subsys,
6121	swap_files));
6122	WARN_ON(cgroup_add_legacy_cftypes(&memory_cgrp_subsys,
6123	memsw_cgroup_files));
6124	}
6125	return 0;
6126	}
6127	subsys_initcall(mem_cgroup_swap_init);
6128
6129	#endif /* CONFIG_MEMCG_SWAP */
6130