path: root/mm/slab.c (plain)
blob: bd498eff4d7396bacd56d43ee97f390419439a6e

1	/*
2	* linux/mm/slab.c
3	* Written by Mark Hemment, 1996/97.
4	* (markhe@nextd.demon.co.uk)
5	*
6	* kmem_cache_destroy() + some cleanup - 1999 Andrea Arcangeli
7	*
8	* Major cleanup, different bufctl logic, per-cpu arrays
9	* (c) 2000 Manfred Spraul
10	*
11	* Cleanup, make the head arrays unconditional, preparation for NUMA
12	* (c) 2002 Manfred Spraul
13	*
14	* An implementation of the Slab Allocator as described in outline in;
15	* UNIX Internals: The New Frontiers by Uresh Vahalia
16	* Pub: Prentice Hall ISBN 0-13-101908-2
17	* or with a little more detail in;
18	* The Slab Allocator: An Object-Caching Kernel Memory Allocator
19	* Jeff Bonwick (Sun Microsystems).
20	* Presented at: USENIX Summer 1994 Technical Conference
21	*
22	* The memory is organized in caches, one cache for each object type.
23	* (e.g. inode_cache, dentry_cache, buffer_head, vm_area_struct)
24	* Each cache consists out of many slabs (they are small (usually one
25	* page long) and always contiguous), and each slab contains multiple
26	* initialized objects.
27	*
28	* This means, that your constructor is used only for newly allocated
29	* slabs and you must pass objects with the same initializations to
30	* kmem_cache_free.
31	*
32	* Each cache can only support one memory type (GFP_DMA, GFP_HIGHMEM,
33	* normal). If you need a special memory type, then must create a new
34	* cache for that memory type.
35	*
36	* In order to reduce fragmentation, the slabs are sorted in 3 groups:
37	* full slabs with 0 free objects
38	* partial slabs
39	* empty slabs with no allocated objects
40	*
41	* If partial slabs exist, then new allocations come from these slabs,
42	* otherwise from empty slabs or new slabs are allocated.
43	*
44	* kmem_cache_destroy() CAN CRASH if you try to allocate from the cache
45	* during kmem_cache_destroy(). The caller must prevent concurrent allocs.
46	*
47	* Each cache has a short per-cpu head array, most allocs
48	* and frees go into that array, and if that array overflows, then 1/2
49	* of the entries in the array are given back into the global cache.
50	* The head array is strictly LIFO and should improve the cache hit rates.
51	* On SMP, it additionally reduces the spinlock operations.
52	*
53	* The c_cpuarray may not be read with enabled local interrupts -
54	* it's changed with a smp_call_function().
55	*
56	* SMP synchronization:
57	* constructors and destructors are called without any locking.
58	* Several members in struct kmem_cache and struct slab never change, they
59	* are accessed without any locking.
60	* The per-cpu arrays are never accessed from the wrong cpu, no locking,
61	* and local interrupts are disabled so slab code is preempt-safe.
62	* The non-constant members are protected with a per-cache irq spinlock.
63	*
64	* Many thanks to Mark Hemment, who wrote another per-cpu slab patch
65	* in 2000 - many ideas in the current implementation are derived from
66	* his patch.
67	*
68	* Further notes from the original documentation:
69	*
70	* 11 April '97. Started multi-threading - markhe
71	* The global cache-chain is protected by the mutex 'slab_mutex'.
72	* The sem is only needed when accessing/extending the cache-chain, which
73	* can never happen inside an interrupt (kmem_cache_create(),
74	* kmem_cache_shrink() and kmem_cache_reap()).
75	*
76	* At present, each engine can be growing a cache. This should be blocked.
77	*
78	* 15 March 2005. NUMA slab allocator.
79	* Shai Fultheim <shai@scalex86.org>.
80	* Shobhit Dayal <shobhit@calsoftinc.com>
81	* Alok N Kataria <alokk@calsoftinc.com>
82	* Christoph Lameter <christoph@lameter.com>
83	*
84	* Modified the slab allocator to be node aware on NUMA systems.
85	* Each node has its own list of partial, free and full slabs.
86	* All object allocations for a node occur from node specific slab lists.
87	*/
88
89	#include <linux/slab.h>
90	#include <linux/mm.h>
91	#include <linux/poison.h>
92	#include <linux/swap.h>
93	#include <linux/cache.h>
94	#include <linux/interrupt.h>
95	#include <linux/init.h>
96	#include <linux/compiler.h>
97	#include <linux/cpuset.h>
98	#include <linux/proc_fs.h>
99	#include <linux/seq_file.h>
100	#include <linux/notifier.h>
101	#include <linux/kallsyms.h>
102	#include <linux/cpu.h>
103	#include <linux/sysctl.h>
104	#include <linux/module.h>
105	#include <linux/rcupdate.h>
106	#include <linux/string.h>
107	#include <linux/uaccess.h>
108	#include <linux/nodemask.h>
109	#include <linux/kmemleak.h>
110	#include <linux/mempolicy.h>
111	#include <linux/mutex.h>
112	#include <linux/fault-inject.h>
113	#include <linux/rtmutex.h>
114	#include <linux/reciprocal_div.h>
115	#include <linux/debugobjects.h>
116	#include <linux/kmemcheck.h>
117	#include <linux/memory.h>
118	#include <linux/prefetch.h>
119
120	#include <net/sock.h>
121
122	#include <asm/cacheflush.h>
123	#include <asm/tlbflush.h>
124	#include <asm/page.h>
125
126	#include <trace/events/kmem.h>
127
128	#include "internal.h"
129
130	#include "slab.h"
131
132	/*
133	* DEBUG - 1 for kmem_cache_create() to honour; SLAB_RED_ZONE & SLAB_POISON.
134	* 0 for faster, smaller code (especially in the critical paths).
135	*
136	* STATS - 1 to collect stats for /proc/slabinfo.
137	* 0 for faster, smaller code (especially in the critical paths).
138	*
139	* FORCED_DEBUG - 1 enables SLAB_RED_ZONE and SLAB_POISON (if possible)
140	*/
141
142	#ifdef CONFIG_DEBUG_SLAB
143	#define DEBUG 1
144	#define STATS 1
145	#define FORCED_DEBUG 1
146	#else
147	#define DEBUG 0
148	#define STATS 0
149	#define FORCED_DEBUG 0
150	#endif
151
152	/* Shouldn't this be in a header file somewhere? */
153	#define BYTES_PER_WORD sizeof(void *)
154	#define REDZONE_ALIGN max(BYTES_PER_WORD, __alignof__(unsigned long long))
155
156	#ifndef ARCH_KMALLOC_FLAGS
157	#define ARCH_KMALLOC_FLAGS SLAB_HWCACHE_ALIGN
158	#endif
159
160	#define FREELIST_BYTE_INDEX (((PAGE_SIZE >> BITS_PER_BYTE) \
161	<= SLAB_OBJ_MIN_SIZE) ? 1 : 0)
162
163	#if FREELIST_BYTE_INDEX
164	typedef unsigned char freelist_idx_t;
165	#else
166	typedef unsigned short freelist_idx_t;
167	#endif
168
169	#define SLAB_OBJ_MAX_NUM ((1 << sizeof(freelist_idx_t) * BITS_PER_BYTE) - 1)
170
171	/*
172	* struct array_cache
173	*
174	* Purpose:
175	* - LIFO ordering, to hand out cache-warm objects from _alloc
176	* - reduce the number of linked list operations
177	* - reduce spinlock operations
178	*
179	* The limit is stored in the per-cpu structure to reduce the data cache
180	* footprint.
181	*
182	*/
183	struct array_cache {
184	unsigned int avail;
185	unsigned int limit;
186	unsigned int batchcount;
187	unsigned int touched;
188	void *entry[]; /*
189	* Must have this definition in here for the proper
190	* alignment of array_cache. Also simplifies accessing
191	* the entries.
192	*/
193	};
194
195	struct alien_cache {
196	spinlock_t lock;
197	struct array_cache ac;
198	};
199
200	/*
201	* Need this for bootstrapping a per node allocator.
202	*/
203	#define NUM_INIT_LISTS (2 * MAX_NUMNODES)
204	static struct kmem_cache_node __initdata init_kmem_cache_node[NUM_INIT_LISTS];
205	#define CACHE_CACHE 0
206	#define SIZE_NODE (MAX_NUMNODES)
207
208	static int drain_freelist(struct kmem_cache *cache,
209	struct kmem_cache_node *n, int tofree);
210	static void free_block(struct kmem_cache *cachep, void **objpp, int len,
211	int node, struct list_head *list);
212	static void slabs_destroy(struct kmem_cache *cachep, struct list_head *list);
213	static int enable_cpucache(struct kmem_cache *cachep, gfp_t gfp);
214	static void cache_reap(struct work_struct *unused);
215
216	static inline void fixup_objfreelist_debug(struct kmem_cache *cachep,
217	void **list);
218	static inline void fixup_slab_list(struct kmem_cache *cachep,
219	struct kmem_cache_node *n, struct page *page,
220	void **list);
221	static int slab_early_init = 1;
222
223	#define INDEX_NODE kmalloc_index(sizeof(struct kmem_cache_node))
224
225	static void kmem_cache_node_init(struct kmem_cache_node *parent)
226	{
227	INIT_LIST_HEAD(&parent->slabs_full);
228	INIT_LIST_HEAD(&parent->slabs_partial);
229	INIT_LIST_HEAD(&parent->slabs_free);
230	parent->shared = NULL;
231	parent->alien = NULL;
232	parent->colour_next = 0;
233	spin_lock_init(&parent->list_lock);
234	parent->free_objects = 0;
235	parent->free_touched = 0;
236	parent->num_slabs = 0;
237	}
238
239	#define MAKE_LIST(cachep, listp, slab, nodeid) \
240	do { \
241	INIT_LIST_HEAD(listp); \
242	list_splice(&get_node(cachep, nodeid)->slab, listp); \
243	} while (0)
244
245	#define MAKE_ALL_LISTS(cachep, ptr, nodeid) \
246	do { \
247	MAKE_LIST((cachep), (&(ptr)->slabs_full), slabs_full, nodeid); \
248	MAKE_LIST((cachep), (&(ptr)->slabs_partial), slabs_partial, nodeid); \
249	MAKE_LIST((cachep), (&(ptr)->slabs_free), slabs_free, nodeid); \
250	} while (0)
251
252	#define CFLGS_OBJFREELIST_SLAB (0x40000000UL)
253	#define CFLGS_OFF_SLAB (0x80000000UL)
254	#define OBJFREELIST_SLAB(x) ((x)->flags & CFLGS_OBJFREELIST_SLAB)
255	#define OFF_SLAB(x) ((x)->flags & CFLGS_OFF_SLAB)
256
257	#define BATCHREFILL_LIMIT 16
258	/*
259	* Optimization question: fewer reaps means less probability for unnessary
260	* cpucache drain/refill cycles.
261	*
262	* OTOH the cpuarrays can contain lots of objects,
263	* which could lock up otherwise freeable slabs.
264	*/
265	#define REAPTIMEOUT_AC (2*HZ)
266	#define REAPTIMEOUT_NODE (4*HZ)
267
268	#if STATS
269	#define STATS_INC_ACTIVE(x) ((x)->num_active++)
270	#define STATS_DEC_ACTIVE(x) ((x)->num_active--)
271	#define STATS_INC_ALLOCED(x) ((x)->num_allocations++)
272	#define STATS_INC_GROWN(x) ((x)->grown++)
273	#define STATS_ADD_REAPED(x,y) ((x)->reaped += (y))
274	#define STATS_SET_HIGH(x) \
275	do { \
276	if ((x)->num_active > (x)->high_mark) \
277	(x)->high_mark = (x)->num_active; \
278	} while (0)
279	#define STATS_INC_ERR(x) ((x)->errors++)
280	#define STATS_INC_NODEALLOCS(x) ((x)->node_allocs++)
281	#define STATS_INC_NODEFREES(x) ((x)->node_frees++)
282	#define STATS_INC_ACOVERFLOW(x) ((x)->node_overflow++)
283	#define STATS_SET_FREEABLE(x, i) \
284	do { \
285	if ((x)->max_freeable < i) \
286	(x)->max_freeable = i; \
287	} while (0)
288	#define STATS_INC_ALLOCHIT(x) atomic_inc(&(x)->allochit)
289	#define STATS_INC_ALLOCMISS(x) atomic_inc(&(x)->allocmiss)
290	#define STATS_INC_FREEHIT(x) atomic_inc(&(x)->freehit)
291	#define STATS_INC_FREEMISS(x) atomic_inc(&(x)->freemiss)
292	#else
293	#define STATS_INC_ACTIVE(x) do { } while (0)
294	#define STATS_DEC_ACTIVE(x) do { } while (0)
295	#define STATS_INC_ALLOCED(x) do { } while (0)
296	#define STATS_INC_GROWN(x) do { } while (0)
297	#define STATS_ADD_REAPED(x,y) do { (void)(y); } while (0)
298	#define STATS_SET_HIGH(x) do { } while (0)
299	#define STATS_INC_ERR(x) do { } while (0)
300	#define STATS_INC_NODEALLOCS(x) do { } while (0)
301	#define STATS_INC_NODEFREES(x) do { } while (0)
302	#define STATS_INC_ACOVERFLOW(x) do { } while (0)
303	#define STATS_SET_FREEABLE(x, i) do { } while (0)
304	#define STATS_INC_ALLOCHIT(x) do { } while (0)
305	#define STATS_INC_ALLOCMISS(x) do { } while (0)
306	#define STATS_INC_FREEHIT(x) do { } while (0)
307	#define STATS_INC_FREEMISS(x) do { } while (0)
308	#endif
309
310	#if DEBUG
311
312	/*
313	* memory layout of objects:
314	* 0 : objp
315	* 0 .. cachep->obj_offset - BYTES_PER_WORD - 1: padding. This ensures that
316	* the end of an object is aligned with the end of the real
317	* allocation. Catches writes behind the end of the allocation.
318	* cachep->obj_offset - BYTES_PER_WORD .. cachep->obj_offset - 1:
319	* redzone word.
320	* cachep->obj_offset: The real object.
321	* cachep->size - 2* BYTES_PER_WORD: redzone word [BYTES_PER_WORD long]
322	* cachep->size - 1* BYTES_PER_WORD: last caller address
323	* [BYTES_PER_WORD long]
324	*/
325	static int obj_offset(struct kmem_cache *cachep)
326	{
327	return cachep->obj_offset;
328	}
329
330	static unsigned long long *dbg_redzone1(struct kmem_cache *cachep, void *objp)
331	{
332	BUG_ON(!(cachep->flags & SLAB_RED_ZONE));
333	return (unsigned long long*) (objp + obj_offset(cachep) -
334	sizeof(unsigned long long));
335	}
336
337	static unsigned long long *dbg_redzone2(struct kmem_cache *cachep, void *objp)
338	{
339	BUG_ON(!(cachep->flags & SLAB_RED_ZONE));
340	if (cachep->flags & SLAB_STORE_USER)
341	return (unsigned long long *)(objp + cachep->size -
342	sizeof(unsigned long long) -
343	REDZONE_ALIGN);
344	return (unsigned long long *) (objp + cachep->size -
345	sizeof(unsigned long long));
346	}
347
348	static void **dbg_userword(struct kmem_cache *cachep, void *objp)
349	{
350	BUG_ON(!(cachep->flags & SLAB_STORE_USER));
351	return (void **)(objp + cachep->size - BYTES_PER_WORD);
352	}
353
354	#else
355
356	#define obj_offset(x) 0
357	#define dbg_redzone1(cachep, objp) ({BUG(); (unsigned long long *)NULL;})
358	#define dbg_redzone2(cachep, objp) ({BUG(); (unsigned long long *)NULL;})
359	#define dbg_userword(cachep, objp) ({BUG(); (void **)NULL;})
360
361	#endif
362
363	#ifdef CONFIG_DEBUG_SLAB_LEAK
364
365	static inline bool is_store_user_clean(struct kmem_cache *cachep)
366	{
367	return atomic_read(&cachep->store_user_clean) == 1;
368	}
369
370	static inline void set_store_user_clean(struct kmem_cache *cachep)
371	{
372	atomic_set(&cachep->store_user_clean, 1);
373	}
374
375	static inline void set_store_user_dirty(struct kmem_cache *cachep)
376	{
377	if (is_store_user_clean(cachep))
378	atomic_set(&cachep->store_user_clean, 0);
379	}
380
381	#else
382	static inline void set_store_user_dirty(struct kmem_cache *cachep) {}
383
384	#endif
385
386	/*
387	* Do not go above this order unless 0 objects fit into the slab or
388	* overridden on the command line.
389	*/
390	#define SLAB_MAX_ORDER_HI 1
391	#define SLAB_MAX_ORDER_LO 0
392	static int slab_max_order = SLAB_MAX_ORDER_LO;
393	static bool slab_max_order_set __initdata;
394
395	static inline struct kmem_cache *virt_to_cache(const void *obj)
396	{
397	struct page *page = virt_to_head_page(obj);
398	return page->slab_cache;
399	}
400
401	static inline void *index_to_obj(struct kmem_cache *cache, struct page *page,
402	unsigned int idx)
403	{
404	return page->s_mem + cache->size * idx;
405	}
406
407	/*
408	* We want to avoid an expensive divide : (offset / cache->size)
409	* Using the fact that size is a constant for a particular cache,
410	* we can replace (offset / cache->size) by
411	* reciprocal_divide(offset, cache->reciprocal_buffer_size)
412	*/
413	static inline unsigned int obj_to_index(const struct kmem_cache *cache,
414	const struct page *page, void *obj)
415	{
416	u32 offset = (obj - page->s_mem);
417	return reciprocal_divide(offset, cache->reciprocal_buffer_size);
418	}
419
420	#define BOOT_CPUCACHE_ENTRIES 1
421	/* internal cache of cache description objs */
422	static struct kmem_cache kmem_cache_boot = {
423	.batchcount = 1,
424	.limit = BOOT_CPUCACHE_ENTRIES,
425	.shared = 1,
426	.size = sizeof(struct kmem_cache),
427	.name = "kmem_cache",
428	};
429
430	static DEFINE_PER_CPU(struct delayed_work, slab_reap_work);
431
432	static inline struct array_cache *cpu_cache_get(struct kmem_cache *cachep)
433	{
434	return this_cpu_ptr(cachep->cpu_cache);
435	}
436
437	/*
438	* Calculate the number of objects and left-over bytes for a given buffer size.
439	*/
440	static unsigned int cache_estimate(unsigned long gfporder, size_t buffer_size,
441	unsigned long flags, size_t *left_over)
442	{
443	unsigned int num;
444	size_t slab_size = PAGE_SIZE << gfporder;
445
446	/*
447	* The slab management structure can be either off the slab or
448	* on it. For the latter case, the memory allocated for a
449	* slab is used for:
450	*
451	* - @buffer_size bytes for each object
452	* - One freelist_idx_t for each object
453	*
454	* We don't need to consider alignment of freelist because
455	* freelist will be at the end of slab page. The objects will be
456	* at the correct alignment.
457	*
458	* If the slab management structure is off the slab, then the
459	* alignment will already be calculated into the size. Because
460	* the slabs are all pages aligned, the objects will be at the
461	* correct alignment when allocated.
462	*/
463	if (flags & (CFLGS_OBJFREELIST_SLAB | CFLGS_OFF_SLAB)) {
464	num = slab_size / buffer_size;
465	*left_over = slab_size % buffer_size;
466	} else {
467	num = slab_size / (buffer_size + sizeof(freelist_idx_t));
468	*left_over = slab_size %
469	(buffer_size + sizeof(freelist_idx_t));
470	}
471
472	return num;
473	}
474
475	#if DEBUG
476	#define slab_error(cachep, msg) __slab_error(__func__, cachep, msg)
477
478	static void __slab_error(const char *function, struct kmem_cache *cachep,
479	char *msg)
480	{
481	pr_err("slab error in %s(): cache `%s': %s\n",
482	function, cachep->name, msg);
483	dump_stack();
484	add_taint(TAINT_BAD_PAGE, LOCKDEP_NOW_UNRELIABLE);
485	}
486	#endif
487
488	/*
489	* By default on NUMA we use alien caches to stage the freeing of
490	* objects allocated from other nodes. This causes massive memory
491	* inefficiencies when using fake NUMA setup to split memory into a
492	* large number of small nodes, so it can be disabled on the command
493	* line
494	*/
495
496	static int use_alien_caches __read_mostly = 1;
497	static int __init noaliencache_setup(char *s)
498	{
499	use_alien_caches = 0;
500	return 1;
501	}
502	__setup("noaliencache", noaliencache_setup);
503
504	static int __init slab_max_order_setup(char *str)
505	{
506	get_option(&str, &slab_max_order);
507	slab_max_order = slab_max_order < 0 ? 0 :
508	min(slab_max_order, MAX_ORDER - 1);
509	slab_max_order_set = true;
510
511	return 1;
512	}
513	__setup("slab_max_order=", slab_max_order_setup);
514
515	#ifdef CONFIG_NUMA
516	/*
517	* Special reaping functions for NUMA systems called from cache_reap().
518	* These take care of doing round robin flushing of alien caches (containing
519	* objects freed on different nodes from which they were allocated) and the
520	* flushing of remote pcps by calling drain_node_pages.
521	*/
522	static DEFINE_PER_CPU(unsigned long, slab_reap_node);
523
524	static void init_reap_node(int cpu)
525	{
526	per_cpu(slab_reap_node, cpu) = next_node_in(cpu_to_mem(cpu),
527	node_online_map);
528	}
529
530	static void next_reap_node(void)
531	{
532	int node = __this_cpu_read(slab_reap_node);
533
534	node = next_node_in(node, node_online_map);
535	__this_cpu_write(slab_reap_node, node);
536	}
537
538	#else
539	#define init_reap_node(cpu) do { } while (0)
540	#define next_reap_node(void) do { } while (0)
541	#endif
542
543	/*
544	* Initiate the reap timer running on the target CPU. We run at around 1 to 2Hz
545	* via the workqueue/eventd.
546	* Add the CPU number into the expiration time to minimize the possibility of
547	* the CPUs getting into lockstep and contending for the global cache chain
548	* lock.
549	*/
550	static void start_cpu_timer(int cpu)
551	{
552	struct delayed_work *reap_work = &per_cpu(slab_reap_work, cpu);
553
554	/*
555	* When this gets called from do_initcalls via cpucache_init(),
556	* init_workqueues() has already run, so keventd will be setup
557	* at that time.
558	*/
559	if (keventd_up() && reap_work->work.func == NULL) {
560	init_reap_node(cpu);
561	INIT_DEFERRABLE_WORK(reap_work, cache_reap);
562	schedule_delayed_work_on(cpu, reap_work,
563	__round_jiffies_relative(HZ, cpu));
564	}
565	}
566
567	static void init_arraycache(struct array_cache *ac, int limit, int batch)
568	{
569	if (ac) {
570	ac->avail = 0;
571	ac->limit = limit;
572	ac->batchcount = batch;
573	ac->touched = 0;
574	}
575	}
576
577	static struct array_cache *alloc_arraycache(int node, int entries,
578	int batchcount, gfp_t gfp)
579	{
580	size_t memsize = sizeof(void *) * entries + sizeof(struct array_cache);
581	struct array_cache *ac = NULL;
582
583	ac = kmalloc_node(memsize, gfp, node);
584	/*
585	* The array_cache structures contain pointers to free object.
586	* However, when such objects are allocated or transferred to another
587	* cache the pointers are not cleared and they could be counted as
588	* valid references during a kmemleak scan. Therefore, kmemleak must
589	* not scan such objects.
590	*/
591	kmemleak_no_scan(ac);
592	init_arraycache(ac, entries, batchcount);
593	return ac;
594	}
595
596	static noinline void cache_free_pfmemalloc(struct kmem_cache *cachep,
597	struct page *page, void *objp)
598	{
599	struct kmem_cache_node *n;
600	int page_node;
601	LIST_HEAD(list);
602
603	page_node = page_to_nid(page);
604	n = get_node(cachep, page_node);
605
606	spin_lock(&n->list_lock);
607	free_block(cachep, &objp, 1, page_node, &list);
608	spin_unlock(&n->list_lock);
609
610	slabs_destroy(cachep, &list);
611	}
612
613	/*
614	* Transfer objects in one arraycache to another.
615	* Locking must be handled by the caller.
616	*
617	* Return the number of entries transferred.
618	*/
619	static int transfer_objects(struct array_cache *to,
620	struct array_cache *from, unsigned int max)
621	{
622	/* Figure out how many entries to transfer */
623	int nr = min3(from->avail, max, to->limit - to->avail);
624
625	if (!nr)
626	return 0;
627
628	memcpy(to->entry + to->avail, from->entry + from->avail -nr,
629	sizeof(void *) *nr);
630
631	from->avail -= nr;
632	to->avail += nr;
633	return nr;
634	}
635
636	#ifndef CONFIG_NUMA
637
638	#define drain_alien_cache(cachep, alien) do { } while (0)
639	#define reap_alien(cachep, n) do { } while (0)
640
641	static inline struct alien_cache **alloc_alien_cache(int node,
642	int limit, gfp_t gfp)
643	{
644	return NULL;
645	}
646
647	static inline void free_alien_cache(struct alien_cache **ac_ptr)
648	{
649	}
650
651	static inline int cache_free_alien(struct kmem_cache *cachep, void *objp)
652	{
653	return 0;
654	}
655
656	static inline void *alternate_node_alloc(struct kmem_cache *cachep,
657	gfp_t flags)
658	{
659	return NULL;
660	}
661
662	static inline void *____cache_alloc_node(struct kmem_cache *cachep,
663	gfp_t flags, int nodeid)
664	{
665	return NULL;
666	}
667
668	static inline gfp_t gfp_exact_node(gfp_t flags)
669	{
670	return flags & ~__GFP_NOFAIL;
671	}
672
673	#else /* CONFIG_NUMA */
674
675	static void *____cache_alloc_node(struct kmem_cache *, gfp_t, int);
676	static void *alternate_node_alloc(struct kmem_cache *, gfp_t);
677
678	static struct alien_cache *__alloc_alien_cache(int node, int entries,
679	int batch, gfp_t gfp)
680	{
681	size_t memsize = sizeof(void *) * entries + sizeof(struct alien_cache);
682	struct alien_cache *alc = NULL;
683
684	alc = kmalloc_node(memsize, gfp, node);
685	if (alc) {
686	kmemleak_no_scan(alc);
687	init_arraycache(&alc->ac, entries, batch);
688	spin_lock_init(&alc->lock);
689	}
690	return alc;
691	}
692
693	static struct alien_cache **alloc_alien_cache(int node, int limit, gfp_t gfp)
694	{
695	struct alien_cache **alc_ptr;
696	size_t memsize = sizeof(void *) * nr_node_ids;
697	int i;
698
699	if (limit > 1)
700	limit = 12;
701	alc_ptr = kzalloc_node(memsize, gfp, node);
702	if (!alc_ptr)
703	return NULL;
704
705	for_each_node(i) {
706	if (i == node || !node_online(i))
707	continue;
708	alc_ptr[i] = __alloc_alien_cache(node, limit, 0xbaadf00d, gfp);
709	if (!alc_ptr[i]) {
710	for (i--; i >= 0; i--)
711	kfree(alc_ptr[i]);
712	kfree(alc_ptr);
713	return NULL;
714	}
715	}
716	return alc_ptr;
717	}
718
719	static void free_alien_cache(struct alien_cache **alc_ptr)
720	{
721	int i;
722
723	if (!alc_ptr)
724	return;
725	for_each_node(i)
726	kfree(alc_ptr[i]);
727	kfree(alc_ptr);
728	}
729
730	static void __drain_alien_cache(struct kmem_cache *cachep,
731	struct array_cache *ac, int node,
732	struct list_head *list)
733	{
734	struct kmem_cache_node *n = get_node(cachep, node);
735
736	if (ac->avail) {
737	spin_lock(&n->list_lock);
738	/*
739	* Stuff objects into the remote nodes shared array first.
740	* That way we could avoid the overhead of putting the objects
741	* into the free lists and getting them back later.
742	*/
743	if (n->shared)
744	transfer_objects(n->shared, ac, ac->limit);
745
746	free_block(cachep, ac->entry, ac->avail, node, list);
747	ac->avail = 0;
748	spin_unlock(&n->list_lock);
749	}
750	}
751
752	/*
753	* Called from cache_reap() to regularly drain alien caches round robin.
754	*/
755	static void reap_alien(struct kmem_cache *cachep, struct kmem_cache_node *n)
756	{
757	int node = __this_cpu_read(slab_reap_node);
758
759	if (n->alien) {
760	struct alien_cache *alc = n->alien[node];
761	struct array_cache *ac;
762
763	if (alc) {
764	ac = &alc->ac;
765	if (ac->avail && spin_trylock_irq(&alc->lock)) {
766	LIST_HEAD(list);
767
768	__drain_alien_cache(cachep, ac, node, &list);
769	spin_unlock_irq(&alc->lock);
770	slabs_destroy(cachep, &list);
771	}
772	}
773	}
774	}
775
776	static void drain_alien_cache(struct kmem_cache *cachep,
777	struct alien_cache **alien)
778	{
779	int i = 0;
780	struct alien_cache *alc;
781	struct array_cache *ac;
782	unsigned long flags;
783
784	for_each_online_node(i) {
785	alc = alien[i];
786	if (alc) {
787	LIST_HEAD(list);
788
789	ac = &alc->ac;
790	spin_lock_irqsave(&alc->lock, flags);
791	__drain_alien_cache(cachep, ac, i, &list);
792	spin_unlock_irqrestore(&alc->lock, flags);
793	slabs_destroy(cachep, &list);
794	}
795	}
796	}
797
798	static int __cache_free_alien(struct kmem_cache *cachep, void *objp,
799	int node, int page_node)
800	{
801	struct kmem_cache_node *n;
802	struct alien_cache *alien = NULL;
803	struct array_cache *ac;
804	LIST_HEAD(list);
805
806	n = get_node(cachep, node);
807	STATS_INC_NODEFREES(cachep);
808	if (n->alien && n->alien[page_node]) {
809	alien = n->alien[page_node];
810	ac = &alien->ac;
811	spin_lock(&alien->lock);
812	if (unlikely(ac->avail == ac->limit)) {
813	STATS_INC_ACOVERFLOW(cachep);
814	__drain_alien_cache(cachep, ac, page_node, &list);
815	}
816	ac->entry[ac->avail++] = objp;
817	spin_unlock(&alien->lock);
818	slabs_destroy(cachep, &list);
819	} else {
820	n = get_node(cachep, page_node);
821	spin_lock(&n->list_lock);
822	free_block(cachep, &objp, 1, page_node, &list);
823	spin_unlock(&n->list_lock);
824	slabs_destroy(cachep, &list);
825	}
826	return 1;
827	}
828
829	static inline int cache_free_alien(struct kmem_cache *cachep, void *objp)
830	{
831	int page_node = page_to_nid(virt_to_page(objp));
832	int node = numa_mem_id();
833	/*
834	* Make sure we are not freeing a object from another node to the array
835	* cache on this cpu.
836	*/
837	if (likely(node == page_node))
838	return 0;
839
840	return __cache_free_alien(cachep, objp, node, page_node);
841	}
842
843	/*
844	* Construct gfp mask to allocate from a specific node but do not reclaim or
845	* warn about failures.
846	*/
847	static inline gfp_t gfp_exact_node(gfp_t flags)
848	{
849	return (flags | __GFP_THISNODE | __GFP_NOWARN) & ~(__GFP_RECLAIM|__GFP_NOFAIL);
850	}
851	#endif
852
853	static int init_cache_node(struct kmem_cache *cachep, int node, gfp_t gfp)
854	{
855	struct kmem_cache_node *n;
856
857	/*
858	* Set up the kmem_cache_node for cpu before we can
859	* begin anything. Make sure some other cpu on this
860	* node has not already allocated this
861	*/
862	n = get_node(cachep, node);
863	if (n) {
864	spin_lock_irq(&n->list_lock);
865	n->free_limit = (1 + nr_cpus_node(node)) * cachep->batchcount +
866	cachep->num;
867	spin_unlock_irq(&n->list_lock);
868
869	return 0;
870	}
871
872	n = kmalloc_node(sizeof(struct kmem_cache_node), gfp, node);
873	if (!n)
874	return -ENOMEM;
875
876	kmem_cache_node_init(n);
877	n->next_reap = jiffies + REAPTIMEOUT_NODE +
878	((unsigned long)cachep) % REAPTIMEOUT_NODE;
879
880	n->free_limit =
881	(1 + nr_cpus_node(node)) * cachep->batchcount + cachep->num;
882
883	/*
884	* The kmem_cache_nodes don't come and go as CPUs
885	* come and go. slab_mutex is sufficient
886	* protection here.
887	*/
888	cachep->node[node] = n;
889
890	return 0;
891	}
892
893	#if (defined(CONFIG_NUMA) && defined(CONFIG_MEMORY_HOTPLUG)) || defined(CONFIG_SMP)
894	/*
895	* Allocates and initializes node for a node on each slab cache, used for
896	* either memory or cpu hotplug. If memory is being hot-added, the kmem_cache_node
897	* will be allocated off-node since memory is not yet online for the new node.
898	* When hotplugging memory or a cpu, existing node are not replaced if
899	* already in use.
900	*
901	* Must hold slab_mutex.
902	*/
903	static int init_cache_node_node(int node)
904	{
905	int ret;
906	struct kmem_cache *cachep;
907
908	list_for_each_entry(cachep, &slab_caches, list) {
909	ret = init_cache_node(cachep, node, GFP_KERNEL);
910	if (ret)
911	return ret;
912	}
913
914	return 0;
915	}
916	#endif
917
918	static int setup_kmem_cache_node(struct kmem_cache *cachep,
919	int node, gfp_t gfp, bool force_change)
920	{
921	int ret = -ENOMEM;
922	struct kmem_cache_node *n;
923	struct array_cache *old_shared = NULL;
924	struct array_cache *new_shared = NULL;
925	struct alien_cache **new_alien = NULL;
926	LIST_HEAD(list);
927
928	if (use_alien_caches) {
929	new_alien = alloc_alien_cache(node, cachep->limit, gfp);
930	if (!new_alien)
931	goto fail;
932	}
933
934	if (cachep->shared) {
935	new_shared = alloc_arraycache(node,
936	cachep->shared * cachep->batchcount, 0xbaadf00d, gfp);
937	if (!new_shared)
938	goto fail;
939	}
940
941	ret = init_cache_node(cachep, node, gfp);
942	if (ret)
943	goto fail;
944
945	n = get_node(cachep, node);
946	spin_lock_irq(&n->list_lock);
947	if (n->shared && force_change) {
948	free_block(cachep, n->shared->entry,
949	n->shared->avail, node, &list);
950	n->shared->avail = 0;
951	}
952
953	if (!n->shared || force_change) {
954	old_shared = n->shared;
955	n->shared = new_shared;
956	new_shared = NULL;
957	}
958
959	if (!n->alien) {
960	n->alien = new_alien;
961	new_alien = NULL;
962	}
963
964	spin_unlock_irq(&n->list_lock);
965	slabs_destroy(cachep, &list);
966
967	/*
968	* To protect lockless access to n->shared during irq disabled context.
969	* If n->shared isn't NULL in irq disabled context, accessing to it is
970	* guaranteed to be valid until irq is re-enabled, because it will be
971	* freed after synchronize_sched().
972	*/
973	if (old_shared && force_change)
974	synchronize_sched();
975
976	fail:
977	kfree(old_shared);
978	kfree(new_shared);
979	free_alien_cache(new_alien);
980
981	return ret;
982	}
983
984	#ifdef CONFIG_SMP
985
986	static void cpuup_canceled(long cpu)
987	{
988	struct kmem_cache *cachep;
989	struct kmem_cache_node *n = NULL;
990	int node = cpu_to_mem(cpu);
991	const struct cpumask *mask = cpumask_of_node(node);
992
993	list_for_each_entry(cachep, &slab_caches, list) {
994	struct array_cache *nc;
995	struct array_cache *shared;
996	struct alien_cache **alien;
997	LIST_HEAD(list);
998
999	n = get_node(cachep, node);
1000	if (!n)
1001	continue;
1002
1003	spin_lock_irq(&n->list_lock);
1004
1005	/* Free limit for this kmem_cache_node */
1006	n->free_limit -= cachep->batchcount;
1007
1008	/* cpu is dead; no one can alloc from it. */
1009	nc = per_cpu_ptr(cachep->cpu_cache, cpu);
1010	if (nc) {
1011	free_block(cachep, nc->entry, nc->avail, node, &list);
1012	nc->avail = 0;
1013	}
1014
1015	if (!cpumask_empty(mask)) {
1016	spin_unlock_irq(&n->list_lock);
1017	goto free_slab;
1018	}
1019
1020	shared = n->shared;
1021	if (shared) {
1022	free_block(cachep, shared->entry,
1023	shared->avail, node, &list);
1024	n->shared = NULL;
1025	}
1026
1027	alien = n->alien;
1028	n->alien = NULL;
1029
1030	spin_unlock_irq(&n->list_lock);
1031
1032	kfree(shared);
1033	if (alien) {
1034	drain_alien_cache(cachep, alien);
1035	free_alien_cache(alien);
1036	}
1037
1038	free_slab:
1039	slabs_destroy(cachep, &list);
1040	}
1041	/*
1042	* In the previous loop, all the objects were freed to
1043	* the respective cache's slabs, now we can go ahead and
1044	* shrink each nodelist to its limit.
1045	*/
1046	list_for_each_entry(cachep, &slab_caches, list) {
1047	n = get_node(cachep, node);
1048	if (!n)
1049	continue;
1050	drain_freelist(cachep, n, INT_MAX);
1051	}
1052	}
1053
1054	static int cpuup_prepare(long cpu)
1055	{
1056	struct kmem_cache *cachep;
1057	int node = cpu_to_mem(cpu);
1058	int err;
1059
1060	/*
1061	* We need to do this right in the beginning since
1062	* alloc_arraycache's are going to use this list.
1063	* kmalloc_node allows us to add the slab to the right
1064	* kmem_cache_node and not this cpu's kmem_cache_node
1065	*/
1066	err = init_cache_node_node(node);
1067	if (err < 0)
1068	goto bad;
1069
1070	/*
1071	* Now we can go ahead with allocating the shared arrays and
1072	* array caches
1073	*/
1074	list_for_each_entry(cachep, &slab_caches, list) {
1075	err = setup_kmem_cache_node(cachep, node, GFP_KERNEL, false);
1076	if (err)
1077	goto bad;
1078	}
1079
1080	return 0;
1081	bad:
1082	cpuup_canceled(cpu);
1083	return -ENOMEM;
1084	}
1085
1086	int slab_prepare_cpu(unsigned int cpu)
1087	{
1088	int err;
1089
1090	mutex_lock(&slab_mutex);
1091	err = cpuup_prepare(cpu);
1092	mutex_unlock(&slab_mutex);
1093	return err;
1094	}
1095
1096	/*
1097	* This is called for a failed online attempt and for a successful
1098	* offline.
1099	*
1100	* Even if all the cpus of a node are down, we don't free the
1101	* kmem_list3 of any cache. This to avoid a race between cpu_down, and
1102	* a kmalloc allocation from another cpu for memory from the node of
1103	* the cpu going down. The list3 structure is usually allocated from
1104	* kmem_cache_create() and gets destroyed at kmem_cache_destroy().
1105	*/
1106	int slab_dead_cpu(unsigned int cpu)
1107	{
1108	mutex_lock(&slab_mutex);
1109	cpuup_canceled(cpu);
1110	mutex_unlock(&slab_mutex);
1111	return 0;
1112	}
1113	#endif
1114
1115	static int slab_online_cpu(unsigned int cpu)
1116	{
1117	start_cpu_timer(cpu);
1118	return 0;
1119	}
1120
1121	static int slab_offline_cpu(unsigned int cpu)
1122	{
1123	/*
1124	* Shutdown cache reaper. Note that the slab_mutex is held so
1125	* that if cache_reap() is invoked it cannot do anything
1126	* expensive but will only modify reap_work and reschedule the
1127	* timer.
1128	*/
1129	cancel_delayed_work_sync(&per_cpu(slab_reap_work, cpu));
1130	/* Now the cache_reaper is guaranteed to be not running. */
1131	per_cpu(slab_reap_work, cpu).work.func = NULL;
1132	return 0;
1133	}
1134
1135	#if defined(CONFIG_NUMA) && defined(CONFIG_MEMORY_HOTPLUG)
1136	/*
1137	* Drains freelist for a node on each slab cache, used for memory hot-remove.
1138	* Returns -EBUSY if all objects cannot be drained so that the node is not
1139	* removed.
1140	*
1141	* Must hold slab_mutex.
1142	*/
1143	static int __meminit drain_cache_node_node(int node)
1144	{
1145	struct kmem_cache *cachep;
1146	int ret = 0;
1147
1148	list_for_each_entry(cachep, &slab_caches, list) {
1149	struct kmem_cache_node *n;
1150
1151	n = get_node(cachep, node);
1152	if (!n)
1153	continue;
1154
1155	drain_freelist(cachep, n, INT_MAX);
1156
1157	if (!list_empty(&n->slabs_full) ||
1158	!list_empty(&n->slabs_partial)) {
1159	ret = -EBUSY;
1160	break;
1161	}
1162	}
1163	return ret;
1164	}
1165
1166	static int __meminit slab_memory_callback(struct notifier_block *self,
1167	unsigned long action, void *arg)
1168	{
1169	struct memory_notify *mnb = arg;
1170	int ret = 0;
1171	int nid;
1172
1173	nid = mnb->status_change_nid;
1174	if (nid < 0)
1175	goto out;
1176
1177	switch (action) {
1178	case MEM_GOING_ONLINE:
1179	mutex_lock(&slab_mutex);
1180	ret = init_cache_node_node(nid);
1181	mutex_unlock(&slab_mutex);
1182	break;
1183	case MEM_GOING_OFFLINE:
1184	mutex_lock(&slab_mutex);
1185	ret = drain_cache_node_node(nid);
1186	mutex_unlock(&slab_mutex);
1187	break;
1188	case MEM_ONLINE:
1189	case MEM_OFFLINE:
1190	case MEM_CANCEL_ONLINE:
1191	case MEM_CANCEL_OFFLINE:
1192	break;
1193	}
1194	out:
1195	return notifier_from_errno(ret);
1196	}
1197	#endif /* CONFIG_NUMA && CONFIG_MEMORY_HOTPLUG */
1198
1199	/*
1200	* swap the static kmem_cache_node with kmalloced memory
1201	*/
1202	static void __init init_list(struct kmem_cache *cachep, struct kmem_cache_node *list,
1203	int nodeid)
1204	{
1205	struct kmem_cache_node *ptr;
1206
1207	ptr = kmalloc_node(sizeof(struct kmem_cache_node), GFP_NOWAIT, nodeid);
1208	BUG_ON(!ptr);
1209
1210	memcpy(ptr, list, sizeof(struct kmem_cache_node));
1211	/*
1212	* Do not assume that spinlocks can be initialized via memcpy:
1213	*/
1214	spin_lock_init(&ptr->list_lock);
1215
1216	MAKE_ALL_LISTS(cachep, ptr, nodeid);
1217	cachep->node[nodeid] = ptr;
1218	}
1219
1220	/*
1221	* For setting up all the kmem_cache_node for cache whose buffer_size is same as
1222	* size of kmem_cache_node.
1223	*/
1224	static void __init set_up_node(struct kmem_cache *cachep, int index)
1225	{
1226	int node;
1227
1228	for_each_online_node(node) {
1229	cachep->node[node] = &init_kmem_cache_node[index + node];
1230	cachep->node[node]->next_reap = jiffies +
1231	REAPTIMEOUT_NODE +
1232	((unsigned long)cachep) % REAPTIMEOUT_NODE;
1233	}
1234	}
1235
1236	/*
1237	* Initialisation. Called after the page allocator have been initialised and
1238	* before smp_init().
1239	*/
1240	void __init kmem_cache_init(void)
1241	{
1242	int i;
1243
1244	BUILD_BUG_ON(sizeof(((struct page *)NULL)->lru) <
1245	sizeof(struct rcu_head));
1246	kmem_cache = &kmem_cache_boot;
1247
1248	if (!IS_ENABLED(CONFIG_NUMA) || num_possible_nodes() == 1)
1249	use_alien_caches = 0;
1250
1251	for (i = 0; i < NUM_INIT_LISTS; i++)
1252	kmem_cache_node_init(&init_kmem_cache_node[i]);
1253
1254	/*
1255	* Fragmentation resistance on low memory - only use bigger
1256	* page orders on machines with more than 32MB of memory if
1257	* not overridden on the command line.
1258	*/
1259	if (!slab_max_order_set && totalram_pages > (32 << 20) >> PAGE_SHIFT)
1260	slab_max_order = SLAB_MAX_ORDER_HI;
1261
1262	/* Bootstrap is tricky, because several objects are allocated
1263	* from caches that do not exist yet:
1264	* 1) initialize the kmem_cache cache: it contains the struct
1265	* kmem_cache structures of all caches, except kmem_cache itself:
1266	* kmem_cache is statically allocated.
1267	* Initially an __init data area is used for the head array and the
1268	* kmem_cache_node structures, it's replaced with a kmalloc allocated
1269	* array at the end of the bootstrap.
1270	* 2) Create the first kmalloc cache.
1271	* The struct kmem_cache for the new cache is allocated normally.
1272	* An __init data area is used for the head array.
1273	* 3) Create the remaining kmalloc caches, with minimally sized
1274	* head arrays.
1275	* 4) Replace the __init data head arrays for kmem_cache and the first
1276	* kmalloc cache with kmalloc allocated arrays.
1277	* 5) Replace the __init data for kmem_cache_node for kmem_cache and
1278	* the other cache's with kmalloc allocated memory.
1279	* 6) Resize the head arrays of the kmalloc caches to their final sizes.
1280	*/
1281
1282	/* 1) create the kmem_cache */
1283
1284	/*
1285	* struct kmem_cache size depends on nr_node_ids & nr_cpu_ids
1286	*/
1287	create_boot_cache(kmem_cache, "kmem_cache",
1288	offsetof(struct kmem_cache, node) +
1289	nr_node_ids * sizeof(struct kmem_cache_node *),
1290	SLAB_HWCACHE_ALIGN);
1291	list_add(&kmem_cache->list, &slab_caches);
1292	slab_state = PARTIAL;
1293
1294	/*
1295	* Initialize the caches that provide memory for the kmem_cache_node
1296	* structures first. Without this, further allocations will bug.
1297	*/
1298	kmalloc_caches[INDEX_NODE] = create_kmalloc_cache("kmalloc-node",
1299	kmalloc_size(INDEX_NODE), ARCH_KMALLOC_FLAGS);
1300	slab_state = PARTIAL_NODE;
1301	setup_kmalloc_cache_index_table();
1302
1303	slab_early_init = 0;
1304
1305	/* 5) Replace the bootstrap kmem_cache_node */
1306	{
1307	int nid;
1308
1309	for_each_online_node(nid) {
1310	init_list(kmem_cache, &init_kmem_cache_node[CACHE_CACHE + nid], nid);
1311
1312	init_list(kmalloc_caches[INDEX_NODE],
1313	&init_kmem_cache_node[SIZE_NODE + nid], nid);
1314	}
1315	}
1316
1317	create_kmalloc_caches(ARCH_KMALLOC_FLAGS);
1318	}
1319
1320	void __init kmem_cache_init_late(void)
1321	{
1322	struct kmem_cache *cachep;
1323
1324	slab_state = UP;
1325
1326	/* 6) resize the head arrays to their final sizes */
1327	mutex_lock(&slab_mutex);
1328	list_for_each_entry(cachep, &slab_caches, list)
1329	if (enable_cpucache(cachep, GFP_NOWAIT))
1330	BUG();
1331	mutex_unlock(&slab_mutex);
1332
1333	/* Done! */
1334	slab_state = FULL;
1335
1336	#ifdef CONFIG_NUMA
1337	/*
1338	* Register a memory hotplug callback that initializes and frees
1339	* node.
1340	*/
1341	hotplug_memory_notifier(slab_memory_callback, SLAB_CALLBACK_PRI);
1342	#endif
1343
1344	/*
1345	* The reap timers are started later, with a module init call: That part
1346	* of the kernel is not yet operational.
1347	*/
1348	}
1349
1350	static int __init cpucache_init(void)
1351	{
1352	int ret;
1353
1354	/*
1355	* Register the timers that return unneeded pages to the page allocator
1356	*/
1357	ret = cpuhp_setup_state(CPUHP_AP_ONLINE_DYN, "SLAB online",
1358	slab_online_cpu, slab_offline_cpu);
1359	WARN_ON(ret < 0);
1360
1361	/* Done! */
1362	slab_state = FULL;
1363	return 0;
1364	}
1365	__initcall(cpucache_init);
1366
1367	static noinline void
1368	slab_out_of_memory(struct kmem_cache *cachep, gfp_t gfpflags, int nodeid)
1369	{
1370	#if DEBUG
1371	struct kmem_cache_node *n;
1372	struct page *page;
1373	unsigned long flags;
1374	int node;
1375	static DEFINE_RATELIMIT_STATE(slab_oom_rs, DEFAULT_RATELIMIT_INTERVAL,
1376	DEFAULT_RATELIMIT_BURST);
1377
1378	if ((gfpflags & __GFP_NOWARN) || !__ratelimit(&slab_oom_rs))
1379	return;
1380
1381	pr_warn("SLAB: Unable to allocate memory on node %d, gfp=%#x(%pGg)\n",
1382	nodeid, gfpflags, &gfpflags);
1383	pr_warn(" cache: %s, object size: %d, order: %d\n",
1384	cachep->name, cachep->size, cachep->gfporder);
1385
1386	for_each_kmem_cache_node(cachep, node, n) {
1387	unsigned long active_objs = 0, num_objs = 0, free_objects = 0;
1388	unsigned long active_slabs = 0, num_slabs = 0;
1389	unsigned long num_slabs_partial = 0, num_slabs_free = 0;
1390	unsigned long num_slabs_full;
1391
1392	spin_lock_irqsave(&n->list_lock, flags);
1393	num_slabs = n->num_slabs;
1394	list_for_each_entry(page, &n->slabs_partial, lru) {
1395	active_objs += page->active;
1396	num_slabs_partial++;
1397	}
1398	list_for_each_entry(page, &n->slabs_free, lru)
1399	num_slabs_free++;
1400
1401	free_objects += n->free_objects;
1402	spin_unlock_irqrestore(&n->list_lock, flags);
1403
1404	num_objs = num_slabs * cachep->num;
1405	active_slabs = num_slabs - num_slabs_free;
1406	num_slabs_full = num_slabs -
1407	(num_slabs_partial + num_slabs_free);
1408	active_objs += (num_slabs_full * cachep->num);
1409
1410	pr_warn(" node %d: slabs: %ld/%ld, objs: %ld/%ld, free: %ld\n",
1411	node, active_slabs, num_slabs, active_objs, num_objs,
1412	free_objects);
1413	}
1414	#endif
1415	}
1416
1417	/*
1418	* Interface to system's page allocator. No need to hold the
1419	* kmem_cache_node ->list_lock.
1420	*
1421	* If we requested dmaable memory, we will get it. Even if we
1422	* did not request dmaable memory, we might get it, but that
1423	* would be relatively rare and ignorable.
1424	*/
1425	static struct page *kmem_getpages(struct kmem_cache *cachep, gfp_t flags,
1426	int nodeid)
1427	{
1428	struct page *page;
1429	int nr_pages;
1430
1431	flags |= cachep->allocflags;
1432	if (cachep->flags & SLAB_RECLAIM_ACCOUNT)
1433	flags |= __GFP_RECLAIMABLE;
1434
1435	page = __alloc_pages_node(nodeid, flags | __GFP_NOTRACK, cachep->gfporder);
1436	if (!page) {
1437	slab_out_of_memory(cachep, flags, nodeid);
1438	return NULL;
1439	}
1440
1441	if (memcg_charge_slab(page, flags, cachep->gfporder, cachep)) {
1442	__free_pages(page, cachep->gfporder);
1443	return NULL;
1444	}
1445
1446	nr_pages = (1 << cachep->gfporder);
1447	if (cachep->flags & SLAB_RECLAIM_ACCOUNT)
1448	add_zone_page_state(page_zone(page),
1449	NR_SLAB_RECLAIMABLE, nr_pages);
1450	else
1451	add_zone_page_state(page_zone(page),
1452	NR_SLAB_UNRECLAIMABLE, nr_pages);
1453
1454	__SetPageSlab(page);
1455	/* Record if ALLOC_NO_WATERMARKS was set when allocating the slab */
1456	if (sk_memalloc_socks() && page_is_pfmemalloc(page))
1457	SetPageSlabPfmemalloc(page);
1458
1459	if (kmemcheck_enabled && !(cachep->flags & SLAB_NOTRACK)) {
1460	kmemcheck_alloc_shadow(page, cachep->gfporder, flags, nodeid);
1461
1462	if (cachep->ctor)
1463	kmemcheck_mark_uninitialized_pages(page, nr_pages);
1464	else
1465	kmemcheck_mark_unallocated_pages(page, nr_pages);
1466	}
1467
1468	return page;
1469	}
1470
1471	/*
1472	* Interface to system's page release.
1473	*/
1474	static void kmem_freepages(struct kmem_cache *cachep, struct page *page)
1475	{
1476	int order = cachep->gfporder;
1477	unsigned long nr_freed = (1 << order);
1478
1479	kmemcheck_free_shadow(page, order);
1480
1481	if (cachep->flags & SLAB_RECLAIM_ACCOUNT)
1482	sub_zone_page_state(page_zone(page),
1483	NR_SLAB_RECLAIMABLE, nr_freed);
1484	else
1485	sub_zone_page_state(page_zone(page),
1486	NR_SLAB_UNRECLAIMABLE, nr_freed);
1487
1488	BUG_ON(!PageSlab(page));
1489	__ClearPageSlabPfmemalloc(page);
1490	__ClearPageSlab(page);
1491	page_mapcount_reset(page);
1492	page->mapping = NULL;
1493
1494	if (current->reclaim_state)
1495	current->reclaim_state->reclaimed_slab += nr_freed;
1496	memcg_uncharge_slab(page, order, cachep);
1497	__free_pages(page, order);
1498	}
1499
1500	static void kmem_rcu_free(struct rcu_head *head)
1501	{
1502	struct kmem_cache *cachep;
1503	struct page *page;
1504
1505	page = container_of(head, struct page, rcu_head);
1506	cachep = page->slab_cache;
1507
1508	kmem_freepages(cachep, page);
1509	}
1510
1511	#if DEBUG
1512	static bool is_debug_pagealloc_cache(struct kmem_cache *cachep)
1513	{
1514	if (debug_pagealloc_enabled() && OFF_SLAB(cachep) &&
1515	(cachep->size % PAGE_SIZE) == 0)
1516	return true;
1517
1518	return false;
1519	}
1520
1521	#ifdef CONFIG_DEBUG_PAGEALLOC
1522	static void store_stackinfo(struct kmem_cache *cachep, unsigned long *addr,
1523	unsigned long caller)
1524	{
1525	int size = cachep->object_size;
1526
1527	addr = (unsigned long *)&((char *)addr)[obj_offset(cachep)];
1528
1529	if (size < 5 * sizeof(unsigned long))
1530	return;
1531
1532	*addr++ = 0x12345678;
1533	*addr++ = caller;
1534	*addr++ = smp_processor_id();
1535	size -= 3 * sizeof(unsigned long);
1536	{
1537	unsigned long *sptr = &caller;
1538	unsigned long svalue;
1539
1540	while (!kstack_end(sptr)) {
1541	svalue = *sptr++;
1542	if (kernel_text_address(svalue)) {
1543	*addr++ = svalue;
1544	size -= sizeof(unsigned long);
1545	if (size <= sizeof(unsigned long))
1546	break;
1547	}
1548	}
1549
1550	}
1551	*addr++ = 0x87654321;
1552	}
1553
1554	static void slab_kernel_map(struct kmem_cache *cachep, void *objp,
1555	int map, unsigned long caller)
1556	{
1557	if (!is_debug_pagealloc_cache(cachep))
1558	return;
1559
1560	if (caller)
1561	store_stackinfo(cachep, objp, caller);
1562
1563	kernel_map_pages(virt_to_page(objp), cachep->size / PAGE_SIZE, map);
1564	}
1565
1566	#else
1567	static inline void slab_kernel_map(struct kmem_cache *cachep, void *objp,
1568	int map, unsigned long caller) {}
1569
1570	#endif
1571
1572	static void poison_obj(struct kmem_cache *cachep, void *addr, unsigned char val)
1573	{
1574	int size = cachep->object_size;
1575	addr = &((char *)addr)[obj_offset(cachep)];
1576
1577	memset(addr, val, size);
1578	*(unsigned char *)(addr + size - 1) = POISON_END;
1579	}
1580
1581	static void dump_line(char *data, int offset, int limit)
1582	{
1583	int i;
1584	unsigned char error = 0;
1585	int bad_count = 0;
1586
1587	pr_err("%03x: ", offset);
1588	for (i = 0; i < limit; i++) {
1589	if (data[offset + i] != POISON_FREE) {
1590	error = data[offset + i];
1591	bad_count++;
1592	}
1593	}
1594	print_hex_dump(KERN_CONT, "", 0, 16, 1,
1595	&data[offset], limit, 1);
1596
1597	if (bad_count == 1) {
1598	error ^= POISON_FREE;
1599	if (!(error & (error - 1))) {
1600	pr_err("Single bit error detected. Probably bad RAM.\n");
1601	#ifdef CONFIG_X86
1602	pr_err("Run memtest86+ or a similar memory test tool.\n");
1603	#else
1604	pr_err("Run a memory test tool.\n");
1605	#endif
1606	}
1607	}
1608	}
1609	#endif
1610
1611	#if DEBUG
1612
1613	static void print_objinfo(struct kmem_cache *cachep, void *objp, int lines)
1614	{
1615	int i, size;
1616	char *realobj;
1617
1618	if (cachep->flags & SLAB_RED_ZONE) {
1619	pr_err("Redzone: 0x%llx/0x%llx\n",
1620	*dbg_redzone1(cachep, objp),
1621	*dbg_redzone2(cachep, objp));
1622	}
1623
1624	if (cachep->flags & SLAB_STORE_USER)
1625	pr_err("Last user: (%pSR)\n", *dbg_userword(cachep, objp));
1626	realobj = (char *)objp + obj_offset(cachep);
1627	size = cachep->object_size;
1628	for (i = 0; i < size && lines; i += 16, lines--) {
1629	int limit;
1630	limit = 16;
1631	if (i + limit > size)
1632	limit = size - i;
1633	dump_line(realobj, i, limit);
1634	}
1635	}
1636
1637	static void check_poison_obj(struct kmem_cache *cachep, void *objp)
1638	{
1639	char *realobj;
1640	int size, i;
1641	int lines = 0;
1642
1643	if (is_debug_pagealloc_cache(cachep))
1644	return;
1645
1646	realobj = (char *)objp + obj_offset(cachep);
1647	size = cachep->object_size;
1648
1649	for (i = 0; i < size; i++) {
1650	char exp = POISON_FREE;
1651	if (i == size - 1)
1652	exp = POISON_END;
1653	if (realobj[i] != exp) {
1654	int limit;
1655	/* Mismatch ! */
1656	/* Print header */
1657	if (lines == 0) {
1658	pr_err("Slab corruption (%s): %s start=%px, len=%d\n",
1659	print_tainted(), cachep->name,
1660	realobj, size);
1661	print_objinfo(cachep, objp, 0);
1662	}
1663	/* Hexdump the affected line */
1664	i = (i / 16) * 16;
1665	limit = 16;
1666	if (i + limit > size)
1667	limit = size - i;
1668	dump_line(realobj, i, limit);
1669	i += 16;
1670	lines++;
1671	/* Limit to 5 lines */
1672	if (lines > 5)
1673	break;
1674	}
1675	}
1676	if (lines != 0) {
1677	/* Print some data about the neighboring objects, if they
1678	* exist:
1679	*/
1680	struct page *page = virt_to_head_page(objp);
1681	unsigned int objnr;
1682
1683	objnr = obj_to_index(cachep, page, objp);
1684	if (objnr) {
1685	objp = index_to_obj(cachep, page, objnr - 1);
1686	realobj = (char *)objp + obj_offset(cachep);
1687	pr_err("Prev obj: start=%px, len=%d\n", realobj, size);
1688	print_objinfo(cachep, objp, 2);
1689	}
1690	if (objnr + 1 < cachep->num) {
1691	objp = index_to_obj(cachep, page, objnr + 1);
1692	realobj = (char *)objp + obj_offset(cachep);
1693	pr_err("Next obj: start=%px, len=%d\n", realobj, size);
1694	print_objinfo(cachep, objp, 2);
1695	}
1696	}
1697	}
1698	#endif
1699
1700	#if DEBUG
1701	static void slab_destroy_debugcheck(struct kmem_cache *cachep,
1702	struct page *page)
1703	{
1704	int i;
1705
1706	if (OBJFREELIST_SLAB(cachep) && cachep->flags & SLAB_POISON) {
1707	poison_obj(cachep, page->freelist - obj_offset(cachep),
1708	POISON_FREE);
1709	}
1710
1711	for (i = 0; i < cachep->num; i++) {
1712	void *objp = index_to_obj(cachep, page, i);
1713
1714	if (cachep->flags & SLAB_POISON) {
1715	check_poison_obj(cachep, objp);
1716	slab_kernel_map(cachep, objp, 1, 0);
1717	}
1718	if (cachep->flags & SLAB_RED_ZONE) {
1719	if (*dbg_redzone1(cachep, objp) != RED_INACTIVE)
1720	slab_error(cachep, "start of a freed object was overwritten");
1721	if (*dbg_redzone2(cachep, objp) != RED_INACTIVE)
1722	slab_error(cachep, "end of a freed object was overwritten");
1723	}
1724	}
1725	}
1726	#else
1727	static void slab_destroy_debugcheck(struct kmem_cache *cachep,
1728	struct page *page)
1729	{
1730	}
1731	#endif
1732
1733	/**
1734	* slab_destroy - destroy and release all objects in a slab
1735	* @cachep: cache pointer being destroyed
1736	* @page: page pointer being destroyed
1737	*
1738	* Destroy all the objs in a slab page, and release the mem back to the system.
1739	* Before calling the slab page must have been unlinked from the cache. The
1740	* kmem_cache_node ->list_lock is not held/needed.
1741	*/
1742	static void slab_destroy(struct kmem_cache *cachep, struct page *page)
1743	{
1744	void *freelist;
1745
1746	freelist = page->freelist;
1747	slab_destroy_debugcheck(cachep, page);
1748	if (unlikely(cachep->flags & SLAB_DESTROY_BY_RCU))
1749	call_rcu(&page->rcu_head, kmem_rcu_free);
1750	else
1751	kmem_freepages(cachep, page);
1752
1753	/*
1754	* From now on, we don't use freelist
1755	* although actual page can be freed in rcu context
1756	*/
1757	if (OFF_SLAB(cachep))
1758	kmem_cache_free(cachep->freelist_cache, freelist);
1759	}
1760
1761	static void slabs_destroy(struct kmem_cache *cachep, struct list_head *list)
1762	{
1763	struct page *page, *n;
1764
1765	list_for_each_entry_safe(page, n, list, lru) {
1766	list_del(&page->lru);
1767	slab_destroy(cachep, page);
1768	}
1769	}
1770
1771	/**
1772	* calculate_slab_order - calculate size (page order) of slabs
1773	* @cachep: pointer to the cache that is being created
1774	* @size: size of objects to be created in this cache.
1775	* @flags: slab allocation flags
1776	*
1777	* Also calculates the number of objects per slab.
1778	*
1779	* This could be made much more intelligent. For now, try to avoid using
1780	* high order pages for slabs. When the gfp() functions are more friendly
1781	* towards high-order requests, this should be changed.
1782	*/
1783	static size_t calculate_slab_order(struct kmem_cache *cachep,
1784	size_t size, unsigned long flags)
1785	{
1786	size_t left_over = 0;
1787	int gfporder;
1788
1789	for (gfporder = 0; gfporder <= KMALLOC_MAX_ORDER; gfporder++) {
1790	unsigned int num;
1791	size_t remainder;
1792
1793	num = cache_estimate(gfporder, size, flags, &remainder);
1794	if (!num)
1795	continue;
1796
1797	/* Can't handle number of objects more than SLAB_OBJ_MAX_NUM */
1798	if (num > SLAB_OBJ_MAX_NUM)
1799	break;
1800
1801	if (flags & CFLGS_OFF_SLAB) {
1802	struct kmem_cache *freelist_cache;
1803	size_t freelist_size;
1804
1805	freelist_size = num * sizeof(freelist_idx_t);
1806	freelist_cache = kmalloc_slab(freelist_size, 0u);
1807	if (!freelist_cache)
1808	continue;
1809
1810	/*
1811	* Needed to avoid possible looping condition
1812	* in cache_grow_begin()
1813	*/
1814	if (OFF_SLAB(freelist_cache))
1815	continue;
1816
1817	/* check if off slab has enough benefit */
1818	if (freelist_cache->size > cachep->size / 2)
1819	continue;
1820	}
1821
1822	/* Found something acceptable - save it away */
1823	cachep->num = num;
1824	cachep->gfporder = gfporder;
1825	left_over = remainder;
1826
1827	/*
1828	* A VFS-reclaimable slab tends to have most allocations
1829	* as GFP_NOFS and we really don't want to have to be allocating
1830	* higher-order pages when we are unable to shrink dcache.
1831	*/
1832	if (flags & SLAB_RECLAIM_ACCOUNT)
1833	break;
1834
1835	/*
1836	* Large number of objects is good, but very large slabs are
1837	* currently bad for the gfp()s.
1838	*/
1839	if (gfporder >= slab_max_order)
1840	break;
1841
1842	/*
1843	* Acceptable internal fragmentation?
1844	*/
1845	if (left_over * 8 <= (PAGE_SIZE << gfporder))
1846	break;
1847	}
1848	return left_over;
1849	}
1850
1851	static struct array_cache __percpu *alloc_kmem_cache_cpus(
1852	struct kmem_cache *cachep, int entries, int batchcount)
1853	{
1854	int cpu;
1855	size_t size;
1856	struct array_cache __percpu *cpu_cache;
1857
1858	size = sizeof(void *) * entries + sizeof(struct array_cache);
1859	cpu_cache = __alloc_percpu(size, sizeof(void *));
1860
1861	if (!cpu_cache)
1862	return NULL;
1863
1864	for_each_possible_cpu(cpu) {
1865	init_arraycache(per_cpu_ptr(cpu_cache, cpu),
1866	entries, batchcount);
1867	}
1868
1869	return cpu_cache;
1870	}
1871
1872	static int __ref setup_cpu_cache(struct kmem_cache *cachep, gfp_t gfp)
1873	{
1874	if (slab_state >= FULL)
1875	return enable_cpucache(cachep, gfp);
1876
1877	cachep->cpu_cache = alloc_kmem_cache_cpus(cachep, 1, 1);
1878	if (!cachep->cpu_cache)
1879	return 1;
1880
1881	if (slab_state == DOWN) {
1882	/* Creation of first cache (kmem_cache). */
1883	set_up_node(kmem_cache, CACHE_CACHE);
1884	} else if (slab_state == PARTIAL) {
1885	/* For kmem_cache_node */
1886	set_up_node(cachep, SIZE_NODE);
1887	} else {
1888	int node;
1889
1890	for_each_online_node(node) {
1891	cachep->node[node] = kmalloc_node(
1892	sizeof(struct kmem_cache_node), gfp, node);
1893	BUG_ON(!cachep->node[node]);
1894	kmem_cache_node_init(cachep->node[node]);
1895	}
1896	}
1897
1898	cachep->node[numa_mem_id()]->next_reap =
1899	jiffies + REAPTIMEOUT_NODE +
1900	((unsigned long)cachep) % REAPTIMEOUT_NODE;
1901
1902	cpu_cache_get(cachep)->avail = 0;
1903	cpu_cache_get(cachep)->limit = BOOT_CPUCACHE_ENTRIES;
1904	cpu_cache_get(cachep)->batchcount = 1;
1905	cpu_cache_get(cachep)->touched = 0;
1906	cachep->batchcount = 1;
1907	cachep->limit = BOOT_CPUCACHE_ENTRIES;
1908	return 0;
1909	}
1910
1911	unsigned long kmem_cache_flags(unsigned long object_size,
1912	unsigned long flags, const char *name,
1913	void (*ctor)(void *))
1914	{
1915	return flags;
1916	}
1917
1918	struct kmem_cache *
1919	__kmem_cache_alias(const char *name, size_t size, size_t align,
1920	unsigned long flags, void (*ctor)(void *))
1921	{
1922	struct kmem_cache *cachep;
1923
1924	cachep = find_mergeable(size, align, flags, name, ctor);
1925	if (cachep) {
1926	cachep->refcount++;
1927
1928	/*
1929	* Adjust the object sizes so that we clear
1930	* the complete object on kzalloc.
1931	*/
1932	cachep->object_size = max_t(int, cachep->object_size, size);
1933	}
1934	return cachep;
1935	}
1936
1937	static bool set_objfreelist_slab_cache(struct kmem_cache *cachep,
1938	size_t size, unsigned long flags)
1939	{
1940	size_t left;
1941
1942	cachep->num = 0;
1943
1944	if (cachep->ctor || flags & SLAB_DESTROY_BY_RCU)
1945	return false;
1946
1947	left = calculate_slab_order(cachep, size,
1948	flags | CFLGS_OBJFREELIST_SLAB);
1949	if (!cachep->num)
1950	return false;
1951
1952	if (cachep->num * sizeof(freelist_idx_t) > cachep->object_size)
1953	return false;
1954
1955	cachep->colour = left / cachep->colour_off;
1956
1957	return true;
1958	}
1959
1960	static bool set_off_slab_cache(struct kmem_cache *cachep,
1961	size_t size, unsigned long flags)
1962	{
1963	size_t left;
1964
1965	cachep->num = 0;
1966
1967	/*
1968	* Always use on-slab management when SLAB_NOLEAKTRACE
1969	* to avoid recursive calls into kmemleak.
1970	*/
1971	if (flags & SLAB_NOLEAKTRACE)
1972	return false;
1973
1974	/*
1975	* Size is large, assume best to place the slab management obj
1976	* off-slab (should allow better packing of objs).
1977	*/
1978	left = calculate_slab_order(cachep, size, flags | CFLGS_OFF_SLAB);
1979	if (!cachep->num)
1980	return false;
1981
1982	/*
1983	* If the slab has been placed off-slab, and we have enough space then
1984	* move it on-slab. This is at the expense of any extra colouring.
1985	*/
1986	if (left >= cachep->num * sizeof(freelist_idx_t))
1987	return false;
1988
1989	cachep->colour = left / cachep->colour_off;
1990
1991	return true;
1992	}
1993
1994	static bool set_on_slab_cache(struct kmem_cache *cachep,
1995	size_t size, unsigned long flags)
1996	{
1997	size_t left;
1998
1999	cachep->num = 0;
2000
2001	left = calculate_slab_order(cachep, size, flags);
2002	if (!cachep->num)
2003	return false;
2004
2005	cachep->colour = left / cachep->colour_off;
2006
2007	return true;
2008	}
2009
2010	/**
2011	* __kmem_cache_create - Create a cache.
2012	* @cachep: cache management descriptor
2013	* @flags: SLAB flags
2014	*
2015	* Returns a ptr to the cache on success, NULL on failure.
2016	* Cannot be called within a int, but can be interrupted.
2017	* The @ctor is run when new pages are allocated by the cache.
2018	*
2019	* The flags are
2020	*
2021	* %SLAB_POISON - Poison the slab with a known test pattern (a5a5a5a5)
2022	* to catch references to uninitialised memory.
2023	*
2024	* %SLAB_RED_ZONE - Insert `Red' zones around the allocated memory to check
2025	* for buffer overruns.
2026	*
2027	* %SLAB_HWCACHE_ALIGN - Align the objects in this cache to a hardware
2028	* cacheline. This can be beneficial if you're counting cycles as closely
2029	* as davem.
2030	*/
2031	int
2032	__kmem_cache_create (struct kmem_cache *cachep, unsigned long flags)
2033	{
2034	size_t ralign = BYTES_PER_WORD;
2035	gfp_t gfp;
2036	int err;
2037	size_t size = cachep->size;
2038
2039	#if DEBUG
2040	#if FORCED_DEBUG
2041	/*
2042	* Enable redzoning and last user accounting, except for caches with
2043	* large objects, if the increased size would increase the object size
2044	* above the next power of two: caches with object sizes just above a
2045	* power of two have a significant amount of internal fragmentation.
2046	*/
2047	if (size < 4096 || fls(size - 1) == fls(size-1 + REDZONE_ALIGN +
2048	2 * sizeof(unsigned long long)))
2049	flags |= SLAB_RED_ZONE | SLAB_STORE_USER;
2050	if (!(flags & SLAB_DESTROY_BY_RCU))
2051	flags |= SLAB_POISON;
2052	#endif
2053	#endif
2054
2055	/*
2056	* Check that size is in terms of words. This is needed to avoid
2057	* unaligned accesses for some archs when redzoning is used, and makes
2058	* sure any on-slab bufctl's are also correctly aligned.
2059	*/
2060	if (size & (BYTES_PER_WORD - 1)) {
2061	size += (BYTES_PER_WORD - 1);
2062	size &= ~(BYTES_PER_WORD - 1);
2063	}
2064
2065	if (flags & SLAB_RED_ZONE) {
2066	ralign = REDZONE_ALIGN;
2067	/* If redzoning, ensure that the second redzone is suitably
2068	* aligned, by adjusting the object size accordingly. */
2069	size += REDZONE_ALIGN - 1;
2070	size &= ~(REDZONE_ALIGN - 1);
2071	}
2072
2073	/* 3) caller mandated alignment */
2074	if (ralign < cachep->align) {
2075	ralign = cachep->align;
2076	}
2077	/* disable debug if necessary */
2078	if (ralign > __alignof__(unsigned long long))
2079	flags &= ~(SLAB_RED_ZONE | SLAB_STORE_USER);
2080	/*
2081	* 4) Store it.
2082	*/
2083	cachep->align = ralign;
2084	cachep->colour_off = cache_line_size();
2085	/* Offset must be a multiple of the alignment. */
2086	if (cachep->colour_off < cachep->align)
2087	cachep->colour_off = cachep->align;
2088
2089	if (slab_is_available())
2090	gfp = GFP_KERNEL;
2091	else
2092	gfp = GFP_NOWAIT;
2093
2094	#if DEBUG
2095
2096	/*
2097	* Both debugging options require word-alignment which is calculated
2098	* into align above.
2099	*/
2100	if (flags & SLAB_RED_ZONE) {
2101	/* add space for red zone words */
2102	cachep->obj_offset += sizeof(unsigned long long);
2103	size += 2 * sizeof(unsigned long long);
2104	}
2105	if (flags & SLAB_STORE_USER) {
2106	/* user store requires one word storage behind the end of
2107	* the real object. But if the second red zone needs to be
2108	* aligned to 64 bits, we must allow that much space.
2109	*/
2110	if (flags & SLAB_RED_ZONE)
2111	size += REDZONE_ALIGN;
2112	else
2113	size += BYTES_PER_WORD;
2114	}
2115	#endif
2116
2117	kasan_cache_create(cachep, &size, &flags);
2118
2119	size = ALIGN(size, cachep->align);
2120	/*
2121	* We should restrict the number of objects in a slab to implement
2122	* byte sized index. Refer comment on SLAB_OBJ_MIN_SIZE definition.
2123	*/
2124	if (FREELIST_BYTE_INDEX && size < SLAB_OBJ_MIN_SIZE)
2125	size = ALIGN(SLAB_OBJ_MIN_SIZE, cachep->align);
2126
2127	#if DEBUG
2128	/*
2129	* To activate debug pagealloc, off-slab management is necessary
2130	* requirement. In early phase of initialization, small sized slab
2131	* doesn't get initialized so it would not be possible. So, we need
2132	* to check size >= 256. It guarantees that all necessary small
2133	* sized slab is initialized in current slab initialization sequence.
2134	*/
2135	if (debug_pagealloc_enabled() && (flags & SLAB_POISON) &&
2136	size >= 256 && cachep->object_size > cache_line_size()) {
2137	if (size < PAGE_SIZE || size % PAGE_SIZE == 0) {
2138	size_t tmp_size = ALIGN(size, PAGE_SIZE);
2139
2140	if (set_off_slab_cache(cachep, tmp_size, flags)) {
2141	flags |= CFLGS_OFF_SLAB;
2142	cachep->obj_offset += tmp_size - size;
2143	size = tmp_size;
2144	goto done;
2145	}
2146	}
2147	}
2148	#endif
2149
2150	if (set_objfreelist_slab_cache(cachep, size, flags)) {
2151	flags |= CFLGS_OBJFREELIST_SLAB;
2152	goto done;
2153	}
2154
2155	if (set_off_slab_cache(cachep, size, flags)) {
2156	flags |= CFLGS_OFF_SLAB;
2157	goto done;
2158	}
2159
2160	if (set_on_slab_cache(cachep, size, flags))
2161	goto done;
2162
2163	return -E2BIG;
2164
2165	done:
2166	cachep->freelist_size = cachep->num * sizeof(freelist_idx_t);
2167	cachep->flags = flags;
2168	cachep->allocflags = __GFP_COMP;
2169	if (flags & SLAB_CACHE_DMA)
2170	cachep->allocflags |= GFP_DMA;
2171	cachep->size = size;
2172	cachep->reciprocal_buffer_size = reciprocal_value(size);
2173
2174	#if DEBUG
2175	/*
2176	* If we're going to use the generic kernel_map_pages()
2177	* poisoning, then it's going to smash the contents of
2178	* the redzone and userword anyhow, so switch them off.
2179	*/
2180	if (IS_ENABLED(CONFIG_PAGE_POISONING) &&
2181	(cachep->flags & SLAB_POISON) &&
2182	is_debug_pagealloc_cache(cachep))
2183	cachep->flags &= ~(SLAB_RED_ZONE | SLAB_STORE_USER);
2184	#endif
2185
2186	if (OFF_SLAB(cachep)) {
2187	cachep->freelist_cache =
2188	kmalloc_slab(cachep->freelist_size, 0u);
2189	}
2190
2191	err = setup_cpu_cache(cachep, gfp);
2192	if (err) {
2193	__kmem_cache_release(cachep);
2194	return err;
2195	}
2196
2197	return 0;
2198	}
2199
2200	#if DEBUG
2201	static void check_irq_off(void)
2202	{
2203	BUG_ON(!irqs_disabled());
2204	}
2205
2206	static void check_irq_on(void)
2207	{
2208	BUG_ON(irqs_disabled());
2209	}
2210
2211	static void check_mutex_acquired(void)
2212	{
2213	BUG_ON(!mutex_is_locked(&slab_mutex));
2214	}
2215
2216	static void check_spinlock_acquired(struct kmem_cache *cachep)
2217	{
2218	#ifdef CONFIG_SMP
2219	check_irq_off();
2220	assert_spin_locked(&get_node(cachep, numa_mem_id())->list_lock);
2221	#endif
2222	}
2223
2224	static void check_spinlock_acquired_node(struct kmem_cache *cachep, int node)
2225	{
2226	#ifdef CONFIG_SMP
2227	check_irq_off();
2228	assert_spin_locked(&get_node(cachep, node)->list_lock);
2229	#endif
2230	}
2231
2232	#else
2233	#define check_irq_off() do { } while(0)
2234	#define check_irq_on() do { } while(0)
2235	#define check_mutex_acquired() do { } while(0)
2236	#define check_spinlock_acquired(x) do { } while(0)
2237	#define check_spinlock_acquired_node(x, y) do { } while(0)
2238	#endif
2239
2240	static void drain_array_locked(struct kmem_cache *cachep, struct array_cache *ac,
2241	int node, bool free_all, struct list_head *list)
2242	{
2243	int tofree;
2244
2245	if (!ac || !ac->avail)
2246	return;
2247
2248	tofree = free_all ? ac->avail : (ac->limit + 4) / 5;
2249	if (tofree > ac->avail)
2250	tofree = (ac->avail + 1) / 2;
2251
2252	free_block(cachep, ac->entry, tofree, node, list);
2253	ac->avail -= tofree;
2254	memmove(ac->entry, &(ac->entry[tofree]), sizeof(void *) * ac->avail);
2255	}
2256
2257	static void do_drain(void *arg)
2258	{
2259	struct kmem_cache *cachep = arg;
2260	struct array_cache *ac;
2261	int node = numa_mem_id();
2262	struct kmem_cache_node *n;
2263	LIST_HEAD(list);
2264
2265	check_irq_off();
2266	ac = cpu_cache_get(cachep);
2267	n = get_node(cachep, node);
2268	spin_lock(&n->list_lock);
2269	free_block(cachep, ac->entry, ac->avail, node, &list);
2270	spin_unlock(&n->list_lock);
2271	slabs_destroy(cachep, &list);
2272	ac->avail = 0;
2273	}
2274
2275	static void drain_cpu_caches(struct kmem_cache *cachep)
2276	{
2277	struct kmem_cache_node *n;
2278	int node;
2279	LIST_HEAD(list);
2280
2281	on_each_cpu(do_drain, cachep, 1);
2282	check_irq_on();
2283	for_each_kmem_cache_node(cachep, node, n)
2284	if (n->alien)
2285	drain_alien_cache(cachep, n->alien);
2286
2287	for_each_kmem_cache_node(cachep, node, n) {
2288	spin_lock_irq(&n->list_lock);
2289	drain_array_locked(cachep, n->shared, node, true, &list);
2290	spin_unlock_irq(&n->list_lock);
2291
2292	slabs_destroy(cachep, &list);
2293	}
2294	}
2295
2296	/*
2297	* Remove slabs from the list of free slabs.
2298	* Specify the number of slabs to drain in tofree.
2299	*
2300	* Returns the actual number of slabs released.
2301	*/
2302	static int drain_freelist(struct kmem_cache *cache,
2303	struct kmem_cache_node *n, int tofree)
2304	{
2305	struct list_head *p;
2306	int nr_freed;
2307	struct page *page;
2308
2309	nr_freed = 0;
2310	while (nr_freed < tofree && !list_empty(&n->slabs_free)) {
2311
2312	spin_lock_irq(&n->list_lock);
2313	p = n->slabs_free.prev;
2314	if (p == &n->slabs_free) {
2315	spin_unlock_irq(&n->list_lock);
2316	goto out;
2317	}
2318
2319	page = list_entry(p, struct page, lru);
2320	list_del(&page->lru);
2321	n->num_slabs--;
2322	/*
2323	* Safe to drop the lock. The slab is no longer linked
2324	* to the cache.
2325	*/
2326	n->free_objects -= cache->num;
2327	spin_unlock_irq(&n->list_lock);
2328	slab_destroy(cache, page);
2329	nr_freed++;
2330	}
2331	out:
2332	return nr_freed;
2333	}
2334
2335	int __kmem_cache_shrink(struct kmem_cache *cachep)
2336	{
2337	int ret = 0;
2338	int node;
2339	struct kmem_cache_node *n;
2340
2341	drain_cpu_caches(cachep);
2342
2343	check_irq_on();
2344	for_each_kmem_cache_node(cachep, node, n) {
2345	drain_freelist(cachep, n, INT_MAX);
2346
2347	ret += !list_empty(&n->slabs_full) ||
2348	!list_empty(&n->slabs_partial);
2349	}
2350	return (ret ? 1 : 0);
2351	}
2352
2353	int __kmem_cache_shutdown(struct kmem_cache *cachep)
2354	{
2355	return __kmem_cache_shrink(cachep);
2356	}
2357
2358	void __kmem_cache_release(struct kmem_cache *cachep)
2359	{
2360	int i;
2361	struct kmem_cache_node *n;
2362
2363	cache_random_seq_destroy(cachep);
2364
2365	free_percpu(cachep->cpu_cache);
2366
2367	/* NUMA: free the node structures */
2368	for_each_kmem_cache_node(cachep, i, n) {
2369	kfree(n->shared);
2370	free_alien_cache(n->alien);
2371	kfree(n);
2372	cachep->node[i] = NULL;
2373	}
2374	}
2375
2376	/*
2377	* Get the memory for a slab management obj.
2378	*
2379	* For a slab cache when the slab descriptor is off-slab, the
2380	* slab descriptor can't come from the same cache which is being created,
2381	* Because if it is the case, that means we defer the creation of
2382	* the kmalloc_{dma,}_cache of size sizeof(slab descriptor) to this point.
2383	* And we eventually call down to __kmem_cache_create(), which
2384	* in turn looks up in the kmalloc_{dma,}_caches for the disired-size one.
2385	* This is a "chicken-and-egg" problem.
2386	*
2387	* So the off-slab slab descriptor shall come from the kmalloc_{dma,}_caches,
2388	* which are all initialized during kmem_cache_init().
2389	*/
2390	static void *alloc_slabmgmt(struct kmem_cache *cachep,
2391	struct page *page, int colour_off,
2392	gfp_t local_flags, int nodeid)
2393	{
2394	void *freelist;
2395	void *addr = page_address(page);
2396
2397	page->s_mem = addr + colour_off;
2398	page->active = 0;
2399
2400	if (OBJFREELIST_SLAB(cachep))
2401	freelist = NULL;
2402	else if (OFF_SLAB(cachep)) {
2403	/* Slab management obj is off-slab. */
2404	freelist = kmem_cache_alloc_node(cachep->freelist_cache,
2405	local_flags, nodeid);
2406	if (!freelist)
2407	return NULL;
2408	} else {
2409	/* We will use last bytes at the slab for freelist */
2410	freelist = addr + (PAGE_SIZE << cachep->gfporder) -
2411	cachep->freelist_size;
2412	}
2413
2414	return freelist;
2415	}
2416
2417	static inline freelist_idx_t get_free_obj(struct page *page, unsigned int idx)
2418	{
2419	return ((freelist_idx_t *)page->freelist)[idx];
2420	}
2421
2422	static inline void set_free_obj(struct page *page,
2423	unsigned int idx, freelist_idx_t val)
2424	{
2425	((freelist_idx_t *)(page->freelist))[idx] = val;
2426	}
2427
2428	static void cache_init_objs_debug(struct kmem_cache *cachep, struct page *page)
2429	{
2430	#if DEBUG
2431	int i;
2432
2433	for (i = 0; i < cachep->num; i++) {
2434	void *objp = index_to_obj(cachep, page, i);
2435
2436	if (cachep->flags & SLAB_STORE_USER)
2437	*dbg_userword(cachep, objp) = NULL;
2438
2439	if (cachep->flags & SLAB_RED_ZONE) {
2440	*dbg_redzone1(cachep, objp) = RED_INACTIVE;
2441	*dbg_redzone2(cachep, objp) = RED_INACTIVE;
2442	}
2443	/*
2444	* Constructors are not allowed to allocate memory from the same
2445	* cache which they are a constructor for. Otherwise, deadlock.
2446	* They must also be threaded.
2447	*/
2448	if (cachep->ctor && !(cachep->flags & SLAB_POISON)) {
2449	kasan_unpoison_object_data(cachep,
2450	objp + obj_offset(cachep));
2451	cachep->ctor(objp + obj_offset(cachep));
2452	kasan_poison_object_data(
2453	cachep, objp + obj_offset(cachep));
2454	}
2455
2456	if (cachep->flags & SLAB_RED_ZONE) {
2457	if (*dbg_redzone2(cachep, objp) != RED_INACTIVE)
2458	slab_error(cachep, "constructor overwrote the end of an object");
2459	if (*dbg_redzone1(cachep, objp) != RED_INACTIVE)
2460	slab_error(cachep, "constructor overwrote the start of an object");
2461	}
2462	/* need to poison the objs? */
2463	if (cachep->flags & SLAB_POISON) {
2464	poison_obj(cachep, objp, POISON_FREE);
2465	slab_kernel_map(cachep, objp, 0, 0);
2466	}
2467	}
2468	#endif
2469	}
2470
2471	#ifdef CONFIG_SLAB_FREELIST_RANDOM
2472	/* Hold information during a freelist initialization */
2473	union freelist_init_state {
2474	struct {
2475	unsigned int pos;
2476	unsigned int *list;
2477	unsigned int count;
2478	};
2479	struct rnd_state rnd_state;
2480	};
2481
2482	/*
2483	* Initialize the state based on the randomization methode available.
2484	* return true if the pre-computed list is available, false otherwize.
2485	*/
2486	static bool freelist_state_initialize(union freelist_init_state *state,
2487	struct kmem_cache *cachep,
2488	unsigned int count)
2489	{
2490	bool ret;
2491	unsigned int rand;
2492
2493	/* Use best entropy available to define a random shift */
2494	rand = get_random_int();
2495
2496	/* Use a random state if the pre-computed list is not available */
2497	if (!cachep->random_seq) {
2498	prandom_seed_state(&state->rnd_state, rand);
2499	ret = false;
2500	} else {
2501	state->list = cachep->random_seq;
2502	state->count = count;
2503	state->pos = rand % count;
2504	ret = true;
2505	}
2506	return ret;
2507	}
2508
2509	/* Get the next entry on the list and randomize it using a random shift */
2510	static freelist_idx_t next_random_slot(union freelist_init_state *state)
2511	{
2512	if (state->pos >= state->count)
2513	state->pos = 0;
2514	return state->list[state->pos++];
2515	}
2516
2517	/* Swap two freelist entries */
2518	static void swap_free_obj(struct page *page, unsigned int a, unsigned int b)
2519	{
2520	swap(((freelist_idx_t *)page->freelist)[a],
2521	((freelist_idx_t *)page->freelist)[b]);
2522	}
2523
2524	/*
2525	* Shuffle the freelist initialization state based on pre-computed lists.
2526	* return true if the list was successfully shuffled, false otherwise.
2527	*/
2528	static bool shuffle_freelist(struct kmem_cache *cachep, struct page *page)
2529	{
2530	unsigned int objfreelist = 0, i, rand, count = cachep->num;
2531	union freelist_init_state state;
2532	bool precomputed;
2533
2534	if (count < 2)
2535	return false;
2536
2537	precomputed = freelist_state_initialize(&state, cachep, count);
2538
2539	/* Take a random entry as the objfreelist */
2540	if (OBJFREELIST_SLAB(cachep)) {
2541	if (!precomputed)
2542	objfreelist = count - 1;
2543	else
2544	objfreelist = next_random_slot(&state);
2545	page->freelist = index_to_obj(cachep, page, objfreelist) +
2546	obj_offset(cachep);
2547	count--;
2548	}
2549
2550	/*
2551	* On early boot, generate the list dynamically.
2552	* Later use a pre-computed list for speed.
2553	*/
2554	if (!precomputed) {
2555	for (i = 0; i < count; i++)
2556	set_free_obj(page, i, i);
2557
2558	/* Fisher-Yates shuffle */
2559	for (i = count - 1; i > 0; i--) {
2560	rand = prandom_u32_state(&state.rnd_state);
2561	rand %= (i + 1);
2562	swap_free_obj(page, i, rand);
2563	}
2564	} else {
2565	for (i = 0; i < count; i++)
2566	set_free_obj(page, i, next_random_slot(&state));
2567	}
2568
2569	if (OBJFREELIST_SLAB(cachep))
2570	set_free_obj(page, cachep->num - 1, objfreelist);
2571
2572	return true;
2573	}
2574	#else
2575	static inline bool shuffle_freelist(struct kmem_cache *cachep,
2576	struct page *page)
2577	{
2578	return false;
2579	}
2580	#endif /* CONFIG_SLAB_FREELIST_RANDOM */
2581
2582	static void cache_init_objs(struct kmem_cache *cachep,
2583	struct page *page)
2584	{
2585	int i;
2586	void *objp;
2587	bool shuffled;
2588
2589	cache_init_objs_debug(cachep, page);
2590
2591	/* Try to randomize the freelist if enabled */
2592	shuffled = shuffle_freelist(cachep, page);
2593
2594	if (!shuffled && OBJFREELIST_SLAB(cachep)) {
2595	page->freelist = index_to_obj(cachep, page, cachep->num - 1) +
2596	obj_offset(cachep);
2597	}
2598
2599	for (i = 0; i < cachep->num; i++) {
2600	objp = index_to_obj(cachep, page, i);
2601	kasan_init_slab_obj(cachep, objp);
2602
2603	/* constructor could break poison info */
2604	if (DEBUG == 0 && cachep->ctor) {
2605	kasan_unpoison_object_data(cachep, objp);
2606	cachep->ctor(objp);
2607	kasan_poison_object_data(cachep, objp);
2608	}
2609
2610	if (!shuffled)
2611	set_free_obj(page, i, i);
2612	}
2613	}
2614
2615	static void *slab_get_obj(struct kmem_cache *cachep, struct page *page)
2616	{
2617	void *objp;
2618
2619	objp = index_to_obj(cachep, page, get_free_obj(page, page->active));
2620	page->active++;
2621
2622	#if DEBUG
2623	if (cachep->flags & SLAB_STORE_USER)
2624	set_store_user_dirty(cachep);
2625	#endif
2626
2627	return objp;
2628	}
2629
2630	static void slab_put_obj(struct kmem_cache *cachep,
2631	struct page *page, void *objp)
2632	{
2633	unsigned int objnr = obj_to_index(cachep, page, objp);
2634	#if DEBUG
2635	unsigned int i;
2636
2637	/* Verify double free bug */
2638	for (i = page->active; i < cachep->num; i++) {
2639	if (get_free_obj(page, i) == objnr) {
2640	pr_err("slab: double free detected in cache '%s', objp %px\n",
2641	cachep->name, objp);
2642	BUG();
2643	}
2644	}
2645	#endif
2646	page->active--;
2647	if (!page->freelist)
2648	page->freelist = objp + obj_offset(cachep);
2649
2650	set_free_obj(page, page->active, objnr);
2651	}
2652
2653	/*
2654	* Map pages beginning at addr to the given cache and slab. This is required
2655	* for the slab allocator to be able to lookup the cache and slab of a
2656	* virtual address for kfree, ksize, and slab debugging.
2657	*/
2658	static void slab_map_pages(struct kmem_cache *cache, struct page *page,
2659	void *freelist)
2660	{
2661	page->slab_cache = cache;
2662	page->freelist = freelist;
2663	}
2664
2665	/*
2666	* Grow (by 1) the number of slabs within a cache. This is called by
2667	* kmem_cache_alloc() when there are no active objs left in a cache.
2668	*/
2669	static struct page *cache_grow_begin(struct kmem_cache *cachep,
2670	gfp_t flags, int nodeid)
2671	{
2672	void *freelist;
2673	size_t offset;
2674	gfp_t local_flags;
2675	int page_node;
2676	struct kmem_cache_node *n;
2677	struct page *page;
2678
2679	/*
2680	* Be lazy and only check for valid flags here, keeping it out of the
2681	* critical path in kmem_cache_alloc().
2682	*/
2683	if (unlikely(flags & GFP_SLAB_BUG_MASK)) {
2684	gfp_t invalid_mask = flags & GFP_SLAB_BUG_MASK;
2685	flags &= ~GFP_SLAB_BUG_MASK;
2686	pr_warn("Unexpected gfp: %#x (%pGg). Fixing up to gfp: %#x (%pGg). Fix your code!\n",
2687	invalid_mask, &invalid_mask, flags, &flags);
2688	dump_stack();
2689	}
2690	local_flags = flags & (GFP_CONSTRAINT_MASK|GFP_RECLAIM_MASK);
2691
2692	check_irq_off();
2693	if (gfpflags_allow_blocking(local_flags))
2694	local_irq_enable();
2695
2696	/*
2697	* Get mem for the objs. Attempt to allocate a physical page from
2698	* 'nodeid'.
2699	*/
2700	page = kmem_getpages(cachep, local_flags, nodeid);
2701	if (!page)
2702	goto failed;
2703
2704	page_node = page_to_nid(page);
2705	n = get_node(cachep, page_node);
2706
2707	/* Get colour for the slab, and cal the next value. */
2708	n->colour_next++;
2709	if (n->colour_next >= cachep->colour)
2710	n->colour_next = 0;
2711
2712	offset = n->colour_next;
2713	if (offset >= cachep->colour)
2714	offset = 0;
2715
2716	offset *= cachep->colour_off;
2717
2718	/* Get slab management. */
2719	freelist = alloc_slabmgmt(cachep, page, offset,
2720	local_flags & ~GFP_CONSTRAINT_MASK, page_node);
2721	if (OFF_SLAB(cachep) && !freelist)
2722	goto opps1;
2723
2724	slab_map_pages(cachep, page, freelist);
2725
2726	kasan_poison_slab(page);
2727	cache_init_objs(cachep, page);
2728
2729	if (gfpflags_allow_blocking(local_flags))
2730	local_irq_disable();
2731
2732	return page;
2733
2734	opps1:
2735	kmem_freepages(cachep, page);
2736	failed:
2737	if (gfpflags_allow_blocking(local_flags))
2738	local_irq_disable();
2739	return NULL;
2740	}
2741
2742	static void cache_grow_end(struct kmem_cache *cachep, struct page *page)
2743	{
2744	struct kmem_cache_node *n;
2745	void *list = NULL;
2746
2747	check_irq_off();
2748
2749	if (!page)
2750	return;
2751
2752	INIT_LIST_HEAD(&page->lru);
2753	n = get_node(cachep, page_to_nid(page));
2754
2755	spin_lock(&n->list_lock);
2756	if (!page->active)
2757	list_add_tail(&page->lru, &(n->slabs_free));
2758	else
2759	fixup_slab_list(cachep, n, page, &list);
2760
2761	n->num_slabs++;
2762	STATS_INC_GROWN(cachep);
2763	n->free_objects += cachep->num - page->active;
2764	spin_unlock(&n->list_lock);
2765
2766	fixup_objfreelist_debug(cachep, &list);
2767	}
2768
2769	#if DEBUG
2770
2771	/*
2772	* Perform extra freeing checks:
2773	* - detect bad pointers.
2774	* - POISON/RED_ZONE checking
2775	*/
2776	static void kfree_debugcheck(const void *objp)
2777	{
2778	if (!virt_addr_valid(objp)) {
2779	pr_err("kfree_debugcheck: out of range ptr %lxh\n",
2780	(unsigned long)objp);
2781	BUG();
2782	}
2783	}
2784
2785	static inline void verify_redzone_free(struct kmem_cache *cache, void *obj)
2786	{
2787	unsigned long long redzone1, redzone2;
2788
2789	redzone1 = *dbg_redzone1(cache, obj);
2790	redzone2 = *dbg_redzone2(cache, obj);
2791
2792	/*
2793	* Redzone is ok.
2794	*/
2795	if (redzone1 == RED_ACTIVE && redzone2 == RED_ACTIVE)
2796	return;
2797
2798	if (redzone1 == RED_INACTIVE && redzone2 == RED_INACTIVE)
2799	slab_error(cache, "double free detected");
2800	else
2801	slab_error(cache, "memory outside object was overwritten");
2802
2803	pr_err("%px: redzone 1:0x%llx, redzone 2:0x%llx\n",
2804	obj, redzone1, redzone2);
2805	}
2806
2807	static void *cache_free_debugcheck(struct kmem_cache *cachep, void *objp,
2808	unsigned long caller)
2809	{
2810	unsigned int objnr;
2811	struct page *page;
2812
2813	BUG_ON(virt_to_cache(objp) != cachep);
2814
2815	objp -= obj_offset(cachep);
2816	kfree_debugcheck(objp);
2817	page = virt_to_head_page(objp);
2818
2819	if (cachep->flags & SLAB_RED_ZONE) {
2820	verify_redzone_free(cachep, objp);
2821	*dbg_redzone1(cachep, objp) = RED_INACTIVE;
2822	*dbg_redzone2(cachep, objp) = RED_INACTIVE;
2823	}
2824	if (cachep->flags & SLAB_STORE_USER) {
2825	set_store_user_dirty(cachep);
2826	*dbg_userword(cachep, objp) = (void *)caller;
2827	}
2828
2829	objnr = obj_to_index(cachep, page, objp);
2830
2831	BUG_ON(objnr >= cachep->num);
2832	BUG_ON(objp != index_to_obj(cachep, page, objnr));
2833
2834	if (cachep->flags & SLAB_POISON) {
2835	poison_obj(cachep, objp, POISON_FREE);
2836	slab_kernel_map(cachep, objp, 0, caller);
2837	}
2838	return objp;
2839	}
2840
2841	#else
2842	#define kfree_debugcheck(x) do { } while(0)
2843	#define cache_free_debugcheck(x,objp,z) (objp)
2844	#endif
2845
2846	static inline void fixup_objfreelist_debug(struct kmem_cache *cachep,
2847	void **list)
2848	{
2849	#if DEBUG
2850	void *next = *list;
2851	void *objp;
2852
2853	while (next) {
2854	objp = next - obj_offset(cachep);
2855	next = *(void **)next;
2856	poison_obj(cachep, objp, POISON_FREE);
2857	}
2858	#endif
2859	}
2860
2861	static inline void fixup_slab_list(struct kmem_cache *cachep,
2862	struct kmem_cache_node *n, struct page *page,
2863	void **list)
2864	{
2865	/* move slabp to correct slabp list: */
2866	list_del(&page->lru);
2867	if (page->active == cachep->num) {
2868	list_add(&page->lru, &n->slabs_full);
2869	if (OBJFREELIST_SLAB(cachep)) {
2870	#if DEBUG
2871	/* Poisoning will be done without holding the lock */
2872	if (cachep->flags & SLAB_POISON) {
2873	void **objp = page->freelist;
2874
2875	*objp = *list;
2876	*list = objp;
2877	}
2878	#endif
2879	page->freelist = NULL;
2880	}
2881	} else
2882	list_add(&page->lru, &n->slabs_partial);
2883	}
2884
2885	/* Try to find non-pfmemalloc slab if needed */
2886	static noinline struct page *get_valid_first_slab(struct kmem_cache_node *n,
2887	struct page *page, bool pfmemalloc)
2888	{
2889	if (!page)
2890	return NULL;
2891
2892	if (pfmemalloc)
2893	return page;
2894
2895	if (!PageSlabPfmemalloc(page))
2896	return page;
2897
2898	/* No need to keep pfmemalloc slab if we have enough free objects */
2899	if (n->free_objects > n->free_limit) {
2900	ClearPageSlabPfmemalloc(page);
2901	return page;
2902	}
2903
2904	/* Move pfmemalloc slab to the end of list to speed up next search */
2905	list_del(&page->lru);
2906	if (!page->active)
2907	list_add_tail(&page->lru, &n->slabs_free);
2908	else
2909	list_add_tail(&page->lru, &n->slabs_partial);
2910
2911	list_for_each_entry(page, &n->slabs_partial, lru) {
2912	if (!PageSlabPfmemalloc(page))
2913	return page;
2914	}
2915
2916	list_for_each_entry(page, &n->slabs_free, lru) {
2917	if (!PageSlabPfmemalloc(page))
2918	return page;
2919	}
2920
2921	return NULL;
2922	}
2923
2924	static struct page *get_first_slab(struct kmem_cache_node *n, bool pfmemalloc)
2925	{
2926	struct page *page;
2927
2928	page = list_first_entry_or_null(&n->slabs_partial,
2929	struct page, lru);
2930	if (!page) {
2931	n->free_touched = 1;
2932	page = list_first_entry_or_null(&n->slabs_free,
2933	struct page, lru);
2934	}
2935
2936	if (sk_memalloc_socks())
2937	return get_valid_first_slab(n, page, pfmemalloc);
2938
2939	return page;
2940	}
2941
2942	static noinline void *cache_alloc_pfmemalloc(struct kmem_cache *cachep,
2943	struct kmem_cache_node *n, gfp_t flags)
2944	{
2945	struct page *page;
2946	void *obj;
2947	void *list = NULL;
2948
2949	if (!gfp_pfmemalloc_allowed(flags))
2950	return NULL;
2951
2952	spin_lock(&n->list_lock);
2953	page = get_first_slab(n, true);
2954	if (!page) {
2955	spin_unlock(&n->list_lock);
2956	return NULL;
2957	}
2958
2959	obj = slab_get_obj(cachep, page);
2960	n->free_objects--;
2961
2962	fixup_slab_list(cachep, n, page, &list);
2963
2964	spin_unlock(&n->list_lock);
2965	fixup_objfreelist_debug(cachep, &list);
2966
2967	return obj;
2968	}
2969
2970	/*
2971	* Slab list should be fixed up by fixup_slab_list() for existing slab
2972	* or cache_grow_end() for new slab
2973	*/
2974	static __always_inline int alloc_block(struct kmem_cache *cachep,
2975	struct array_cache *ac, struct page *page, int batchcount)
2976	{
2977	/*
2978	* There must be at least one object available for
2979	* allocation.
2980	*/
2981	BUG_ON(page->active >= cachep->num);
2982
2983	while (page->active < cachep->num && batchcount--) {
2984	STATS_INC_ALLOCED(cachep);
2985	STATS_INC_ACTIVE(cachep);
2986	STATS_SET_HIGH(cachep);
2987
2988	ac->entry[ac->avail++] = slab_get_obj(cachep, page);
2989	}
2990
2991	return batchcount;
2992	}
2993
2994	static void *cache_alloc_refill(struct kmem_cache *cachep, gfp_t flags)
2995	{
2996	int batchcount;
2997	struct kmem_cache_node *n;
2998	struct array_cache *ac, *shared;
2999	int node;
3000	void *list = NULL;
3001	struct page *page;
3002
3003	check_irq_off();
3004	node = numa_mem_id();
3005
3006	ac = cpu_cache_get(cachep);
3007	batchcount = ac->batchcount;
3008	if (!ac->touched && batchcount > BATCHREFILL_LIMIT) {
3009	/*
3010	* If there was little recent activity on this cache, then
3011	* perform only a partial refill. Otherwise we could generate
3012	* refill bouncing.
3013	*/
3014	batchcount = BATCHREFILL_LIMIT;
3015	}
3016	n = get_node(cachep, node);
3017
3018	BUG_ON(ac->avail > 0 || !n);
3019	shared = READ_ONCE(n->shared);
3020	if (!n->free_objects && (!shared || !shared->avail))
3021	goto direct_grow;
3022
3023	spin_lock(&n->list_lock);
3024	shared = READ_ONCE(n->shared);
3025
3026	/* See if we can refill from the shared array */
3027	if (shared && transfer_objects(ac, shared, batchcount)) {
3028	shared->touched = 1;
3029	goto alloc_done;
3030	}
3031
3032	while (batchcount > 0) {
3033	/* Get slab alloc is to come from. */
3034	page = get_first_slab(n, false);
3035	if (!page)
3036	goto must_grow;
3037
3038	check_spinlock_acquired(cachep);
3039
3040	batchcount = alloc_block(cachep, ac, page, batchcount);
3041	fixup_slab_list(cachep, n, page, &list);
3042	}
3043
3044	must_grow:
3045	n->free_objects -= ac->avail;
3046	alloc_done:
3047	spin_unlock(&n->list_lock);
3048	fixup_objfreelist_debug(cachep, &list);
3049
3050	direct_grow:
3051	if (unlikely(!ac->avail)) {
3052	/* Check if we can use obj in pfmemalloc slab */
3053	if (sk_memalloc_socks()) {
3054	void *obj = cache_alloc_pfmemalloc(cachep, n, flags);
3055
3056	if (obj)
3057	return obj;
3058	}
3059
3060	page = cache_grow_begin(cachep, gfp_exact_node(flags), node);
3061
3062	/*
3063	* cache_grow_begin() can reenable interrupts,
3064	* then ac could change.
3065	*/
3066	ac = cpu_cache_get(cachep);
3067	if (!ac->avail && page)
3068	alloc_block(cachep, ac, page, batchcount);
3069	cache_grow_end(cachep, page);
3070
3071	if (!ac->avail)
3072	return NULL;
3073	}
3074	ac->touched = 1;
3075
3076	return ac->entry[--ac->avail];
3077	}
3078
3079	static inline void cache_alloc_debugcheck_before(struct kmem_cache *cachep,
3080	gfp_t flags)
3081	{
3082	might_sleep_if(gfpflags_allow_blocking(flags));
3083	}
3084
3085	#if DEBUG
3086	static void *cache_alloc_debugcheck_after(struct kmem_cache *cachep,
3087	gfp_t flags, void *objp, unsigned long caller)
3088	{
3089	if (!objp)
3090	return objp;
3091	if (cachep->flags & SLAB_POISON) {
3092	check_poison_obj(cachep, objp);
3093	slab_kernel_map(cachep, objp, 1, 0);
3094	poison_obj(cachep, objp, POISON_INUSE);
3095	}
3096	if (cachep->flags & SLAB_STORE_USER)
3097	*dbg_userword(cachep, objp) = (void *)caller;
3098
3099	if (cachep->flags & SLAB_RED_ZONE) {
3100	if (*dbg_redzone1(cachep, objp) != RED_INACTIVE ||
3101	*dbg_redzone2(cachep, objp) != RED_INACTIVE) {
3102	slab_error(cachep, "double free, or memory outside object was overwritten");
3103	pr_err("%px: redzone 1:0x%llx, redzone 2:0x%llx\n",
3104	objp, *dbg_redzone1(cachep, objp),
3105	*dbg_redzone2(cachep, objp));
3106	}
3107	*dbg_redzone1(cachep, objp) = RED_ACTIVE;
3108	*dbg_redzone2(cachep, objp) = RED_ACTIVE;
3109	}
3110
3111	objp += obj_offset(cachep);
3112	if (cachep->ctor && cachep->flags & SLAB_POISON)
3113	cachep->ctor(objp);
3114	if (ARCH_SLAB_MINALIGN &&
3115	((unsigned long)objp & (ARCH_SLAB_MINALIGN-1))) {
3116	pr_err("0x%px: not aligned to ARCH_SLAB_MINALIGN=%d\n",
3117	objp, (int)ARCH_SLAB_MINALIGN);
3118	}
3119	return objp;
3120	}
3121	#else
3122	#define cache_alloc_debugcheck_after(a,b,objp,d) (objp)
3123	#endif
3124
3125	static inline void *____cache_alloc(struct kmem_cache *cachep, gfp_t flags)
3126	{
3127	void *objp;
3128	struct array_cache *ac;
3129
3130	check_irq_off();
3131
3132	ac = cpu_cache_get(cachep);
3133	if (likely(ac->avail)) {
3134	ac->touched = 1;
3135	objp = ac->entry[--ac->avail];
3136
3137	STATS_INC_ALLOCHIT(cachep);
3138	goto out;
3139	}
3140
3141	STATS_INC_ALLOCMISS(cachep);
3142	objp = cache_alloc_refill(cachep, flags);
3143	/*
3144	* the 'ac' may be updated by cache_alloc_refill(),
3145	* and kmemleak_erase() requires its correct value.
3146	*/
3147	ac = cpu_cache_get(cachep);
3148
3149	out:
3150	/*
3151	* To avoid a false negative, if an object that is in one of the
3152	* per-CPU caches is leaked, we need to make sure kmemleak doesn't
3153	* treat the array pointers as a reference to the object.
3154	*/
3155	if (objp)
3156	kmemleak_erase(&ac->entry[ac->avail]);
3157	return objp;
3158	}
3159
3160	#ifdef CONFIG_NUMA
3161	/*
3162	* Try allocating on another node if PFA_SPREAD_SLAB is a mempolicy is set.
3163	*
3164	* If we are in_interrupt, then process context, including cpusets and
3165	* mempolicy, may not apply and should not be used for allocation policy.
3166	*/
3167	static void *alternate_node_alloc(struct kmem_cache *cachep, gfp_t flags)
3168	{
3169	int nid_alloc, nid_here;
3170
3171	if (in_interrupt() || (flags & __GFP_THISNODE))
3172	return NULL;
3173	nid_alloc = nid_here = numa_mem_id();
3174	if (cpuset_do_slab_mem_spread() && (cachep->flags & SLAB_MEM_SPREAD))
3175	nid_alloc = cpuset_slab_spread_node();
3176	else if (current->mempolicy)
3177	nid_alloc = mempolicy_slab_node();
3178	if (nid_alloc != nid_here)
3179	return ____cache_alloc_node(cachep, flags, nid_alloc);
3180	return NULL;
3181	}
3182
3183	/*
3184	* Fallback function if there was no memory available and no objects on a
3185	* certain node and fall back is permitted. First we scan all the
3186	* available node for available objects. If that fails then we
3187	* perform an allocation without specifying a node. This allows the page
3188	* allocator to do its reclaim / fallback magic. We then insert the
3189	* slab into the proper nodelist and then allocate from it.
3190	*/
3191	static void *fallback_alloc(struct kmem_cache *cache, gfp_t flags)
3192	{
3193	struct zonelist *zonelist;
3194	struct zoneref *z;
3195	struct zone *zone;
3196	enum zone_type high_zoneidx = gfp_zone(flags);
3197	void *obj = NULL;
3198	struct page *page;
3199	int nid;
3200	unsigned int cpuset_mems_cookie;
3201
3202	if (flags & __GFP_THISNODE)
3203	return NULL;
3204
3205	retry_cpuset:
3206	cpuset_mems_cookie = read_mems_allowed_begin();
3207	zonelist = node_zonelist(mempolicy_slab_node(), flags);
3208
3209	retry:
3210	/*
3211	* Look through allowed nodes for objects available
3212	* from existing per node queues.
3213	*/
3214	for_each_zone_zonelist(zone, z, zonelist, high_zoneidx) {
3215	nid = zone_to_nid(zone);
3216
3217	if (cpuset_zone_allowed(zone, flags) &&
3218	get_node(cache, nid) &&
3219	get_node(cache, nid)->free_objects) {
3220	obj = ____cache_alloc_node(cache,
3221	gfp_exact_node(flags), nid);
3222	if (obj)
3223	break;
3224	}
3225	}
3226
3227	if (!obj) {
3228	/*
3229	* This allocation will be performed within the constraints
3230	* of the current cpuset / memory policy requirements.
3231	* We may trigger various forms of reclaim on the allowed
3232	* set and go into memory reserves if necessary.
3233	*/
3234	page = cache_grow_begin(cache, flags, numa_mem_id());
3235	cache_grow_end(cache, page);
3236	if (page) {
3237	nid = page_to_nid(page);
3238	obj = ____cache_alloc_node(cache,
3239	gfp_exact_node(flags), nid);
3240
3241	/*
3242	* Another processor may allocate the objects in
3243	* the slab since we are not holding any locks.
3244	*/
3245	if (!obj)
3246	goto retry;
3247	}
3248	}
3249
3250	if (unlikely(!obj && read_mems_allowed_retry(cpuset_mems_cookie)))
3251	goto retry_cpuset;
3252	return obj;
3253	}
3254
3255	/*
3256	* A interface to enable slab creation on nodeid
3257	*/
3258	static void *____cache_alloc_node(struct kmem_cache *cachep, gfp_t flags,
3259	int nodeid)
3260	{
3261	struct page *page;
3262	struct kmem_cache_node *n;
3263	void *obj = NULL;
3264	void *list = NULL;
3265
3266	VM_BUG_ON(nodeid < 0 || nodeid >= MAX_NUMNODES);
3267	n = get_node(cachep, nodeid);
3268	BUG_ON(!n);
3269
3270	check_irq_off();
3271	spin_lock(&n->list_lock);
3272	page = get_first_slab(n, false);
3273	if (!page)
3274	goto must_grow;
3275
3276	check_spinlock_acquired_node(cachep, nodeid);
3277
3278	STATS_INC_NODEALLOCS(cachep);
3279	STATS_INC_ACTIVE(cachep);
3280	STATS_SET_HIGH(cachep);
3281
3282	BUG_ON(page->active == cachep->num);
3283
3284	obj = slab_get_obj(cachep, page);
3285	n->free_objects--;
3286
3287	fixup_slab_list(cachep, n, page, &list);
3288
3289	spin_unlock(&n->list_lock);
3290	fixup_objfreelist_debug(cachep, &list);
3291	return obj;
3292
3293	must_grow:
3294	spin_unlock(&n->list_lock);
3295	page = cache_grow_begin(cachep, gfp_exact_node(flags), nodeid);
3296	if (page) {
3297	/* This slab isn't counted yet so don't update free_objects */
3298	obj = slab_get_obj(cachep, page);
3299	}
3300	cache_grow_end(cachep, page);
3301
3302	return obj ? obj : fallback_alloc(cachep, flags);
3303	}
3304
3305	static __always_inline void *
3306	slab_alloc_node(struct kmem_cache *cachep, gfp_t flags, int nodeid,
3307	unsigned long caller)
3308	{
3309	unsigned long save_flags;
3310	void *ptr;
3311	int slab_node = numa_mem_id();
3312
3313	flags &= gfp_allowed_mask;
3314	cachep = slab_pre_alloc_hook(cachep, flags);
3315	if (unlikely(!cachep))
3316	return NULL;
3317
3318	cache_alloc_debugcheck_before(cachep, flags);
3319	local_irq_save(save_flags);
3320
3321	if (nodeid == NUMA_NO_NODE)
3322	nodeid = slab_node;
3323
3324	if (unlikely(!get_node(cachep, nodeid))) {
3325	/* Node not bootstrapped yet */
3326	ptr = fallback_alloc(cachep, flags);
3327	goto out;
3328	}
3329
3330	if (nodeid == slab_node) {
3331	/*
3332	* Use the locally cached objects if possible.
3333	* However ____cache_alloc does not allow fallback
3334	* to other nodes. It may fail while we still have
3335	* objects on other nodes available.
3336	*/
3337	ptr = ____cache_alloc(cachep, flags);
3338	if (ptr)
3339	goto out;
3340	}
3341	/* ___cache_alloc_node can fall back to other nodes */
3342	ptr = ____cache_alloc_node(cachep, flags, nodeid);
3343	out:
3344	local_irq_restore(save_flags);
3345	ptr = cache_alloc_debugcheck_after(cachep, flags, ptr, caller);
3346
3347	if (unlikely(flags & __GFP_ZERO) && ptr)
3348	memset(ptr, 0, cachep->object_size);
3349
3350	slab_post_alloc_hook(cachep, flags, 1, &ptr);
3351	return ptr;
3352	}
3353
3354	static __always_inline void *
3355	__do_cache_alloc(struct kmem_cache *cache, gfp_t flags)
3356	{
3357	void *objp;
3358
3359	if (current->mempolicy || cpuset_do_slab_mem_spread()) {
3360	objp = alternate_node_alloc(cache, flags);
3361	if (objp)
3362	goto out;
3363	}
3364	objp = ____cache_alloc(cache, flags);
3365
3366	/*
3367	* We may just have run out of memory on the local node.
3368	* ____cache_alloc_node() knows how to locate memory on other nodes
3369	*/
3370	if (!objp)
3371	objp = ____cache_alloc_node(cache, flags, numa_mem_id());
3372
3373	out:
3374	return objp;
3375	}
3376	#else
3377
3378	static __always_inline void *
3379	__do_cache_alloc(struct kmem_cache *cachep, gfp_t flags)
3380	{
3381	return ____cache_alloc(cachep, flags);
3382	}
3383
3384	#endif /* CONFIG_NUMA */
3385
3386	static __always_inline void *
3387	slab_alloc(struct kmem_cache *cachep, gfp_t flags, unsigned long caller)
3388	{
3389	unsigned long save_flags;
3390	void *objp;
3391
3392	flags &= gfp_allowed_mask;
3393	cachep = slab_pre_alloc_hook(cachep, flags);
3394	if (unlikely(!cachep))
3395	return NULL;
3396
3397	cache_alloc_debugcheck_before(cachep, flags);
3398	local_irq_save(save_flags);
3399	objp = __do_cache_alloc(cachep, flags);
3400	local_irq_restore(save_flags);
3401	objp = cache_alloc_debugcheck_after(cachep, flags, objp, caller);
3402	prefetchw(objp);
3403
3404	if (unlikely(flags & __GFP_ZERO) && objp)
3405	memset(objp, 0, cachep->object_size);
3406
3407	slab_post_alloc_hook(cachep, flags, 1, &objp);
3408	return objp;
3409	}
3410
3411	/*
3412	* Caller needs to acquire correct kmem_cache_node's list_lock
3413	* @list: List of detached free slabs should be freed by caller
3414	*/
3415	static void free_block(struct kmem_cache *cachep, void **objpp,
3416	int nr_objects, int node, struct list_head *list)
3417	{
3418	int i;
3419	struct kmem_cache_node *n = get_node(cachep, node);
3420	struct page *page;
3421
3422	n->free_objects += nr_objects;
3423
3424	for (i = 0; i < nr_objects; i++) {
3425	void *objp;
3426	struct page *page;
3427
3428	objp = objpp[i];
3429
3430	page = virt_to_head_page(objp);
3431	list_del(&page->lru);
3432	check_spinlock_acquired_node(cachep, node);
3433	slab_put_obj(cachep, page, objp);
3434	STATS_DEC_ACTIVE(cachep);
3435
3436	/* fixup slab chains */
3437	if (page->active == 0)
3438	list_add(&page->lru, &n->slabs_free);
3439	else {
3440	/* Unconditionally move a slab to the end of the
3441	* partial list on free - maximum time for the
3442	* other objects to be freed, too.
3443	*/
3444	list_add_tail(&page->lru, &n->slabs_partial);
3445	}
3446	}
3447
3448	while (n->free_objects > n->free_limit && !list_empty(&n->slabs_free)) {
3449	n->free_objects -= cachep->num;
3450
3451	page = list_last_entry(&n->slabs_free, struct page, lru);
3452	list_move(&page->lru, list);
3453	n->num_slabs--;
3454	}
3455	}
3456
3457	static void cache_flusharray(struct kmem_cache *cachep, struct array_cache *ac)
3458	{
3459	int batchcount;
3460	struct kmem_cache_node *n;
3461	int node = numa_mem_id();
3462	LIST_HEAD(list);
3463
3464	batchcount = ac->batchcount;
3465
3466	check_irq_off();
3467	n = get_node(cachep, node);
3468	spin_lock(&n->list_lock);
3469	if (n->shared) {
3470	struct array_cache *shared_array = n->shared;
3471	int max = shared_array->limit - shared_array->avail;
3472	if (max) {
3473	if (batchcount > max)
3474	batchcount = max;
3475	memcpy(&(shared_array->entry[shared_array->avail]),
3476	ac->entry, sizeof(void *) * batchcount);
3477	shared_array->avail += batchcount;
3478	goto free_done;
3479	}
3480	}
3481
3482	free_block(cachep, ac->entry, batchcount, node, &list);
3483	free_done:
3484	#if STATS
3485	{
3486	int i = 0;
3487	struct page *page;
3488
3489	list_for_each_entry(page, &n->slabs_free, lru) {
3490	BUG_ON(page->active);
3491
3492	i++;
3493	}
3494	STATS_SET_FREEABLE(cachep, i);
3495	}
3496	#endif
3497	spin_unlock(&n->list_lock);
3498	slabs_destroy(cachep, &list);
3499	ac->avail -= batchcount;
3500	memmove(ac->entry, &(ac->entry[batchcount]), sizeof(void *)*ac->avail);
3501	}
3502
3503	/*
3504	* Release an obj back to its cache. If the obj has a constructed state, it must
3505	* be in this state _before_ it is released. Called with disabled ints.
3506	*/
3507	static inline void __cache_free(struct kmem_cache *cachep, void *objp,
3508	unsigned long caller)
3509	{
3510	/* Put the object into the quarantine, don't touch it for now. */
3511	if (kasan_slab_free(cachep, objp))
3512	return;
3513
3514	___cache_free(cachep, objp, caller);
3515	}
3516
3517	void ___cache_free(struct kmem_cache *cachep, void *objp,
3518	unsigned long caller)
3519	{
3520	struct array_cache *ac = cpu_cache_get(cachep);
3521
3522	check_irq_off();
3523	kmemleak_free_recursive(objp, cachep->flags);
3524	objp = cache_free_debugcheck(cachep, objp, caller);
3525
3526	kmemcheck_slab_free(cachep, objp, cachep->object_size);
3527
3528	/*
3529	* Skip calling cache_free_alien() when the platform is not numa.
3530	* This will avoid cache misses that happen while accessing slabp (which
3531	* is per page memory reference) to get nodeid. Instead use a global
3532	* variable to skip the call, which is mostly likely to be present in
3533	* the cache.
3534	*/
3535	if (nr_online_nodes > 1 && cache_free_alien(cachep, objp))
3536	return;
3537
3538	if (ac->avail < ac->limit) {
3539	STATS_INC_FREEHIT(cachep);
3540	} else {
3541	STATS_INC_FREEMISS(cachep);
3542	cache_flusharray(cachep, ac);
3543	}
3544
3545	if (sk_memalloc_socks()) {
3546	struct page *page = virt_to_head_page(objp);
3547
3548	if (unlikely(PageSlabPfmemalloc(page))) {
3549	cache_free_pfmemalloc(cachep, page, objp);
3550	return;
3551	}
3552	}
3553
3554	ac->entry[ac->avail++] = objp;
3555	}
3556
3557	/**
3558	* kmem_cache_alloc - Allocate an object
3559	* @cachep: The cache to allocate from.
3560	* @flags: See kmalloc().
3561	*
3562	* Allocate an object from this cache. The flags are only relevant
3563	* if the cache has no available objects.
3564	*/
3565	void *kmem_cache_alloc(struct kmem_cache *cachep, gfp_t flags)
3566	{
3567	void *ret = slab_alloc(cachep, flags, _RET_IP_);
3568
3569	kasan_slab_alloc(cachep, ret, flags);
3570	trace_kmem_cache_alloc(_RET_IP_, ret,
3571	cachep->object_size, cachep->size, flags);
3572
3573	return ret;
3574	}
3575	EXPORT_SYMBOL(kmem_cache_alloc);
3576
3577	static __always_inline void
3578	cache_alloc_debugcheck_after_bulk(struct kmem_cache *s, gfp_t flags,
3579	size_t size, void **p, unsigned long caller)
3580	{
3581	size_t i;
3582
3583	for (i = 0; i < size; i++)
3584	p[i] = cache_alloc_debugcheck_after(s, flags, p[i], caller);
3585	}
3586
3587	int kmem_cache_alloc_bulk(struct kmem_cache *s, gfp_t flags, size_t size,
3588	void **p)
3589	{
3590	size_t i;
3591
3592	s = slab_pre_alloc_hook(s, flags);
3593	if (!s)
3594	return 0;
3595
3596	cache_alloc_debugcheck_before(s, flags);
3597
3598	local_irq_disable();
3599	for (i = 0; i < size; i++) {
3600	void *objp = __do_cache_alloc(s, flags);
3601
3602	if (unlikely(!objp))
3603	goto error;
3604	p[i] = objp;
3605	}
3606	local_irq_enable();
3607
3608	cache_alloc_debugcheck_after_bulk(s, flags, size, p, _RET_IP_);
3609
3610	/* Clear memory outside IRQ disabled section */
3611	if (unlikely(flags & __GFP_ZERO))
3612	for (i = 0; i < size; i++)
3613	memset(p[i], 0, s->object_size);
3614
3615	slab_post_alloc_hook(s, flags, size, p);
3616	/* FIXME: Trace call missing. Christoph would like a bulk variant */
3617	return size;
3618	error:
3619	local_irq_enable();
3620	cache_alloc_debugcheck_after_bulk(s, flags, i, p, _RET_IP_);
3621	slab_post_alloc_hook(s, flags, i, p);
3622	__kmem_cache_free_bulk(s, i, p);
3623	return 0;
3624	}
3625	EXPORT_SYMBOL(kmem_cache_alloc_bulk);
3626
3627	#ifdef CONFIG_TRACING
3628	void *
3629	kmem_cache_alloc_trace(struct kmem_cache *cachep, gfp_t flags, size_t size)
3630	{
3631	void *ret;
3632
3633	ret = slab_alloc(cachep, flags, _RET_IP_);
3634
3635	kasan_kmalloc(cachep, ret, size, flags);
3636	trace_kmalloc(_RET_IP_, ret,
3637	size, cachep->size, flags);
3638	return ret;
3639	}
3640	EXPORT_SYMBOL(kmem_cache_alloc_trace);
3641	#endif
3642
3643	#ifdef CONFIG_NUMA
3644	/**
3645	* kmem_cache_alloc_node - Allocate an object on the specified node
3646	* @cachep: The cache to allocate from.
3647	* @flags: See kmalloc().
3648	* @nodeid: node number of the target node.
3649	*
3650	* Identical to kmem_cache_alloc but it will allocate memory on the given
3651	* node, which can improve the performance for cpu bound structures.
3652	*
3653	* Fallback to other node is possible if __GFP_THISNODE is not set.
3654	*/
3655	void *kmem_cache_alloc_node(struct kmem_cache *cachep, gfp_t flags, int nodeid)
3656	{
3657	void *ret = slab_alloc_node(cachep, flags, nodeid, _RET_IP_);
3658
3659	kasan_slab_alloc(cachep, ret, flags);
3660	trace_kmem_cache_alloc_node(_RET_IP_, ret,
3661	cachep->object_size, cachep->size,
3662	flags, nodeid);
3663
3664	return ret;
3665	}
3666	EXPORT_SYMBOL(kmem_cache_alloc_node);
3667
3668	#ifdef CONFIG_TRACING
3669	void *kmem_cache_alloc_node_trace(struct kmem_cache *cachep,
3670	gfp_t flags,
3671	int nodeid,
3672	size_t size)
3673	{
3674	void *ret;
3675
3676	ret = slab_alloc_node(cachep, flags, nodeid, _RET_IP_);
3677
3678	kasan_kmalloc(cachep, ret, size, flags);
3679	trace_kmalloc_node(_RET_IP_, ret,
3680	size, cachep->size,
3681	flags, nodeid);
3682	return ret;
3683	}
3684	EXPORT_SYMBOL(kmem_cache_alloc_node_trace);
3685	#endif
3686
3687	static __always_inline void *
3688	__do_kmalloc_node(size_t size, gfp_t flags, int node, unsigned long caller)
3689	{
3690	struct kmem_cache *cachep;
3691	void *ret;
3692
3693	if (unlikely(size > KMALLOC_MAX_CACHE_SIZE))
3694	return NULL;
3695	cachep = kmalloc_slab(size, flags);
3696	if (unlikely(ZERO_OR_NULL_PTR(cachep)))
3697	return cachep;
3698	ret = kmem_cache_alloc_node_trace(cachep, flags, node, size);
3699	kasan_kmalloc(cachep, ret, size, flags);
3700
3701	return ret;
3702	}
3703
3704	void *__kmalloc_node(size_t size, gfp_t flags, int node)
3705	{
3706	return __do_kmalloc_node(size, flags, node, _RET_IP_);
3707	}
3708	EXPORT_SYMBOL(__kmalloc_node);
3709
3710	void *__kmalloc_node_track_caller(size_t size, gfp_t flags,
3711	int node, unsigned long caller)
3712	{
3713	return __do_kmalloc_node(size, flags, node, caller);
3714	}
3715	EXPORT_SYMBOL(__kmalloc_node_track_caller);
3716	#endif /* CONFIG_NUMA */
3717
3718	/**
3719	* __do_kmalloc - allocate memory
3720	* @size: how many bytes of memory are required.
3721	* @flags: the type of memory to allocate (see kmalloc).
3722	* @caller: function caller for debug tracking of the caller
3723	*/
3724	static __always_inline void *__do_kmalloc(size_t size, gfp_t flags,
3725	unsigned long caller)
3726	{
3727	struct kmem_cache *cachep;
3728	void *ret;
3729
3730	if (unlikely(size > KMALLOC_MAX_CACHE_SIZE))
3731	return NULL;
3732	cachep = kmalloc_slab(size, flags);
3733	if (unlikely(ZERO_OR_NULL_PTR(cachep)))
3734	return cachep;
3735	ret = slab_alloc(cachep, flags, caller);
3736
3737	kasan_kmalloc(cachep, ret, size, flags);
3738	trace_kmalloc(caller, ret,
3739	size, cachep->size, flags);
3740
3741	return ret;
3742	}
3743
3744	void *__kmalloc(size_t size, gfp_t flags)
3745	{
3746	return __do_kmalloc(size, flags, _RET_IP_);
3747	}
3748	EXPORT_SYMBOL(__kmalloc);
3749
3750	void *__kmalloc_track_caller(size_t size, gfp_t flags, unsigned long caller)
3751	{
3752	return __do_kmalloc(size, flags, caller);
3753	}
3754	EXPORT_SYMBOL(__kmalloc_track_caller);
3755
3756	/**
3757	* kmem_cache_free - Deallocate an object
3758	* @cachep: The cache the allocation was from.
3759	* @objp: The previously allocated object.
3760	*
3761	* Free an object which was previously allocated from this
3762	* cache.
3763	*/
3764	void kmem_cache_free(struct kmem_cache *cachep, void *objp)
3765	{
3766	unsigned long flags;
3767	cachep = cache_from_obj(cachep, objp);
3768	if (!cachep)
3769	return;
3770
3771	local_irq_save(flags);
3772	debug_check_no_locks_freed(objp, cachep->object_size);
3773	if (!(cachep->flags & SLAB_DEBUG_OBJECTS))
3774	debug_check_no_obj_freed(objp, cachep->object_size);
3775	__cache_free(cachep, objp, _RET_IP_);
3776	local_irq_restore(flags);
3777
3778	trace_kmem_cache_free(_RET_IP_, objp);
3779	}
3780	EXPORT_SYMBOL(kmem_cache_free);
3781
3782	void kmem_cache_free_bulk(struct kmem_cache *orig_s, size_t size, void **p)
3783	{
3784	struct kmem_cache *s;
3785	size_t i;
3786
3787	local_irq_disable();
3788	for (i = 0; i < size; i++) {
3789	void *objp = p[i];
3790
3791	if (!orig_s) /* called via kfree_bulk */
3792	s = virt_to_cache(objp);
3793	else
3794	s = cache_from_obj(orig_s, objp);
3795
3796	debug_check_no_locks_freed(objp, s->object_size);
3797	if (!(s->flags & SLAB_DEBUG_OBJECTS))
3798	debug_check_no_obj_freed(objp, s->object_size);
3799
3800	__cache_free(s, objp, _RET_IP_);
3801	}
3802	local_irq_enable();
3803
3804	/* FIXME: add tracing */
3805	}
3806	EXPORT_SYMBOL(kmem_cache_free_bulk);
3807
3808	/**
3809	* kfree - free previously allocated memory
3810	* @objp: pointer returned by kmalloc.
3811	*
3812	* If @objp is NULL, no operation is performed.
3813	*
3814	* Don't free memory not originally allocated by kmalloc()
3815	* or you will run into trouble.
3816	*/
3817	void kfree(const void *objp)
3818	{
3819	struct kmem_cache *c;
3820	unsigned long flags;
3821
3822	trace_kfree(_RET_IP_, objp);
3823
3824	if (unlikely(ZERO_OR_NULL_PTR(objp)))
3825	return;
3826	local_irq_save(flags);
3827	kfree_debugcheck(objp);
3828	c = virt_to_cache(objp);
3829	debug_check_no_locks_freed(objp, c->object_size);
3830
3831	debug_check_no_obj_freed(objp, c->object_size);
3832	__cache_free(c, (void *)objp, _RET_IP_);
3833	local_irq_restore(flags);
3834	}
3835	EXPORT_SYMBOL(kfree);
3836
3837	/*
3838	* This initializes kmem_cache_node or resizes various caches for all nodes.
3839	*/
3840	static int setup_kmem_cache_nodes(struct kmem_cache *cachep, gfp_t gfp)
3841	{
3842	int ret;
3843	int node;
3844	struct kmem_cache_node *n;
3845
3846	for_each_online_node(node) {
3847	ret = setup_kmem_cache_node(cachep, node, gfp, true);
3848	if (ret)
3849	goto fail;
3850
3851	}
3852
3853	return 0;
3854
3855	fail:
3856	if (!cachep->list.next) {
3857	/* Cache is not active yet. Roll back what we did */
3858	node--;
3859	while (node >= 0) {
3860	n = get_node(cachep, node);
3861	if (n) {
3862	kfree(n->shared);
3863	free_alien_cache(n->alien);
3864	kfree(n);
3865	cachep->node[node] = NULL;
3866	}
3867	node--;
3868	}
3869	}
3870	return -ENOMEM;
3871	}
3872
3873	/* Always called with the slab_mutex held */
3874	static int __do_tune_cpucache(struct kmem_cache *cachep, int limit,
3875	int batchcount, int shared, gfp_t gfp)
3876	{
3877	struct array_cache __percpu *cpu_cache, *prev;
3878	int cpu;
3879
3880	cpu_cache = alloc_kmem_cache_cpus(cachep, limit, batchcount);
3881	if (!cpu_cache)
3882	return -ENOMEM;
3883
3884	prev = cachep->cpu_cache;
3885	cachep->cpu_cache = cpu_cache;
3886	kick_all_cpus_sync();
3887
3888	check_irq_on();
3889	cachep->batchcount = batchcount;
3890	cachep->limit = limit;
3891	cachep->shared = shared;
3892
3893	if (!prev)
3894	goto setup_node;
3895
3896	for_each_online_cpu(cpu) {
3897	LIST_HEAD(list);
3898	int node;
3899	struct kmem_cache_node *n;
3900	struct array_cache *ac = per_cpu_ptr(prev, cpu);
3901
3902	node = cpu_to_mem(cpu);
3903	n = get_node(cachep, node);
3904	spin_lock_irq(&n->list_lock);
3905	free_block(cachep, ac->entry, ac->avail, node, &list);
3906	spin_unlock_irq(&n->list_lock);
3907	slabs_destroy(cachep, &list);
3908	}
3909	free_percpu(prev);
3910
3911	setup_node:
3912	return setup_kmem_cache_nodes(cachep, gfp);
3913	}
3914
3915	static int do_tune_cpucache(struct kmem_cache *cachep, int limit,
3916	int batchcount, int shared, gfp_t gfp)
3917	{
3918	int ret;
3919	struct kmem_cache *c;
3920
3921	ret = __do_tune_cpucache(cachep, limit, batchcount, shared, gfp);
3922
3923	if (slab_state < FULL)
3924	return ret;
3925
3926	if ((ret < 0) || !is_root_cache(cachep))
3927	return ret;
3928
3929	lockdep_assert_held(&slab_mutex);
3930	for_each_memcg_cache(c, cachep) {
3931	/* return value determined by the root cache only */
3932	__do_tune_cpucache(c, limit, batchcount, shared, gfp);
3933	}
3934
3935	return ret;
3936	}
3937
3938	/* Called with slab_mutex held always */
3939	static int enable_cpucache(struct kmem_cache *cachep, gfp_t gfp)
3940	{
3941	int err;
3942	int limit = 0;
3943	int shared = 0;
3944	int batchcount = 0;
3945
3946	err = cache_random_seq_create(cachep, cachep->num, gfp);
3947	if (err)
3948	goto end;
3949
3950	if (!is_root_cache(cachep)) {
3951	struct kmem_cache *root = memcg_root_cache(cachep);
3952	limit = root->limit;
3953	shared = root->shared;
3954	batchcount = root->batchcount;
3955	}
3956
3957	if (limit && shared && batchcount)
3958	goto skip_setup;
3959	/*
3960	* The head array serves three purposes:
3961	* - create a LIFO ordering, i.e. return objects that are cache-warm
3962	* - reduce the number of spinlock operations.
3963	* - reduce the number of linked list operations on the slab and
3964	* bufctl chains: array operations are cheaper.
3965	* The numbers are guessed, we should auto-tune as described by
3966	* Bonwick.
3967	*/
3968	if (cachep->size > 131072)
3969	limit = 1;
3970	else if (cachep->size > PAGE_SIZE)
3971	limit = 8;
3972	else if (cachep->size > 1024)
3973	limit = 24;
3974	else if (cachep->size > 256)
3975	limit = 54;
3976	else
3977	limit = 120;
3978
3979	/*
3980	* CPU bound tasks (e.g. network routing) can exhibit cpu bound
3981	* allocation behaviour: Most allocs on one cpu, most free operations
3982	* on another cpu. For these cases, an efficient object passing between
3983	* cpus is necessary. This is provided by a shared array. The array
3984	* replaces Bonwick's magazine layer.
3985	* On uniprocessor, it's functionally equivalent (but less efficient)
3986	* to a larger limit. Thus disabled by default.
3987	*/
3988	shared = 0;
3989	if (cachep->size <= PAGE_SIZE && num_possible_cpus() > 1)
3990	shared = 8;
3991
3992	#if DEBUG
3993	/*
3994	* With debugging enabled, large batchcount lead to excessively long
3995	* periods with disabled local interrupts. Limit the batchcount
3996	*/
3997	if (limit > 32)
3998	limit = 32;
3999	#endif
4000	batchcount = (limit + 1) / 2;
4001	skip_setup:
4002	err = do_tune_cpucache(cachep, limit, batchcount, shared, gfp);
4003	end:
4004	if (err)
4005	pr_err("enable_cpucache failed for %s, error %d\n",
4006	cachep->name, -err);
4007	return err;
4008	}
4009
4010	/*
4011	* Drain an array if it contains any elements taking the node lock only if
4012	* necessary. Note that the node listlock also protects the array_cache
4013	* if drain_array() is used on the shared array.
4014	*/
4015	static void drain_array(struct kmem_cache *cachep, struct kmem_cache_node *n,
4016	struct array_cache *ac, int node)
4017	{
4018	LIST_HEAD(list);
4019
4020	/* ac from n->shared can be freed if we don't hold the slab_mutex. */
4021	check_mutex_acquired();
4022
4023	if (!ac || !ac->avail)
4024	return;
4025
4026	if (ac->touched) {
4027	ac->touched = 0;
4028	return;
4029	}
4030
4031	spin_lock_irq(&n->list_lock);
4032	drain_array_locked(cachep, ac, node, false, &list);
4033	spin_unlock_irq(&n->list_lock);
4034
4035	slabs_destroy(cachep, &list);
4036	}
4037
4038	/**
4039	* cache_reap - Reclaim memory from caches.
4040	* @w: work descriptor
4041	*
4042	* Called from workqueue/eventd every few seconds.
4043	* Purpose:
4044	* - clear the per-cpu caches for this CPU.
4045	* - return freeable pages to the main free memory pool.
4046	*
4047	* If we cannot acquire the cache chain mutex then just give up - we'll try
4048	* again on the next iteration.
4049	*/
4050	static void cache_reap(struct work_struct *w)
4051	{
4052	struct kmem_cache *searchp;
4053	struct kmem_cache_node *n;
4054	int node = numa_mem_id();
4055	struct delayed_work *work = to_delayed_work(w);
4056
4057	if (!mutex_trylock(&slab_mutex))
4058	/* Give up. Setup the next iteration. */
4059	goto out;
4060
4061	list_for_each_entry(searchp, &slab_caches, list) {
4062	check_irq_on();
4063
4064	/*
4065	* We only take the node lock if absolutely necessary and we
4066	* have established with reasonable certainty that
4067	* we can do some work if the lock was obtained.
4068	*/
4069	n = get_node(searchp, node);
4070
4071	reap_alien(searchp, n);
4072
4073	drain_array(searchp, n, cpu_cache_get(searchp), node);
4074
4075	/*
4076	* These are racy checks but it does not matter
4077	* if we skip one check or scan twice.
4078	*/
4079	if (time_after(n->next_reap, jiffies))
4080	goto next;
4081
4082	n->next_reap = jiffies + REAPTIMEOUT_NODE;
4083
4084	drain_array(searchp, n, n->shared, node);
4085
4086	if (n->free_touched)
4087	n->free_touched = 0;
4088	else {
4089	int freed;
4090
4091	freed = drain_freelist(searchp, n, (n->free_limit +
4092	5 * searchp->num - 1) / (5 * searchp->num));
4093	STATS_ADD_REAPED(searchp, freed);
4094	}
4095	next:
4096	cond_resched();
4097	}
4098	check_irq_on();
4099	mutex_unlock(&slab_mutex);
4100	next_reap_node();
4101	out:
4102	/* Set up the next iteration */
4103	schedule_delayed_work_on(smp_processor_id(), work,
4104	round_jiffies_relative(REAPTIMEOUT_AC));
4105	}
4106
4107	#ifdef CONFIG_SLABINFO
4108	void get_slabinfo(struct kmem_cache *cachep, struct slabinfo *sinfo)
4109	{
4110	struct page *page;
4111	unsigned long active_objs;
4112	unsigned long num_objs;
4113	unsigned long active_slabs = 0;
4114	unsigned long num_slabs, free_objects = 0, shared_avail = 0;
4115	unsigned long num_slabs_partial = 0, num_slabs_free = 0;
4116	unsigned long num_slabs_full = 0;
4117	const char *name;
4118	char *error = NULL;
4119	int node;
4120	struct kmem_cache_node *n;
4121
4122	active_objs = 0;
4123	num_slabs = 0;
4124	for_each_kmem_cache_node(cachep, node, n) {
4125
4126	check_irq_on();
4127	spin_lock_irq(&n->list_lock);
4128
4129	num_slabs += n->num_slabs;
4130
4131	list_for_each_entry(page, &n->slabs_partial, lru) {
4132	if (page->active == cachep->num && !error)
4133	error = "slabs_partial accounting error";
4134	if (!page->active && !error)
4135	error = "slabs_partial accounting error";
4136	active_objs += page->active;
4137	num_slabs_partial++;
4138	}
4139
4140	list_for_each_entry(page, &n->slabs_free, lru) {
4141	if (page->active && !error)
4142	error = "slabs_free accounting error";
4143	num_slabs_free++;
4144	}
4145
4146	free_objects += n->free_objects;
4147	if (n->shared)
4148	shared_avail += n->shared->avail;
4149
4150	spin_unlock_irq(&n->list_lock);
4151	}
4152	num_objs = num_slabs * cachep->num;
4153	active_slabs = num_slabs - num_slabs_free;
4154	num_slabs_full = num_slabs - (num_slabs_partial + num_slabs_free);
4155	active_objs += (num_slabs_full * cachep->num);
4156
4157	if (num_objs - active_objs != free_objects && !error)
4158	error = "free_objects accounting error";
4159
4160	name = cachep->name;
4161	if (error)
4162	pr_err("slab: cache %s error: %s\n", name, error);
4163
4164	sinfo->active_objs = active_objs;
4165	sinfo->num_objs = num_objs;
4166	sinfo->active_slabs = active_slabs;
4167	sinfo->num_slabs = num_slabs;
4168	sinfo->shared_avail = shared_avail;
4169	sinfo->limit = cachep->limit;
4170	sinfo->batchcount = cachep->batchcount;
4171	sinfo->shared = cachep->shared;
4172	sinfo->objects_per_slab = cachep->num;
4173	sinfo->cache_order = cachep->gfporder;
4174	}
4175
4176	void slabinfo_show_stats(struct seq_file *m, struct kmem_cache *cachep)
4177	{
4178	#if STATS
4179	{ /* node stats */
4180	unsigned long high = cachep->high_mark;
4181	unsigned long allocs = cachep->num_allocations;
4182	unsigned long grown = cachep->grown;
4183	unsigned long reaped = cachep->reaped;
4184	unsigned long errors = cachep->errors;
4185	unsigned long max_freeable = cachep->max_freeable;
4186	unsigned long node_allocs = cachep->node_allocs;
4187	unsigned long node_frees = cachep->node_frees;
4188	unsigned long overflows = cachep->node_overflow;
4189
4190	seq_printf(m, " : globalstat %7lu %6lu %5lu %4lu %4lu %4lu %4lu %4lu %4lu",
4191	allocs, high, grown,
4192	reaped, errors, max_freeable, node_allocs,
4193	node_frees, overflows);
4194	}
4195	/* cpu stats */
4196	{
4197	unsigned long allochit = atomic_read(&cachep->allochit);
4198	unsigned long allocmiss = atomic_read(&cachep->allocmiss);
4199	unsigned long freehit = atomic_read(&cachep->freehit);
4200	unsigned long freemiss = atomic_read(&cachep->freemiss);
4201
4202	seq_printf(m, " : cpustat %6lu %6lu %6lu %6lu",
4203	allochit, allocmiss, freehit, freemiss);
4204	}
4205	#endif
4206	}
4207
4208	#define MAX_SLABINFO_WRITE 128
4209	/**
4210	* slabinfo_write - Tuning for the slab allocator
4211	* @file: unused
4212	* @buffer: user buffer
4213	* @count: data length
4214	* @ppos: unused
4215	*/
4216	ssize_t slabinfo_write(struct file *file, const char __user *buffer,
4217	size_t count, loff_t *ppos)
4218	{
4219	char kbuf[MAX_SLABINFO_WRITE + 1], *tmp;
4220	int limit, batchcount, shared, res;
4221	struct kmem_cache *cachep;
4222
4223	if (count > MAX_SLABINFO_WRITE)
4224	return -EINVAL;
4225	if (copy_from_user(&kbuf, buffer, count))
4226	return -EFAULT;
4227	kbuf[MAX_SLABINFO_WRITE] = '\0';
4228
4229	tmp = strchr(kbuf, ' ');
4230	if (!tmp)
4231	return -EINVAL;
4232	*tmp = '\0';
4233	tmp++;
4234	if (sscanf(tmp, " %d %d %d", &limit, &batchcount, &shared) != 3)
4235	return -EINVAL;
4236
4237	/* Find the cache in the chain of caches. */
4238	mutex_lock(&slab_mutex);
4239	res = -EINVAL;
4240	list_for_each_entry(cachep, &slab_caches, list) {
4241	if (!strcmp(cachep->name, kbuf)) {
4242	if (limit < 1 || batchcount < 1 ||
4243	batchcount > limit || shared < 0) {
4244	res = 0;
4245	} else {
4246	res = do_tune_cpucache(cachep, limit,
4247	batchcount, shared,
4248	GFP_KERNEL);
4249	}
4250	break;
4251	}
4252	}
4253	mutex_unlock(&slab_mutex);
4254	if (res >= 0)
4255	res = count;
4256	return res;
4257	}
4258
4259	#ifdef CONFIG_DEBUG_SLAB_LEAK
4260
4261	static inline int add_caller(unsigned long *n, unsigned long v)
4262	{
4263	unsigned long *p;
4264	int l;
4265	if (!v)
4266	return 1;
4267	l = n[1];
4268	p = n + 2;
4269	while (l) {
4270	int i = l/2;
4271	unsigned long *q = p + 2 * i;
4272	if (*q == v) {
4273	q[1]++;
4274	return 1;
4275	}
4276	if (*q > v) {
4277	l = i;
4278	} else {
4279	p = q + 2;
4280	l -= i + 1;
4281	}
4282	}
4283	if (++n[1] == n[0])
4284	return 0;
4285	memmove(p + 2, p, n[1] * 2 * sizeof(unsigned long) - ((void *)p - (void *)n));
4286	p[0] = v;
4287	p[1] = 1;
4288	return 1;
4289	}
4290
4291	static void handle_slab(unsigned long *n, struct kmem_cache *c,
4292	struct page *page)
4293	{
4294	void *p;
4295	int i, j;
4296	unsigned long v;
4297
4298	if (n[0] == n[1])
4299	return;
4300	for (i = 0, p = page->s_mem; i < c->num; i++, p += c->size) {
4301	bool active = true;
4302
4303	for (j = page->active; j < c->num; j++) {
4304	if (get_free_obj(page, j) == i) {
4305	active = false;
4306	break;
4307	}
4308	}
4309
4310	if (!active)
4311	continue;
4312
4313	/*
4314	* probe_kernel_read() is used for DEBUG_PAGEALLOC. page table
4315	* mapping is established when actual object allocation and
4316	* we could mistakenly access the unmapped object in the cpu
4317	* cache.
4318	*/
4319	if (probe_kernel_read(&v, dbg_userword(c, p), sizeof(v)))
4320	continue;
4321
4322	if (!add_caller(n, v))
4323	return;
4324	}
4325	}
4326
4327	static void show_symbol(struct seq_file *m, unsigned long address)
4328	{
4329	#ifdef CONFIG_KALLSYMS
4330	unsigned long offset, size;
4331	char modname[MODULE_NAME_LEN], name[KSYM_NAME_LEN];
4332
4333	if (lookup_symbol_attrs(address, &size, &offset, modname, name) == 0) {
4334	seq_printf(m, "%s+%#lx/%#lx", name, offset, size);
4335	if (modname[0])
4336	seq_printf(m, " [%s]", modname);
4337	return;
4338	}
4339	#endif
4340	seq_printf(m, "%px", (void *)address);
4341	}
4342
4343	static int leaks_show(struct seq_file *m, void *p)
4344	{
4345	struct kmem_cache *cachep = list_entry(p, struct kmem_cache, list);
4346	struct page *page;
4347	struct kmem_cache_node *n;
4348	const char *name;
4349	unsigned long *x = m->private;
4350	int node;
4351	int i;
4352
4353	if (!(cachep->flags & SLAB_STORE_USER))
4354	return 0;
4355	if (!(cachep->flags & SLAB_RED_ZONE))
4356	return 0;
4357
4358	/*
4359	* Set store_user_clean and start to grab stored user information
4360	* for all objects on this cache. If some alloc/free requests comes
4361	* during the processing, information would be wrong so restart
4362	* whole processing.
4363	*/
4364	do {
4365	set_store_user_clean(cachep);
4366	drain_cpu_caches(cachep);
4367
4368	x[1] = 0;
4369
4370	for_each_kmem_cache_node(cachep, node, n) {
4371
4372	check_irq_on();
4373	spin_lock_irq(&n->list_lock);
4374
4375	list_for_each_entry(page, &n->slabs_full, lru)
4376	handle_slab(x, cachep, page);
4377	list_for_each_entry(page, &n->slabs_partial, lru)
4378	handle_slab(x, cachep, page);
4379	spin_unlock_irq(&n->list_lock);
4380	}
4381	} while (!is_store_user_clean(cachep));
4382
4383	name = cachep->name;
4384	if (x[0] == x[1]) {
4385	/* Increase the buffer size */
4386	mutex_unlock(&slab_mutex);
4387	m->private = kzalloc(x[0] * 4 * sizeof(unsigned long), GFP_KERNEL);
4388	if (!m->private) {
4389	/* Too bad, we are really out */
4390	m->private = x;
4391	mutex_lock(&slab_mutex);
4392	return -ENOMEM;
4393	}
4394	*(unsigned long *)m->private = x[0] * 2;
4395	kfree(x);
4396	mutex_lock(&slab_mutex);
4397	/* Now make sure this entry will be retried */
4398	m->count = m->size;
4399	return 0;
4400	}
4401	for (i = 0; i < x[1]; i++) {
4402	seq_printf(m, "%s: %lu ", name, x[2*i+3]);
4403	show_symbol(m, x[2*i+2]);
4404	seq_putc(m, '\n');
4405	}
4406
4407	return 0;
4408	}
4409
4410	static const struct seq_operations slabstats_op = {
4411	.start = slab_start,
4412	.next = slab_next,
4413	.stop = slab_stop,
4414	.show = leaks_show,
4415	};
4416
4417	static int slabstats_open(struct inode *inode, struct file *file)
4418	{
4419	unsigned long *n;
4420
4421	n = __seq_open_private(file, &slabstats_op, PAGE_SIZE);
4422	if (!n)
4423	return -ENOMEM;
4424
4425	*n = PAGE_SIZE / (2 * sizeof(unsigned long));
4426
4427	return 0;
4428	}
4429
4430	static const struct file_operations proc_slabstats_operations = {
4431	.open = slabstats_open,
4432	.read = seq_read,
4433	.llseek = seq_lseek,
4434	.release = seq_release_private,
4435	};
4436	#endif
4437
4438	static int __init slab_proc_init(void)
4439	{
4440	#ifdef CONFIG_DEBUG_SLAB_LEAK
4441	proc_create("slab_allocators", 0, NULL, &proc_slabstats_operations);
4442	#endif
4443	return 0;
4444	}
4445	module_init(slab_proc_init);
4446	#endif
4447
4448	#ifdef CONFIG_HARDENED_USERCOPY
4449	/*
4450	* Rejects objects that are incorrectly sized.
4451	*
4452	* Returns NULL if check passes, otherwise const char * to name of cache
4453	* to indicate an error.
4454	*/
4455	const char *__check_heap_object(const void *ptr, unsigned long n,
4456	struct page *page)
4457	{
4458	struct kmem_cache *cachep;
4459	unsigned int objnr;
4460	unsigned long offset;
4461
4462	/* Find and validate object. */
4463	cachep = page->slab_cache;
4464	objnr = obj_to_index(cachep, page, (void *)ptr);
4465	BUG_ON(objnr >= cachep->num);
4466
4467	/* Find offset within object. */
4468	offset = ptr - index_to_obj(cachep, page, objnr) - obj_offset(cachep);
4469
4470	/* Allow address range falling entirely within object size. */
4471	if (offset <= cachep->object_size && n <= cachep->object_size - offset)
4472	return NULL;
4473
4474	return cachep->name;
4475	}
4476	#endif /* CONFIG_HARDENED_USERCOPY */
4477
4478	/**
4479	* ksize - get the actual amount of memory allocated for a given object
4480	* @objp: Pointer to the object
4481	*
4482	* kmalloc may internally round up allocations and return more memory
4483	* than requested. ksize() can be used to determine the actual amount of
4484	* memory allocated. The caller may use this additional memory, even though
4485	* a smaller amount of memory was initially specified with the kmalloc call.
4486	* The caller must guarantee that objp points to a valid object previously
4487	* allocated with either kmalloc() or kmem_cache_alloc(). The object
4488	* must not be freed during the duration of the call.
4489	*/
4490	size_t ksize(const void *objp)
4491	{
4492	size_t size;
4493
4494	BUG_ON(!objp);
4495	if (unlikely(objp == ZERO_SIZE_PTR))
4496	return 0;
4497
4498	size = virt_to_cache(objp)->object_size;
4499	/* We assume that ksize callers could use the whole allocated area,
4500	* so we need to unpoison this area.
4501	*/
4502	kasan_unpoison_shadow(objp, size);
4503
4504	return size;
4505	}
4506	EXPORT_SYMBOL(ksize);
4507